Use Chroma as new default syntax highlighter

If you want to use Pygments, set `pygmentsUseClassic=true` in your site config.

Fixes #3888

commands/genchromastyles.go

@@ -0,0 +1,70 @@
+// Copyright 2017-present The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package commands
+
+import (
+	"os"
+
+	"github.com/alecthomas/chroma"
+	"github.com/alecthomas/chroma/formatters/html"
+	"github.com/alecthomas/chroma/styles"
+	"github.com/spf13/cobra"
+)
+
+type genChromaStyles struct {
+	style          string
+	highlightStyle string
+	linesStyle     string
+	cmd            *cobra.Command
+}
+
+// TODO(bep) highlight
+func createGenChromaStyles() *genChromaStyles {
+	g := &genChromaStyles{
+		cmd: &cobra.Command{
+			Use:   "chromastyles",
+			Short: "Generate CSS stylesheet for the Chroma code highlighter",
+			Long: `Generate CSS stylesheet for the Chroma code highlighter for a given style. This stylesheet is needed if pygmentsUseClasses is enabled in config.
+
+See https://help.farbox.com/pygments.html for preview of available styles`,
+		},
+	}
+
+	g.cmd.RunE = func(cmd *cobra.Command, args []string) error {
+		return g.generate()
+	}
+
+	g.cmd.PersistentFlags().StringVar(&g.style, "style", "friendly", "highlighter style (see https://help.farbox.com/pygments.html)")
+	g.cmd.PersistentFlags().StringVar(&g.highlightStyle, "highlightStyle", "bg:#ffffcc", "style used for highlighting lines (see https://github.com/alecthomas/chroma)")
+	g.cmd.PersistentFlags().StringVar(&g.linesStyle, "linesStyle", "", "style used for line numbers (see https://github.com/alecthomas/chroma)")
+
+	return g
+}
+
+func (g *genChromaStyles) generate() error {
+	builder := styles.Get(g.style).Builder()
+	if g.highlightStyle != "" {
+		builder.Add(chroma.LineHighlight, g.highlightStyle)
+	}
+	if g.linesStyle != "" {
+		builder.Add(chroma.LineNumbers, g.linesStyle)
+	}
+	style, err := builder.Build()
+	if err != nil {
+		return err
+	}
+	formatter := html.New(html.WithClasses())
+	formatter.WriteCSS(os.Stdout, style)
+	return nil
+}

commands/hugo.go

@@ -199,6 +199,7 @@ func AddCommands() {
 	genCmd.AddCommand(gendocCmd)
 	genCmd.AddCommand(genmanCmd)
 	genCmd.AddCommand(createGenDocsHelper().cmd)
+	genCmd.AddCommand(createGenChromaStyles().cmd)
 }
 
 // initHugoBuilderFlags initializes all common flags, typically used by the

deps/deps.go

@@ -114,6 +114,11 @@ func New(cfg DepsCfg) (*Deps, error) {
 		return nil, err
 	}
 
+	contentSpec, err := helpers.NewContentSpec(cfg.Language)
+	if err != nil {
+		return nil, err
+	}
+
 	d := &Deps{
 		Fs:                  fs,
 		Log:                 logger,
@@ -121,7 +126,7 @@ func New(cfg DepsCfg) (*Deps, error) {
 		translationProvider: cfg.TranslationProvider,
 		WithTemplate:        cfg.WithTemplate,
 		PathSpec:            ps,
-		ContentSpec:         helpers.NewContentSpec(cfg.Language),
+		ContentSpec:         contentSpec,
 		Cfg:                 cfg.Language,
 		Language:            cfg.Language,
 	}
@@ -139,7 +144,11 @@ func (d Deps) ForLanguage(l *helpers.Language) (*Deps, error) {
 		return nil, err
 	}
 
-	d.ContentSpec = helpers.NewContentSpec(l)
+	d.ContentSpec, err = helpers.NewContentSpec(l)
+	if err != nil {
+		return nil, err
+	}
+
 	d.Cfg = l
 	d.Language = l
 

helpers/content.go

@@ -48,19 +48,49 @@ type ContentSpec struct {
 	footnoteAnchorPrefix       string
 	footnoteReturnLinkContents string
 
+	Highlight            func(code, lang, optsStr string) (string, error)
+	defatultPygmentsOpts map[string]string
+
 	cfg config.Provider
 }
 
 // NewContentSpec returns a ContentSpec initialized
 // with the appropriate fields from the given config.Provider.
-func NewContentSpec(cfg config.Provider) *ContentSpec {
+func NewContentSpec(cfg config.Provider) (*ContentSpec, error) {
-	return &ContentSpec{
+	spec := &ContentSpec{
 		blackfriday:                cfg.GetStringMap("blackfriday"),
 		footnoteAnchorPrefix:       cfg.GetString("footnoteAnchorPrefix"),
 		footnoteReturnLinkContents: cfg.GetString("footnoteReturnLinkContents"),
 
 		cfg: cfg,
 	}
+
+	// Highlighting setup
+	options, err := parseDefaultPygmentsOpts(cfg)
+	if err != nil {
+		return nil, err
+	}
+	spec.defatultPygmentsOpts = options
+
+	// Use the Pygmentize on path if present
+	useClassic := false
+	h := newHiglighters(spec)
+
+	if cfg.GetBool("pygmentsUseClassic") {
+		if !hasPygments() {
+			jww.WARN.Println("Highlighting with pygmentsUseClassic set requires Pygments to be installed and in the path")
+		} else {
+			useClassic = true
+		}
+	}
+
+	if useClassic {
+		spec.Highlight = h.pygmentsHighlight
+	} else {
+		spec.Highlight = h.chromaHighlight
+	}
+
+	return spec, nil
 }
 
 // Blackfriday holds configuration values for Blackfriday rendering.
@@ -198,7 +228,7 @@ func BytesToHTML(b []byte) template.HTML {
 }
 
 // getHTMLRenderer creates a new Blackfriday HTML Renderer with the given configuration.
-func (c ContentSpec) getHTMLRenderer(defaultFlags int, ctx *RenderingContext) blackfriday.Renderer {
+func (c *ContentSpec) getHTMLRenderer(defaultFlags int, ctx *RenderingContext) blackfriday.Renderer {
 	renderParameters := blackfriday.HtmlRendererParameters{
 		FootnoteAnchorPrefix:       c.footnoteAnchorPrefix,
 		FootnoteReturnLinkContents: c.footnoteReturnLinkContents,
@@ -248,6 +278,7 @@ func (c ContentSpec) getHTMLRenderer(defaultFlags int, ctx *RenderingContext) bl
 	}
 
 	return &HugoHTMLRenderer{
+		cs:               c,
 		RenderingContext: ctx,
 		Renderer:         blackfriday.HtmlRendererWithParameters(htmlFlags, "", "", renderParameters),
 	}
@@ -299,7 +330,7 @@ func (c ContentSpec) markdownRender(ctx *RenderingContext) []byte {
 }
 
 // getMmarkHTMLRenderer creates a new mmark HTML Renderer with the given configuration.
-func (c ContentSpec) getMmarkHTMLRenderer(defaultFlags int, ctx *RenderingContext) mmark.Renderer {
+func (c *ContentSpec) getMmarkHTMLRenderer(defaultFlags int, ctx *RenderingContext) mmark.Renderer {
 	renderParameters := mmark.HtmlRendererParameters{
 		FootnoteAnchorPrefix:       c.footnoteAnchorPrefix,
 		FootnoteReturnLinkContents: c.footnoteReturnLinkContents,
@@ -320,8 +351,9 @@ func (c ContentSpec) getMmarkHTMLRenderer(defaultFlags int, ctx *RenderingContex
 	htmlFlags |= mmark.HTML_FOOTNOTE_RETURN_LINKS
 
 	return &HugoMmarkHTMLRenderer{
-		mmark.HtmlRendererWithParameters(htmlFlags, "", "", renderParameters),
+		cs:       c,
-		c.cfg,
+		Renderer: mmark.HtmlRendererWithParameters(htmlFlags, "", "", renderParameters),
+		Cfg:      c.cfg,
 	}
 }
 

helpers/content_renderer.go

@@ -16,6 +16,7 @@ package helpers
 import (
 	"bytes"
 	"html"
+	"strings"
 
 	"github.com/gohugoio/hugo/config"
 	"github.com/miekg/mmark"
@@ -25,6 +26,7 @@ import (
 // HugoHTMLRenderer wraps a blackfriday.Renderer, typically a blackfriday.Html
 // Enabling Hugo to customise the rendering experience
 type HugoHTMLRenderer struct {
+	cs *ContentSpec
 	*RenderingContext
 	blackfriday.Renderer
 }
@@ -34,8 +36,9 @@ type HugoHTMLRenderer struct {
 func (r *HugoHTMLRenderer) BlockCode(out *bytes.Buffer, text []byte, lang string) {
 	if r.Cfg.GetBool("pygmentsCodeFences") && (lang != "" || r.Cfg.GetBool("pygmentsCodeFencesGuessSyntax")) {
 		opts := r.Cfg.GetString("pygmentsOptions")
-		str := html.UnescapeString(string(text))
+		str := strings.Trim(html.UnescapeString(string(text)), "\n\r")
-		out.WriteString(Highlight(r.RenderingContext.Cfg, str, lang, opts))
+		highlighted, _ := r.cs.Highlight(str, lang, opts)
+		out.WriteString(highlighted)
 	} else {
 		r.Renderer.BlockCode(out, text, lang)
 	}
@@ -88,6 +91,7 @@ func (r *HugoHTMLRenderer) List(out *bytes.Buffer, text func() bool, flags int)
 // HugoMmarkHTMLRenderer wraps a mmark.Renderer, typically a mmark.html,
 // enabling Hugo to customise the rendering experience.
 type HugoMmarkHTMLRenderer struct {
+	cs *ContentSpec
 	mmark.Renderer
 	Cfg config.Provider
 }
@@ -96,8 +100,9 @@ type HugoMmarkHTMLRenderer struct {
 // Pygments is used if it is setup to handle code fences.
 func (r *HugoMmarkHTMLRenderer) BlockCode(out *bytes.Buffer, text []byte, lang string, caption []byte, subfigure bool, callouts bool) {
 	if r.Cfg.GetBool("pygmentsCodeFences") && (lang != "" || r.Cfg.GetBool("pygmentsCodeFencesGuessSyntax")) {
-		str := html.UnescapeString(string(text))
+		str := strings.Trim(html.UnescapeString(string(text)), "\n\r")
-		out.WriteString(Highlight(r.Cfg, str, lang, ""))
+		highlighted, _ := r.cs.Highlight(str, lang, "")
+		out.WriteString(highlighted)
 	} else {
 		r.Renderer.BlockCode(out, text, lang, caption, subfigure, callouts)
 	}

helpers/content_renderer_test.go

@@ -19,6 +19,7 @@ import (
 	"testing"
 
 	"github.com/spf13/viper"
+	"github.com/stretchr/testify/require"
 )
 
 // Renders a codeblock using Blackfriday
@@ -42,11 +43,7 @@ func (c ContentSpec) renderWithMmark(input string) string {
 }
 
 func TestCodeFence(t *testing.T) {
-
+	assert := require.New(t)
-	if !HasPygments() {
-		t.Skip("Skipping Pygments test as Pygments is not installed or available.")
-		return
-	}
 
 	type test struct {
 		enabled         bool
@@ -55,36 +52,39 @@ func TestCodeFence(t *testing.T) {
 
 	// Pygments 2.0 and 2.1 have slightly different outputs so only do partial matching
 	data := []test{
-		{true, "<html></html>", `(?s)^<div class="highlight"><pre><code class="language-html" data-lang="html">.*?</code></pre></div>\n$`},
+		{true, "<html></html>", `(?s)^<div class="highlight">\n?<pre.*><code class="language-html" data-lang="html">.*?</code></pre>\n?</div>\n?$`},
-		{false, "<html></html>", `(?s)^<pre><code class="language-html">.*?</code></pre>\n$`},
+		{false, "<html></html>", `(?s)^<pre.*><code class="language-html">.*?</code></pre>\n$`},
 	}
 
-	for i, d := range data {
+	for _, useClassic := range []bool{false, true} {
-		v := viper.New()
+		for i, d := range data {
+			v := viper.New()
+			v.Set("pygmentsStyle", "monokai")
+			v.Set("pygmentsUseClasses", true)
+			v.Set("pygmentsCodeFences", d.enabled)
+			v.Set("pygmentsUseClassic", useClassic)
 
-		v.Set("pygmentsStyle", "monokai")
+			c, err := NewContentSpec(v)
-		v.Set("pygmentsUseClasses", true)
+			assert.NoError(err)
-		v.Set("pygmentsCodeFences", d.enabled)
 
-		c := NewContentSpec(v)
+			result := c.render(d.input)
 
-		result := c.render(d.input)
+			expectedRe, err := regexp.Compile(d.expected)
 
-		expectedRe, err := regexp.Compile(d.expected)
+			if err != nil {
+				t.Fatal("Invalid regexp", err)
+			}
+			matched := expectedRe.MatchString(result)
 
-		if err != nil {
+			if !matched {
-			t.Fatal("Invalid regexp", err)
+				t.Errorf("Test %d failed. BlackFriday enabled:%t, Expected:\n%q got:\n%q", i, d.enabled, d.expected, result)
-		}
+			}
-		matched := expectedRe.MatchString(result)
 
-		if !matched {
+			result = c.renderWithMmark(d.input)
-			t.Errorf("Test %d failed. BlackFriday enabled:%t, Expected:\n%q got:\n%q", i, d.enabled, d.expected, result)
+			matched = expectedRe.MatchString(result)
-		}
+			if !matched {
-
+				t.Errorf("Test %d failed. Mmark enabled:%t, Expected:\n%q got:\n%q", i, d.enabled, d.expected, result)
-		result = c.renderWithMmark(d.input)
+			}
-		matched = expectedRe.MatchString(result)
-		if !matched {
-			t.Errorf("Test %d failed. Mmark enabled:%t, Expected:\n%q got:\n%q", i, d.enabled, d.expected, result)
 		}
 	}
 }

helpers/pygments.go

@@ -21,9 +21,18 @@ import (
 	"io/ioutil"
 	"os/exec"
 	"path/filepath"
+	"regexp"
 	"sort"
+	"strconv"
 	"strings"
 
+	"github.com/alecthomas/chroma"
+	"github.com/alecthomas/chroma/formatters"
+	"github.com/alecthomas/chroma/formatters/html"
+	"github.com/alecthomas/chroma/lexers"
+	"github.com/alecthomas/chroma/styles"
+	bp "github.com/gohugoio/hugo/bufferpool"
+
 	"github.com/gohugoio/hugo/config"
 	"github.com/gohugoio/hugo/hugofs"
 	jww "github.com/spf13/jwalterweatherman"
@@ -31,27 +40,62 @@ import (
 
 const pygmentsBin = "pygmentize"
 
-// HasPygments checks to see if Pygments is installed and available
+// TODO(bep) document chroma -s perldoc --html --html-styles
+// hasPygments checks to see if Pygments is installed and available
 // on the system.
-func HasPygments() bool {
+func hasPygments() bool {
 	if _, err := exec.LookPath(pygmentsBin); err != nil {
 		return false
 	}
 	return true
 }
 
-// Highlight takes some code and returns highlighted code.
+type highlighters struct {
-func Highlight(cfg config.Provider, code, lang, optsStr string) string {
+	cs          *ContentSpec
-	if !HasPygments() {
+	ignoreCache bool
-		jww.WARN.Println("Highlighting requires Pygments to be installed and in the path")
+	cacheDir    string
-		return code
+}
+
+func newHiglighters(cs *ContentSpec) highlighters {
+	return highlighters{cs: cs, ignoreCache: cs.cfg.GetBool("ignoreCache"), cacheDir: cs.cfg.GetString("cacheDir")}
+}
+
+func (h highlighters) chromaHighlight(code, lang, optsStr string) (string, error) {
+	opts, err := h.cs.parsePygmentsOpts(optsStr)
+	if err != nil {
+		jww.ERROR.Print(err.Error())
+		return code, err
 	}
 
-	options, err := parsePygmentsOpts(cfg, optsStr)
+	style, found := opts["style"]
+	if !found || style == "" {
+		style = "friendly"
+	}
+
+	f, err := h.cs.chromaFormatterFromOptions(opts)
+	if err != nil {
+		jww.ERROR.Print(err.Error())
+		return code, err
+	}
+
+	b := bp.GetBuffer()
+	defer bp.PutBuffer(b)
+
+	err = chromaHighlight(b, code, lang, style, f)
+	if err != nil {
+		jww.ERROR.Print(err.Error())
+		return code, err
+	}
+
+	return h.injectCodeTag(`<div class="highlight">`+b.String()+"</div>", lang), nil
+}
+
+func (h highlighters) pygmentsHighlight(code, lang, optsStr string) (string, error) {
+	options, err := h.cs.createPygmentsOptionsString(optsStr)
 
 	if err != nil {
 		jww.ERROR.Print(err.Error())
-		return code
+		return code, nil
 	}
 
 	// Try to read from cache first
@@ -62,32 +106,30 @@ func Highlight(cfg config.Provider, code, lang, optsStr string) string {
 
 	fs := hugofs.Os
 
-	ignoreCache := cfg.GetBool("ignoreCache")
-	cacheDir := cfg.GetString("cacheDir")
 	var cachefile string
 
-	if !ignoreCache && cacheDir != "" {
+	if !h.ignoreCache && h.cacheDir != "" {
-		cachefile = filepath.Join(cacheDir, fmt.Sprintf("pygments-%x", hash.Sum(nil)))
+		cachefile = filepath.Join(h.cacheDir, fmt.Sprintf("pygments-%x", hash.Sum(nil)))
 
 		exists, err := Exists(cachefile, fs)
 		if err != nil {
 			jww.ERROR.Print(err.Error())
-			return code
+			return code, nil
 		}
 		if exists {
 			f, err := fs.Open(cachefile)
 			if err != nil {
 				jww.ERROR.Print(err.Error())
-				return code
+				return code, nil
 			}
 
 			s, err := ioutil.ReadAll(f)
 			if err != nil {
 				jww.ERROR.Print(err.Error())
-				return code
+				return code, nil
 			}
 
-			return string(s)
+			return string(s), nil
 		}
 	}
 
@@ -109,26 +151,58 @@ func Highlight(cfg config.Provider, code, lang, optsStr string) string {
 
 	if err := cmd.Run(); err != nil {
 		jww.ERROR.Print(stderr.String())
-		return code
+		return code, err
 	}
 
 	str := string(normalizeExternalHelperLineFeeds([]byte(out.String())))
 
-	// inject code tag into Pygments output
+	str = h.injectCodeTag(str, lang)
-	if lang != "" && strings.Contains(str, "<pre>") {
-		codeTag := fmt.Sprintf(`<pre><code class="language-%s" data-lang="%s">`, lang, lang)
-		str = strings.Replace(str, "<pre>", codeTag, 1)
-		str = strings.Replace(str, "</pre>", "</code></pre>", 1)
-	}
 
-	if !ignoreCache && cachefile != "" {
+	if !h.ignoreCache && cachefile != "" {
 		// Write cache file
 		if err := WriteToDisk(cachefile, strings.NewReader(str), fs); err != nil {
 			jww.ERROR.Print(stderr.String())
 		}
 	}
 
-	return str
+	return str, nil
+}
+
+var preRe = regexp.MustCompile(`(?s)(.*?<pre.*?>)(.*?)(</pre>)`)
+
+func (h highlighters) injectCodeTag(code, lang string) string {
+	if lang == "" {
+		return code
+	}
+	codeTag := fmt.Sprintf(`<code class="language-%s" data-lang="%s">`, lang, lang)
+	return preRe.ReplaceAllString(code, fmt.Sprintf("$1%s$2</code>$3", codeTag))
+}
+
+func chromaHighlight(w io.Writer, source, lexer, style string, f chroma.Formatter) error {
+	l := lexers.Get(lexer)
+	if l == nil {
+		l = lexers.Analyse(source)
+	}
+	if l == nil {
+		l = lexers.Fallback
+	}
+	l = chroma.Coalesce(l)
+
+	if f == nil {
+		f = formatters.Fallback
+	}
+
+	s := styles.Get(style)
+	if s == nil {
+		s = styles.Fallback
+	}
+
+	it, err := l.Tokenise(nil, source)
+	if err != nil {
+		return err
+	}
+
+	return f.Format(w, s, it)
 }
 
 var pygmentsKeywords = make(map[string]bool)
@@ -158,23 +232,30 @@ func init() {
 	pygmentsKeywords["startinline"] = true
 }
 
-func parseOptions(options map[string]string, in string) error {
+func parseOptions(defaults map[string]string, in string) (map[string]string, error) {
 	in = strings.Trim(in, " ")
+	opts := make(map[string]string)
+
+	if defaults != nil {
+		for k, v := range defaults {
+			opts[k] = v
+		}
+	}
 
 	if in == "" {
-		return nil
+		return opts, nil
 	}
 
 	for _, v := range strings.Split(in, ",") {
 		keyVal := strings.Split(v, "=")
 		key := strings.ToLower(strings.Trim(keyVal[0], " "))
 		if len(keyVal) != 2 || !pygmentsKeywords[key] {
-			return fmt.Errorf("invalid Pygments option: %s", key)
+			return opts, fmt.Errorf("invalid Pygments option: %s", key)
 		}
-		options[key] = keyVal[1]
+		opts[key] = keyVal[1]
 	}
 
-	return nil
+	return opts, nil
 }
 
 func createOptionsString(options map[string]string) string {
@@ -196,8 +277,7 @@ func createOptionsString(options map[string]string) string {
 }
 
 func parseDefaultPygmentsOpts(cfg config.Provider) (map[string]string, error) {
-	options := make(map[string]string)
+	options, err := parseOptions(nil, cfg.GetString("pygmentsOptions"))
-	err := parseOptions(options, cfg.GetString("pygmentsOptions"))
 	if err != nil {
 		return nil, err
 	}
@@ -222,16 +302,100 @@ func parseDefaultPygmentsOpts(cfg config.Provider) (map[string]string, error) {
 	return options, nil
 }
 
-func parsePygmentsOpts(cfg config.Provider, in string) (string, error) {
+func (cs *ContentSpec) chromaFormatterFromOptions(pygmentsOpts map[string]string) (chroma.Formatter, error) {
-	options, err := parseDefaultPygmentsOpts(cfg)
+	var options = []html.Option{html.TabWidth(4)}
-	if err != nil {
+
-		return "", err
+	if pygmentsOpts["noclasses"] == "false" {
+		options = append(options, html.WithClasses())
 	}
 
-	err = parseOptions(options, in)
+	if pygmentsOpts["linenos"] != "" {
-	if err != nil {
+		options = append(options, html.WithLineNumbers())
-		return "", err
 	}
 
-	return createOptionsString(options), nil
+	startLineStr := pygmentsOpts["linenostart"]
+	var startLine = 1
+	if startLineStr != "" {
+
+		line, err := strconv.Atoi(strings.TrimSpace(startLineStr))
+		if err == nil {
+			startLine = line
+			options = append(options, html.BaseLineNumber(startLine))
+		}
+	}
+
+	hlLines := pygmentsOpts["hl_lines"]
+
+	if hlLines != "" {
+		ranges, err := hlLinesToRanges(startLine, hlLines)
+
+		if err == nil {
+			options = append(options, html.HighlightLines(ranges))
+		}
+	}
+
+	return html.New(options...), nil
+}
+
+func (cs *ContentSpec) parsePygmentsOpts(in string) (map[string]string, error) {
+	opts, err := parseOptions(cs.defatultPygmentsOpts, in)
+	if err != nil {
+		return nil, err
+	}
+	return opts, nil
+
+}
+
+func (cs *ContentSpec) createPygmentsOptionsString(in string) (string, error) {
+	opts, err := cs.parsePygmentsOpts(in)
+	if err != nil {
+		return "", err
+	}
+	return createOptionsString(opts), nil
+}
+
+// startLine compansates for https://github.com/alecthomas/chroma/issues/30
+func hlLinesToRanges(startLine int, s string) ([][2]int, error) {
+	var ranges [][2]int
+	s = strings.TrimSpace(s)
+
+	if s == "" {
+		return ranges, nil
+	}
+
+	// Variants:
+	// 1 2 3 4
+	// 1-2 3-4
+	// 1-2 3
+	// 1 3-4
+	// 1    3-4
+	fields := strings.Split(s, " ")
+	for _, field := range fields {
+		field = strings.TrimSpace(field)
+		if field == "" {
+			continue
+		}
+		numbers := strings.Split(field, "-")
+		var r [2]int
+		first, err := strconv.Atoi(numbers[0])
+		if err != nil {
+			return ranges, err
+		}
+		first = first + startLine - 1
+		r[0] = first
+		if len(numbers) > 1 {
+			second, err := strconv.Atoi(numbers[1])
+			if err != nil {
+				return ranges, err
+			}
+			second = second + startLine - 1
+			r[1] = second
+		} else {
+			r[1] = first
+		}
+
+		ranges = append(ranges, r)
+	}
+	return ranges, nil
+
 }

helpers/pygments_test.go

@@ -14,12 +14,19 @@
 package helpers
 
 import (
+	"fmt"
+	"reflect"
 	"testing"
 
+	"github.com/alecthomas/chroma/formatters/html"
+
 	"github.com/spf13/viper"
+	"github.com/stretchr/testify/require"
 )
 
 func TestParsePygmentsArgs(t *testing.T) {
+	assert := require.New(t)
+
 	for i, this := range []struct {
 		in                 string
 		pygmentsStyle      string
@@ -38,8 +45,10 @@ func TestParsePygmentsArgs(t *testing.T) {
 		v := viper.New()
 		v.Set("pygmentsStyle", this.pygmentsStyle)
 		v.Set("pygmentsUseClasses", this.pygmentsUseClasses)
+		spec, err := NewContentSpec(v)
+		assert.NoError(err)
 
-		result1, err := parsePygmentsOpts(v, this.in)
+		result1, err := spec.createPygmentsOptionsString(this.in)
 		if b, ok := this.expect1.(bool); ok && !b {
 			if err == nil {
 				t.Errorf("[%d] parsePygmentArgs didn't return an expected error", i)
@@ -58,6 +67,8 @@ func TestParsePygmentsArgs(t *testing.T) {
 }
 
 func TestParseDefaultPygmentsArgs(t *testing.T) {
+	assert := require.New(t)
+
 	expect := "encoding=utf8,noclasses=false,style=foo"
 
 	for i, this := range []struct {
@@ -83,7 +94,10 @@ func TestParseDefaultPygmentsArgs(t *testing.T) {
 			v.Set("pygmentsUseClasses", b)
 		}
 
-		result, err := parsePygmentsOpts(v, this.in)
+		spec, err := NewContentSpec(v)
+		assert.NoError(err)
+
+		result, err := spec.createPygmentsOptionsString(this.in)
 		if err != nil {
 			t.Errorf("[%d] parsePygmentArgs failed: %s", i, err)
 			continue
@@ -93,3 +107,186 @@ func TestParseDefaultPygmentsArgs(t *testing.T) {
 		}
 	}
 }
+
+type chromaInfo struct {
+	classes            bool
+	lineNumbers        bool
+	highlightRangesLen int
+	highlightRangesStr string
+	baseLineNumber     int
+}
+
+func formatterChromaInfo(f *html.Formatter) chromaInfo {
+	v := reflect.ValueOf(f).Elem()
+	c := chromaInfo{}
+	// Hack:
+	c.classes = v.FieldByName("classes").Bool()
+	c.lineNumbers = v.FieldByName("lineNumbers").Bool()
+	c.baseLineNumber = int(v.FieldByName("baseLineNumber").Int())
+	vv := v.FieldByName("highlightRanges")
+	c.highlightRangesLen = vv.Len()
+	c.highlightRangesStr = fmt.Sprint(vv)
+
+	return c
+}
+
+func TestChromaHTMLHighlight(t *testing.T) {
+	assert := require.New(t)
+
+	v := viper.New()
+	v.Set("pygmentsUseClasses", true)
+	spec, err := NewContentSpec(v)
+	assert.NoError(err)
+
+	result, err := spec.Highlight(`echo "Hello"`, "bash", "")
+	assert.NoError(err)
+
+	assert.Contains(result, `<code class="language-bash" data-lang="bash"><span class="s7d2">echo</span> <span class="sc1c">&#34;Hello&#34;</span></code>`)
+
+}
+
+func TestChromaHTMLFormatterFromOptions(t *testing.T) {
+	assert := require.New(t)
+
+	for i, this := range []struct {
+		in                 string
+		pygmentsStyle      interface{}
+		pygmentsUseClasses interface{}
+		pygmentsOptions    string
+		assert             func(c chromaInfo)
+	}{
+		{"", "monokai", true, "style=manni,noclasses=true", func(c chromaInfo) {
+			assert.True(c.classes)
+			assert.False(c.lineNumbers)
+			assert.Equal(0, c.highlightRangesLen)
+
+		}},
+		{"", nil, nil, "style=monokai,noclasses=false", func(c chromaInfo) {
+			assert.True(c.classes)
+		}},
+		{"linenos=sure,hl_lines=1 2 3", nil, nil, "style=monokai,noclasses=false", func(c chromaInfo) {
+			assert.True(c.classes)
+			assert.True(c.lineNumbers)
+			assert.Equal(3, c.highlightRangesLen)
+			assert.Equal("[[1 1] [2 2] [3 3]]", c.highlightRangesStr)
+			assert.Equal(1, c.baseLineNumber)
+		}},
+		{"linenos=sure,hl_lines=1,linenostart=4", nil, nil, "style=monokai,noclasses=false", func(c chromaInfo) {
+			assert.True(c.classes)
+			assert.True(c.lineNumbers)
+			assert.Equal(1, c.highlightRangesLen)
+			// This compansates for https://github.com/alecthomas/chroma/issues/30
+			assert.Equal("[[4 4]]", c.highlightRangesStr)
+			assert.Equal(4, c.baseLineNumber)
+		}},
+		{"style=monokai,noclasses=false", nil, nil, "style=manni,noclasses=true", func(c chromaInfo) {
+			assert.True(c.classes)
+		}},
+		{"style=monokai,noclasses=true", "friendly", false, "style=manni,noclasses=false", func(c chromaInfo) {
+			assert.False(c.classes)
+		}},
+	} {
+		v := viper.New()
+
+		v.Set("pygmentsOptions", this.pygmentsOptions)
+
+		if s, ok := this.pygmentsStyle.(string); ok {
+			v.Set("pygmentsStyle", s)
+		}
+
+		if b, ok := this.pygmentsUseClasses.(bool); ok {
+			v.Set("pygmentsUseClasses", b)
+		}
+
+		spec, err := NewContentSpec(v)
+		assert.NoError(err)
+
+		opts, err := spec.parsePygmentsOpts(this.in)
+		if err != nil {
+			t.Fatalf("[%d] parsePygmentsOpts failed: %s", i, err)
+		}
+
+		chromaFormatter, err := spec.chromaFormatterFromOptions(opts)
+		if err != nil {
+			t.Fatalf("[%d] chromaFormatterFromOptions failed: %s", i, err)
+		}
+
+		this.assert(formatterChromaInfo(chromaFormatter.(*html.Formatter)))
+	}
+}
+
+func TestHlLinesToRanges(t *testing.T) {
+	var zero [][2]int
+
+	for _, this := range []struct {
+		in        string
+		startLine int
+		expected  interface{}
+	}{
+		{"", 1, zero},
+		{"1 4", 1, [][2]int{[2]int{1, 1}, [2]int{4, 4}}},
+		{"1 4", 2, [][2]int{[2]int{2, 2}, [2]int{5, 5}}},
+		{"1-4 5-8", 1, [][2]int{[2]int{1, 4}, [2]int{5, 8}}},
+		{" 1   4 ", 1, [][2]int{[2]int{1, 1}, [2]int{4, 4}}},
+		{"1-4    5-8 ", 1, [][2]int{[2]int{1, 4}, [2]int{5, 8}}},
+		{"1-4 5", 1, [][2]int{[2]int{1, 4}, [2]int{5, 5}}},
+		{"4 5-9", 1, [][2]int{[2]int{4, 4}, [2]int{5, 9}}},
+		{" 1 -4 5 - 8  ", 1, true},
+		{"a b", 1, true},
+	} {
+		got, err := hlLinesToRanges(this.startLine, this.in)
+
+		if expectErr, ok := this.expected.(bool); ok && expectErr {
+			if err == nil {
+				t.Fatal("No error")
+			}
+		} else if err != nil {
+			t.Fatalf("Got error: %s", err)
+		} else if !reflect.DeepEqual(this.expected, got) {
+			t.Fatalf("Expected\n%v but got\n%v", this.expected, got)
+		}
+	}
+}
+
+func BenchmarkChromaHighlight(b *testing.B) {
+	assert := require.New(b)
+	v := viper.New()
+
+	v.Set("pygmentsstyle", "trac")
+	v.Set("pygmentsuseclasses", false)
+	v.Set("pygmentsuseclassic", false)
+
+	code := `// GetTitleFunc returns a func that can be used to transform a string to
+// title case.
+//
+// The supported styles are
+//
+// - "Go" (strings.Title)
+// - "AP" (see https://www.apstylebook.com/)
+// - "Chicago" (see http://www.chicagomanualofstyle.org/home.html)
+//
+// If an unknown or empty style is provided, AP style is what you get.
+func GetTitleFunc(style string) func(s string) string {
+  switch strings.ToLower(style) {
+  case "go":
+    return strings.Title
+  case "chicago":
+    tc := transform.NewTitleConverter(transform.ChicagoStyle)
+    return tc.Title
+  default:
+    tc := transform.NewTitleConverter(transform.APStyle)
+    return tc.Title
+  }
+}
+`
+
+	spec, err := NewContentSpec(v)
+	assert.NoError(err)
+
+	for i := 0; i < b.N; i++ {
+		_, err := spec.Highlight(code, "go", "linenos=inline,hl_lines=8 15-17")
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

helpers/testhelpers_test.go

@@ -34,5 +34,9 @@ func newTestCfg(fs *hugofs.Fs) *viper.Viper {
 
 func newTestContentSpec() *ContentSpec {
 	v := viper.New()
-	return NewContentSpec(v)
+	spec, err := NewContentSpec(v)
+	if err != nil {
+		panic(err)
+	}
+	return spec
 }

hugolib/config.go

@@ -16,11 +16,12 @@ package hugolib
 import (
 	"fmt"
 
+	"io"
+	"strings"
+
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/spf13/afero"
 	"github.com/spf13/viper"
-	"io"
-	"strings"
 )
 
 // LoadConfig loads Hugo configuration into a new Viper and then adds
@@ -84,9 +85,12 @@ func LoadConfig(fs afero.Fs, relativeSourcePath, configFilename string) (*viper.
 	return v, nil
 }
 
-func loadDefaultSettingsFor(v *viper.Viper) {
+func loadDefaultSettingsFor(v *viper.Viper) error {
 
-	c := helpers.NewContentSpec(v)
+	c, err := helpers.NewContentSpec(v)
+	if err != nil {
+		return err
+	}
 
 	v.SetDefault("cleanDestinationDir", false)
 	v.SetDefault("watch", false)
@@ -120,6 +124,7 @@ func loadDefaultSettingsFor(v *viper.Viper) {
 	v.SetDefault("pygmentsStyle", "monokai")
 	v.SetDefault("pygmentsUseClasses", false)
 	v.SetDefault("pygmentsCodeFences", false)
+	v.SetDefault("pygmentsUseClassic", false)
 	v.SetDefault("pygmentsOptions", "")
 	v.SetDefault("disableLiveReload", false)
 	v.SetDefault("pluralizeListTitles", true)
@@ -146,4 +151,6 @@ func loadDefaultSettingsFor(v *viper.Viper) {
 	v.SetDefault("ignoreFiles", make([]string, 0))
 	v.SetDefault("disableAliases", false)
 	v.SetDefault("debug", false)
+
+	return nil
 }

hugolib/embedded_shortcodes_test.go

@@ -24,7 +24,6 @@ import (
 
 	"github.com/gohugoio/hugo/deps"
 
-	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/tpl"
 	"github.com/stretchr/testify/require"
 )
@@ -80,22 +79,18 @@ func doTestShortcodeCrossrefs(t *testing.T, relative bool) {
 func TestShortcodeHighlight(t *testing.T) {
 	t.Parallel()
 
-	if !helpers.HasPygments() {
-		t.Skip("Skip test as Pygments is not installed")
-	}
-
 	for _, this := range []struct {
 		in, expected string
 	}{
 		{`{{< highlight java >}}
 void do();
 {{< /highlight >}}`,
-			"(?s)<div class=\"highlight\" style=\"background: #ffffff\"><pre style=\"line-height: 125%\">.*?void</span> do().*?</pre></div>\n",
+			`(?s)<div class="highlight"><pre style="background-color:#fff;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-java"`,
 		},
 		{`{{< highlight java "style=friendly" >}}
 void do();
 {{< /highlight >}}`,
-			"(?s)<div class=\"highlight\" style=\"background: #f0f0f0\"><pre style=\"line-height: 125%\">.*?void</span>.*?do</span>.*?().*?</pre></div>\n",
+			`(?s)<div class="highlight"><pre style="background-color:#f0f0f0;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-java" data-lang="java">`,
 		},
 	} {
 

hugolib/shortcode_test.go

@@ -482,7 +482,7 @@ e`,
 		// #2223 pygments
 		{"sect/doc6.md", "\n```bash\nb: {{< b >}} c: {{% c %}}\n```\n",
 			filepath.FromSlash("public/sect/doc6/index.html"),
-			"b: b c: c\n</code></pre></div>\n"},
+			`<span class="s1f40">b: b c: c`},
 		// #2249
 		{"sect/doc7.ad", `_Shortcodes:_ *b: {{< b >}} c: {{% c %}}*`,
 			filepath.FromSlash("public/sect/doc7/index.html"),
@@ -561,7 +561,7 @@ tags:
 		} else if strings.HasSuffix(test.contentPath, ".rst") && !helpers.HasRst() {
 			fmt.Println("Skip Rst test case as no rst2html present.")
 			continue
-		} else if strings.Contains(test.expected, "code") && !helpers.HasPygments() {
+		} else if strings.Contains(test.expected, "code") {
 			fmt.Println("Skip Pygments test case as no pygments present.")
 			continue
 		}

hugolib/site.go

@@ -291,7 +291,9 @@ func NewSite(cfg deps.DepsCfg) (*Site, error) {
 // Note: This is mainly used in single site tests.
 func NewSiteDefaultLang(withTemplate ...func(templ tpl.TemplateHandler) error) (*Site, error) {
 	v := viper.New()
-	loadDefaultSettingsFor(v)
+	if err := loadDefaultSettingsFor(v); err != nil {
+		return nil, err
+	}
 	return newSiteForLang(helpers.NewDefaultLanguage(v), withTemplate...)
 }
 
@@ -300,7 +302,9 @@ func NewSiteDefaultLang(withTemplate ...func(templ tpl.TemplateHandler) error) (
 // Note: This is mainly used in single site tests.
 func NewEnglishSite(withTemplate ...func(templ tpl.TemplateHandler) error) (*Site, error) {
 	v := viper.New()
-	loadDefaultSettingsFor(v)
+	if err := loadDefaultSettingsFor(v); err != nil {
+		return nil, err
+	}
 	return newSiteForLang(helpers.NewLanguage("en", v), withTemplate...)
 }
 

tpl/collections/collections_test.go

@@ -776,10 +776,14 @@ type TstX struct {
 func newDeps(cfg config.Provider) *deps.Deps {
 	l := helpers.NewLanguage("en", cfg)
 	l.Set("i18nDir", "i18n")
+	cs, err := helpers.NewContentSpec(l)
+	if err != nil {
+		panic(err)
+	}
 	return &deps.Deps{
 		Cfg:         cfg,
 		Fs:          hugofs.NewMem(l),
-		ContentSpec: helpers.NewContentSpec(l),
+		ContentSpec: cs,
 		Log:         jww.NewNotepad(jww.LevelError, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime),
 	}
 }

tpl/data/resources_test.go

@@ -166,9 +166,13 @@ func TestScpGetRemoteParallel(t *testing.T) {
 func newDeps(cfg config.Provider) *deps.Deps {
 	l := helpers.NewLanguage("en", cfg)
 	l.Set("i18nDir", "i18n")
+	cs, err := helpers.NewContentSpec(l)
+	if err != nil {
+		panic(err)
+	}
 	return &deps.Deps{
 		Cfg:         cfg,
 		Fs:          hugofs.NewMem(l),
-		ContentSpec: helpers.NewContentSpec(l),
+		ContentSpec: cs,
 	}
 }

tpl/transform/transform.go

@@ -55,7 +55,8 @@ func (ns *Namespace) Highlight(s interface{}, lang, opts string) (template.HTML,
 		return "", err
 	}
 
-	return template.HTML(helpers.Highlight(ns.deps.Cfg, html.UnescapeString(ss), lang, opts)), nil
+	highlighted, _ := ns.deps.ContentSpec.Highlight(html.UnescapeString(ss), lang, opts)
+	return template.HTML(highlighted), nil
 }
 
 // HTMLEscape returns a copy of s with reserved HTML characters escaped.

tpl/transform/transform_test.go

@@ -226,9 +226,14 @@ func TestPlainify(t *testing.T) {
 func newDeps(cfg config.Provider) *deps.Deps {
 	l := helpers.NewLanguage("en", cfg)
 	l.Set("i18nDir", "i18n")
+	cs, err := helpers.NewContentSpec(l)
+	if err != nil {
+		panic(err)
+	}
+
 	return &deps.Deps{
 		Cfg:         cfg,
 		Fs:          hugofs.NewMem(l),
-		ContentSpec: helpers.NewContentSpec(l),
+		ContentSpec: cs,
 	}
 }

vendor/vendor.json

@@ -20,6 +20,36 @@
 			"revision": "bbf7a2afc14f93e1e0a5c06df524fbd75e5031e5",
 			"revisionTime": "2017-03-24T14:02:28Z"
 		},
+		{
+			"checksumSHA1": "Aq9XVBGDFH92BXKVPK+rexqDkTo=",
+			"path": "github.com/alecthomas/chroma",
+			"revision": "b0295f66bdb7c61d54906003d7649185794e21b4",
+			"revisionTime": "2017-09-25T05:25:32Z"
+		},
+		{
+			"checksumSHA1": "Q/9AbXGrFHtlZB6tyoYUq1ipvqU=",
+			"path": "github.com/alecthomas/chroma/formatters",
+			"revision": "b0295f66bdb7c61d54906003d7649185794e21b4",
+			"revisionTime": "2017-09-25T05:25:32Z"
+		},
+		{
+			"checksumSHA1": "EbtkLGHGij3Q91njJQJeZRKD3OI=",
+			"path": "github.com/alecthomas/chroma/formatters/html",
+			"revision": "b0295f66bdb7c61d54906003d7649185794e21b4",
+			"revisionTime": "2017-09-25T05:25:32Z"
+		},
+		{
+			"checksumSHA1": "ANyNTHVz5LdPPADsExM5WpBJe4c=",
+			"path": "github.com/alecthomas/chroma/lexers",
+			"revision": "1af7e1a0bc5c04ec39b8e6d25d70de8eafcf76ab",
+			"revisionTime": "2017-09-23T12:45:05Z"
+		},
+		{
+			"checksumSHA1": "Nm8r5bmokRePD0D7WU+rXYxOO9A=",
+			"path": "github.com/alecthomas/chroma/styles",
+			"revision": "b0295f66bdb7c61d54906003d7649185794e21b4",
+			"revisionTime": "2017-09-25T05:25:32Z"
+		},
 		{
 			"checksumSHA1": "7yrV1Gzr1ajco1xJ1gsyqRDTY2U=",
 			"path": "github.com/bep/gitmap",
@@ -38,6 +68,12 @@
 			"revision": "23709d0847197db6021a51fdb193e66e9222d4e7",
 			"revisionTime": "2017-06-03T12:52:39Z"
 		},
+		{
+			"checksumSHA1": "d/czTNq3bacK85PFEKcHvW6aR80=",
+			"path": "github.com/danwakefield/fnmatch",
+			"revision": "cbb64ac3d964b81592e64f957ad53df015803288",
+			"revisionTime": "2016-04-03T17:12:40Z"
+		},
 		{
 			"checksumSHA1": "OFu4xJEIjiI8Suu+j/gabfp+y6Q=",
 			"origin": "github.com/stretchr/testify/vendor/github.com/davecgh/go-spew/spew",
@@ -51,6 +87,18 @@
 			"revision": "fb8d9b44afdc258bfff6052d3667521babcb2239",
 			"revisionTime": "2015-12-10T17:00:30Z"
 		},
+		{
+			"checksumSHA1": "6y/Ht8J58EotTDBEIuE3+s4AnL8=",
+			"path": "github.com/dlclark/regexp2",
+			"revision": "487489b64fb796de2e55f4e8a4ad1e145f80e957",
+			"revisionTime": "2017-07-18T21:59:41Z"
+		},
+		{
+			"checksumSHA1": "k0JXX65FspyueQ8/1i50DGRiCUk=",
+			"path": "github.com/dlclark/regexp2/syntax",
+			"revision": "487489b64fb796de2e55f4e8a4ad1e145f80e957",
+			"revisionTime": "2017-07-18T21:59:41Z"
+		},
 		{
 			"checksumSHA1": "248k9rTfZ4kAknuomoKsdBG9zCU=",
 			"path": "github.com/eknkc/amber",
@@ -219,12 +267,6 @@
 			"revision": "3e70a1a463008cea6726380c908b1a6a8bdf7b24",
 			"revisionTime": "2017-05-12T15:20:54Z"
 		},
-		{
-			"checksumSHA1": "F1IYMLBLAZaTOWnmXsgaxTGvrWI=",
-			"path": "github.com/pelletier/go-buffruneio",
-			"revision": "c37440a7cf42ac63b919c752ca73a85067e05992",
-			"revisionTime": "2017-02-27T22:03:11Z"
-		},
 		{
 			"checksumSHA1": "zZg0J0MqvnqXVYo644QDvnUinrc=",
 			"path": "github.com/pelletier/go-toml",