markup: Reimplement pygmentsCodefencesGuessSyntax

Fixes #6565

go.mod

@@ -54,8 +54,8 @@ require (
 	github.com/spf13/viper v1.4.0
 	github.com/tdewolff/minify/v2 v2.6.1
 	github.com/yosssi/ace v0.0.5
-	github.com/yuin/goldmark v1.1.10
-	github.com/yuin/goldmark-highlighting v0.0.0-20191124122839-ede94e40cc3a
+	github.com/yuin/goldmark v1.1.11
+	github.com/yuin/goldmark-highlighting v0.0.0-20191202084645-78f32c8dd6d5
 	go.opencensus.io v0.22.0 // indirect
 	gocloud.dev v0.15.0
 	golang.org/x/image v0.0.0-20190523035834-f03afa92d3ff

go.sum

@@ -359,8 +359,14 @@ github.com/yuin/goldmark v1.1.8 h1:d0m8Ac9JaetYjPZLC4P4W32ac7I0lpJpQbvxZtFqBoM=
 github.com/yuin/goldmark v1.1.8/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.10 h1:bg3TC1aj4DbjGdhvjSSffGfAgVUdBEIpccuCozwOYWo=
 github.com/yuin/goldmark v1.1.10/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.11 h1:OO08ilczi3F4swaYWPB99s08WRxP9DdLBemiLFQ6vCo=
+github.com/yuin/goldmark v1.1.11/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark-highlighting v0.0.0-20191124122839-ede94e40cc3a h1:L7FTUnbc0WEBqGWgjbx4sPNAOX1/q5W/3KCD6g8XkKo=
 github.com/yuin/goldmark-highlighting v0.0.0-20191124122839-ede94e40cc3a/go.mod h1:1gshkGdH4gcrIH5MGSScGH42rOOCO+4Ks6acjAkA9C0=
+github.com/yuin/goldmark-highlighting v0.0.0-20191126180129-d7a4bf4d7ea4 h1:vI4Jv29V1cMPqetuLPMW1CMB9xNgxsHVBo8Mid6bwH8=
+github.com/yuin/goldmark-highlighting v0.0.0-20191126180129-d7a4bf4d7ea4/go.mod h1:4QGn5rJFOASBa2uK4Q2h3BRTyJqRfsAucPFIipSTcaM=
+github.com/yuin/goldmark-highlighting v0.0.0-20191202084645-78f32c8dd6d5 h1:QbH7ca1qtgZHrzvcVAEoiJIwBqrXxMOfHYfwZIniIK0=
+github.com/yuin/goldmark-highlighting v0.0.0-20191202084645-78f32c8dd6d5/go.mod h1:4QGn5rJFOASBa2uK4Q2h3BRTyJqRfsAucPFIipSTcaM=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.mongodb.org/mongo-driver v1.0.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
 go.opencensus.io v0.15.0/go.mod h1:UffZAU+4sDEINUGP/B7UfBBkq4fqLu9zXAX7ke6CHW0=

hugolib/hugo_sites_build_test.go

@@ -671,7 +671,7 @@ END
 
 	b.CreateSites().Build(BuildCfg{})
 
-	contentMatchers := []string{"<h2 id=\"another-header\">Another header</h2>", "<h2 id=\"another-header99\">Another header</h2>", "<p>The End.</p>"}
+	contentMatchers := []string{"<h2 id=\"another-header\">Another header</h2>", "<h2 id=\"another-header-99\">Another header</h2>", "<p>The End.</p>"}
 
 	for i := 1; i <= numPages; i++ {
 		if i%3 != 0 {
@@ -691,13 +691,13 @@ END
 		checkContent(b, fmt.Sprintf("public/%s/page%d/index.json", section, i), contentMatchers...)
 	}
 
-	checkContent(b, "public/s1/index.html", "P: s1/_index.md\nList: 10|List Content: 8033\n\n\nL1: 500 L2: 5\n\nRender 0: View: 8033\n\nRender 1: View: 8033\n\nRender 2: View: 8033\n\nRender 3: View: 8033\n\nRender 4: View: 8033\n\nEND\n")
-	checkContent(b, "public/s2/index.html", "P: s2/_index.md\nList: 10|List Content: 8033", "Render 4: View: 8033\n\nEND")
-	checkContent(b, "public/index.html", "P: _index.md\nList: 10|List Content: 8033", "4: View: 8033\n\nEND")
+	checkContent(b, "public/s1/index.html", "P: s1/_index.md\nList: 10|List Content: 8132\n\n\nL1: 500 L2: 5\n\nRender 0: View: 8132\n\nRender 1: View: 8132\n\nRender 2: View: 8132\n\nRender 3: View: 8132\n\nRender 4: View: 8132\n\nEND\n")
+	checkContent(b, "public/s2/index.html", "P: s2/_index.md\nList: 10|List Content: 8132", "Render 4: View: 8132\n\nEND")
+	checkContent(b, "public/index.html", "P: _index.md\nList: 10|List Content: 8132", "4: View: 8132\n\nEND")
 
 	// Check paginated pages
 	for i := 2; i <= 9; i++ {
-		checkContent(b, fmt.Sprintf("public/page/%d/index.html", i), fmt.Sprintf("Page: %d", i), "Content: 8033\n\n\nL1: 500 L2: 5\n\nRender 0: View: 8033", "Render 4: View: 8033\n\nEND")
+		checkContent(b, fmt.Sprintf("public/page/%d/index.html", i), fmt.Sprintf("Page: %d", i), "Content: 8132\n\n\nL1: 500 L2: 5\n\nRender 0: View: 8132", "Render 4: View: 8132\n\nEND")
 	}
 }
 

hugolib/page_test.go

@@ -1658,7 +1658,7 @@ $$$
 
 	b.AssertFileContent("public/page/index.html",
 		`<nav id="TableOfContents">`,
-		`<li><a href="#shortcode-tshort-in-header">Shortcode T-SHORT in header</a></li>`,
+		`<li><a href="#shortcode-t-short-in-header">Shortcode T-SHORT in header</a></li>`,
 		`<code class="language-bash" data-lang="bash"><span class="hl">SHORT`,
 		`<code class="language-bash" data-lang="bash"><span class="hl">MARKDOWN`)
 }

markup/goldmark/convert.go

@@ -198,6 +198,7 @@ func newHighlighting(cfg highlight.Config) goldmark.Extender {
 
 	e := hl.NewHighlighting(
 		hl.WithStyle(cfg.Style),
+		hl.WithGuessLanguage(cfg.GuessSyntax),
 		hl.WithCodeBlockOptions(highlight.GetCodeBlockOptions()),
 		hl.WithFormatOptions(
 			cfg.ToHTMLOptions()...,

markup/goldmark/convert_test.go

@@ -224,4 +224,25 @@ LINE5
 		result = convertForConfig(c, cfg, lines, "bash {linenos=table}")
 		c.Assert(result, qt.Contains, "<span class=\"lnt\">1\n</span>")
 	})
+
+	c.Run("No language", func(c *qt.C) {
+		cfg := highlight.DefaultConfig
+		cfg.NoClasses = false
+		cfg.LineNos = true
+		cfg.LineNumbersInTable = false
+
+		result := convertForConfig(c, cfg, lines, "")
+		c.Assert(result, qt.Contains, "<pre><code>LINE1\n")
+	})
+
+	c.Run("No language, guess syntax", func(c *qt.C) {
+		cfg := highlight.DefaultConfig
+		cfg.NoClasses = false
+		cfg.GuessSyntax = true
+		cfg.LineNos = true
+		cfg.LineNumbersInTable = false
+
+		result := convertForConfig(c, cfg, lines, "")
+		c.Assert(result, qt.Contains, "<span class=\"ln\">2</span>LINE2\n<")
+	})
 }

markup/highlight/config.go

@@ -58,6 +58,8 @@ type Config struct {
 
 	// TabWidth sets the number of characters for a tab. Defaults to 4.
 	TabWidth int
+
+	GuessSyntax bool
 }
 
 func (cfg Config) ToHTMLOptions() []html.Option {
@@ -104,6 +106,10 @@ func ApplyLegacyConfig(cfg config.Provider, conf *Config) error {
 		conf.CodeFences = cfg.GetBool("pygmentsCodeFences")
 	}
 
+	if conf.GuessSyntax == DefaultConfig.GuessSyntax && cfg.IsSet("pygmentsCodefencesGuessSyntax") {
+		conf.GuessSyntax = cfg.GetBool("pygmentsCodefencesGuessSyntax")
+	}
+
 	if cfg.IsSet("pygmentsOptions") {
 		if err := applyOptionsFromString(cfg.GetString("pygmentsOptions"), conf); err != nil {
 			return err

markup/highlight/highlight.go

@@ -52,6 +52,14 @@ func highlight(code, lang string, cfg Config) (string, error) {
 		lexer = lexers.Get(lang)
 	}
 
+	if lexer == nil && cfg.GuessSyntax {
+		lexer = lexers.Analyse(code)
+		if lexer == nil {
+			lexer = lexers.Fallback
+		}
+		lang = strings.ToLower(lexer.Config().Name)
+	}
+
 	if lexer == nil {
 		wrapper := getPreWrapper(lang)
 		fmt.Fprint(w, wrapper.Start(true, ""))

markup/highlight/highlight_test.go

@@ -84,4 +84,26 @@ LINE5
 		result, _ = h.Highlight(lines, "bash", "linenos=table")
 		c.Assert(result, qt.Contains, "<span class=\"lnt\">1\n</span>")
 	})
+
+	c.Run("No language", func(c *qt.C) {
+		cfg := DefaultConfig
+		cfg.NoClasses = false
+		cfg.LineNos = true
+		h := New(cfg)
+
+		result, _ := h.Highlight(lines, "", "")
+		c.Assert(result, qt.Equals, "<pre><code>LINE1\nLINE2\nLINE3\nLINE4\nLINE5\n</code></pre>")
+	})
+
+	c.Run("No language, guess syntax", func(c *qt.C) {
+		cfg := DefaultConfig
+		cfg.NoClasses = false
+		cfg.GuessSyntax = true
+		cfg.LineNos = true
+		cfg.LineNumbersInTable = false
+		h := New(cfg)
+
+		result, _ := h.Highlight(lines, "", "")
+		c.Assert(result, qt.Contains, "<span class=\"ln\">2</span>LINE2\n<")
+	})
 }

markup/markup_config/config_test.go

@@ -55,7 +55,7 @@ func TestConfig(t *testing.T) {
 		v.Set("footnoteAnchorPrefix", "myprefix")
 		v.Set("footnoteReturnLinkContents", "myreturn")
 		v.Set("pygmentsStyle", "hugo")
-
+		v.Set("pygmentsCodefencesGuessSyntax", true)
 		conf, err := Decode(v)
 
 		c.Assert(err, qt.IsNil)
@@ -64,6 +64,7 @@ func TestConfig(t *testing.T) {
 		c.Assert(conf.BlackFriday.FootnoteReturnLinkContents, qt.Equals, "myreturn")
 		c.Assert(conf.Highlight.Style, qt.Equals, "hugo")
 		c.Assert(conf.Highlight.CodeFences, qt.Equals, true)
+		c.Assert(conf.Highlight.GuessSyntax, qt.Equals, true)
 	})
 
 }