Add Goldmark as the new default markdown handler

This commit adds the fast and CommonMark compliant Goldmark as the new default markdown handler in Hugo.

If you want to continue using BlackFriday as the default for md/markdown extensions, you can use this configuration:

```toml
[markup]
defaultMarkdownHandler="blackfriday"
```

Fixes #5963
Fixes #1778
Fixes #6355

commands/genchromastyles.go

@@ -68,7 +68,7 @@ func (g *genChromaStyles) generate() error {
 	if err != nil {
 		return err
 	}
-	formatter := html.New(html.WithClasses())
+	formatter := html.New(html.WithClasses(true))
 	formatter.WriteCSS(os.Stdout, style)
 	return nil
 }

docs/config.toml

@@ -15,18 +15,6 @@ pluralizeListTitles = false
 # We do redirects via Netlify's _redirects file, generated by Hugo (see "outputs" below).
 disableAliases = true
 
-# Highlighting config (Pygments)
-# It is (currently) not in use, but you can do ```go in a content file if you want to.
-pygmentsCodeFences = true
-
-pygmentsOptions = ""
-# Use the Chroma stylesheet
-pygmentsUseClasses = true
-pygmentsUseClassic = false
-
-# See https://help.farbox.com/pygments.html
-pygmentsStyle = "trac"
-
 [module]
   [module.hugoVersion]
     min = "0.56.0"

docs/config/_default/config.toml

@@ -14,17 +14,11 @@ pluralizeListTitles = false
 # We do redirects via Netlify's _redirects file, generated by Hugo (see "outputs" below).
 disableAliases = true
 
-# Highlighting config (Pygments)
+[markup]
-# It is (currently) not in use, but you can do ```go in a content file if you want to.
+[markup.highlight]
-pygmentsCodeFences = true
+style = "trac"
-
+lineNumbersInTable = true
-pygmentsOptions = ""
+noClasses = false
-# Use the Chroma stylesheet
-pygmentsUseClasses = true
-pygmentsUseClassic = false
-
-# See https://help.farbox.com/pygments.html
-pygmentsStyle = "trac"
 
 [outputs]
 home = [ "HTML", "RSS", "REDIR", "HEADERS" ]

docs/content/en/about/features.md

@@ -52,7 +52,7 @@ toc: true
 * Integrated [Google Analytics][] support
 * Automatic [RSS][] creation
 * Support for [Go][], [Amber], and [Ace][] HTML templates
-* [Syntax highlighting][] powered by [Chroma][] (partly compatible with Pygments)
+* [Syntax highlighting][] powered by [Chroma][]
 
 
 [Ace]: /templates/alternatives/

docs/content/en/content-management/formats.md

@@ -1,6 +1,6 @@
 ---
-title: Supported Content Formats
+title: Content Formats
-linktitle: Supported Content Formats
+linktitle: Content Formats
 description: Both HTML and Markdown are supported content formats.
 date: 2017-01-10
 publishdate: 2017-01-10
@@ -13,191 +13,37 @@ menu:
     weight: 20
 weight: 20	#rem
 draft: false
-aliases: [/content/markdown-extras/,/content/supported-formats/,/doc/supported-formats/,/tutorials/mathjax/]
+aliases: [/content/markdown-extras/,/content/supported-formats/,/doc/supported-formats/]
 toc: true
 ---
 
-**Markdown is the main content format** and comes in two flavours:  The excellent [Blackfriday project][blackfriday] (name your files `*.md` or set `markup = "markdown"` in front matter) or its fork [Mmark][mmark] (name your files `*.mmark` or set `markup = "mmark"` in front matter), both very fast markdown engines written in Go.
+You can put any file type into your `/content` directories, but Hugo uses the `markup` front matter value if set or the file extension (see `Markup identifiers` in the table below) to determine if the markup needs to be processed, e.g.:
 
-For Emacs users, [go-org](https://github.com/niklasfasching/go-org) provides built-in native support for Org-mode  (name your files `*.org` or set `markup = "org"` in front matter)
+* Markdown converted to HTML
+* [Shortcodes](/content-management/shortcodes/) processed
+* Layout applied
 
-But in many situations, plain HTML is what you want. Just name your files with `.html` or `.htm` extension inside your content folder. Note that if you want your HTML files to have a layout, they need front matter. It can be empty, but it has to be there:
+## List of content formats
 
-```html
+The current list of content formats in Hugo:
----
-title: "This is a content file in HTML"
----
 
-<div>
+| Name  | Markup identifiers | Comment | 
-  <h1>Hello, Hugo!</h1>
+| ------------- | ------------- |-------------|
-</div>
+| Goldmark  | md, markdown, goldmark  |Note that you can set the default handler of `md` and `markdown` to something else, see [Configure Markup](/getting-started/configuration-markup/).{{< new-in "0.60.0" >}} |
-```
+| Blackfriday | blackfriday  |Blackfriday will eventually be deprecated.|
+|MMark|mmark|Mmark is deprecated and will be removed in a future release.|
+|Emacs Org-Mode|org|See [go-org](https://github.com/niklasfasching/go-org).|
+|Asciidoc|asciidoc, adoc, ad|Needs Asciidoc or [Asciidoctor][ascii] installed.|
+|RST|rst|Needs [RST](http://docutils.sourceforge.net/rst.html) installed.|
+|Pandoc|pandoc, pdc|Needs [Pandoc](https://www.pandoc.org/) installed.|
+|HTML|html, htm|To be treated as a content file, with layout, shortcodes etc., it must have front matter. If not, it will be copied as-is.|
 
-{{% note "Deeply Nested Lists" %}}
+The `markup identifier` is fetched from either the `markup` variable in front matter or from the file extension. For markup-related configuration, see [Configure Markup](/getting-started/configuration-markup/).
-Before you begin writing your content in markdown, Blackfriday has a known issue [(#329)](https://github.com/russross/blackfriday/issues/329) with handling deeply nested lists. Luckily, there is an easy workaround. Use 4-spaces (i.e., <kbd>tab</kbd>) rather than 2-space indentations.
-{{% /note %}}
 
-## Configure BlackFriday Markdown Rendering
 
-You can configure multiple aspects of Blackfriday as shown in the following list. See the docs on [Configuration][config] for the full list of explicit directions you can give to Hugo when rendering your site.
+## External Helpers
 
-{{< readfile file="/content/en/readfiles/bfconfig.md" markdown="true" >}}
+Some of the formats in the table above needs external helpers installed on your PC. For example, for Asciidoc files, Hugo will try to call the `asciidoctor` or `asciidoc` command. This means that you will have to install the associated tool on your machine to be able to use these formats. ([See the Asciidoctor docs for installation instructions](https://asciidoctor.org/docs/install-toolchain/)).
-
-## Extend Markdown
-
-Hugo provides some convenient methods for extending markdown.
-
-### Task Lists
-
-Hugo supports [GitHub-styled task lists (i.e., TODO lists)][gfmtasks] for the Blackfriday markdown renderer. If you do not want to use this feature, you can disable it in your configuration.
-
-#### Example Task List Input
-
-{{< code file="content/my-to-do-list.md" >}}
-- [ ] a task list item
-- [ ] list syntax required
-- [ ] incomplete
-- [x] completed
-{{< /code >}}
-
-#### Example Task List Output
-
-The preceding markdown produces the following HTML in your rendered website:
-
-```
-<ul class="task-list">
-    <li><input type="checkbox" disabled="" class="task-list-item"> a task list item</li>
-    <li><input type="checkbox" disabled="" class="task-list-item"> list syntax required</li>
-    <li><input type="checkbox" disabled="" class="task-list-item"> incomplete</li>
-    <li><input type="checkbox" checked="" disabled="" class="task-list-item"> completed</li>
-</ul>
-```
-
-#### Example Task List Display
-
-The following shows how the example task list will look to the end users of your website. Note that visual styling of lists is up to you. This list has been styled according to [the Hugo Docs stylesheet][hugocss].
-
-- [ ] a task list item
-- [ ] list syntax required
-- [ ] incomplete
-- [x] completed
-
-### Emojis
-
-To add emojis directly to content, set `enableEmoji` to `true` in your [site configuration][config]. To use emojis in templates or shortcodes, see [`emojify` function][].
-
-For a full list of emojis, see the [Emoji cheat sheet][emojis].
-
-### Shortcodes
-
-If you write in Markdown and find yourself frequently embedding your content with raw HTML, Hugo provides built-in shortcodes functionality. This is one of the most powerful features in Hugo and allows you to create your own Markdown extensions very quickly.
-
-See [Shortcodes][sc] for usage, particularly for the built-in shortcodes that ship with Hugo, and [Shortcode Templating][sct] to learn how to build your own.
-
-### Code Blocks
-
-Hugo supports GitHub-flavored markdown's use of triple back ticks, as well as provides a special [`highlight` shortcode][hlsc], and syntax highlights those code blocks natively using *Chroma*. Users also have an option to use *Pygments* instead. See the [Syntax Highlighting][hl] section for details.
-
-## Mmark
-
-Mmark is a [fork of BlackFriday][mmark] and markdown superset that is well suited for writing [IETF documentation][ietf]. You can see examples of the syntax in the [Mmark GitHub repository][mmark] or the full syntax on [Miek Gieben's website][].
-
-### Use Mmark
-
-As Hugo ships with Mmark, using the syntax is as easy as changing the extension of your content files from `.md` to `.mmark`.
-
-In the event that you want to only use Mmark in specific files, you can also define the Mmark syntax in your content's front matter:
-
-```
----
-title: My Post
-date: 2017-04-01
-markup: mmark
----
-```
-
-{{% warning %}}
-Thare are some features not available in Mmark; one example being that shortcodes are not translated when used in an included `.mmark` file ([#3131](https://github.com/gohugoio/hugo/issues/3137)), and `EXTENSION_ABBREVIATION` ([#1970](https://github.com/gohugoio/hugo/issues/1970)) and the aforementioned GFM todo lists ([#2270](https://github.com/gohugoio/hugo/issues/2270)) are not fully supported. Contributions are welcome.
-{{% /warning %}}
-
-## MathJax with Hugo
-
-[MathJax](https://www.mathjax.org/) is a JavaScript library that allows the display of mathematical expressions described via a LaTeX-style syntax in the HTML (or Markdown) source of a web page. As it is a pure a JavaScript library, getting it to work within Hugo is fairly straightforward, but does have some oddities that will be discussed here.
-
-This is not an introduction into actually using MathJax to render typeset mathematics on your website. Instead, this page is a collection of tips and hints for one way to get MathJax working on a website built with Hugo.
-
-### Enable MathJax
-
-The first step is to enable MathJax on pages that you would like to have typeset math. There are multiple ways to do this (adventurous readers can consult the [Loading and Configuring](https://docs.mathjax.org/en/latest/web/configuration.html) section of the MathJax documentation for additional methods of including MathJax), but the easiest way is to use the secure MathJax CDN by include a `<script>` tag for the officially recommended secure CDN ([cdn.js.com](https://cdnjs.com)):
-
-{{< code file="add-mathjax-to-page.html" >}}
-<script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
-</script>
-{{< /code >}}
-
-One way to ensure that this code is included in all pages is to put it in one of the templates that live in the `layouts/partials/` directory. For example, I have included this in the bottom of my template `footer.html` because I know that the footer will be included in every page of my website.
-
-### Options and Features
-
-MathJax is a stable open-source library with many features. I encourage the interested reader to view the [MathJax Documentation](https://docs.mathjax.org/en/latest/index.html), specifically the sections on [Basic Usage](http://docs.mathjax.org/en/latest/index.html#basic-usage) and [MathJax Configuration Options](http://docs.mathjax.org/en/latest/index.html#mathjax-configuration-options).
-
-### Issues with Markdown
-
-{{% note %}}
-The following issues with Markdown assume you are using `.md` for content and BlackFriday for parsing. Using [Mmark](#mmark) as your content format will obviate the need for the following workarounds.
-
-When using Mmark with MathJax, use `displayMath: [['$$','$$'], ['\\[','\\]']]`. See the [Mmark `README.md`](https://github.com/miekg/mmark/wiki/Syntax#math-blocks) for more information. In addition to MathJax, Mmark has been shown to work well with [KaTeX](https://github.com/Khan/KaTeX). See this [related blog post from a Hugo user](http://nosubstance.me/post/a-great-toolset-for-static-blogging/).
-{{% /note %}}
-
-After enabling MathJax, any math entered between proper markers (see the [MathJax documentation][mathjaxdocs]) will be processed and typeset in the web page. One issue that comes up, however, with Markdown is that the underscore character (`_`) is interpreted by Markdown as a way to wrap text in `emph` blocks while LaTeX (MathJax) interprets the underscore as a way to create a subscript. This "double speak" of the underscore can result in some unexpected and unwanted behavior.
-
-### Solution
-
-There are multiple ways to remedy this problem. One solution is to simply escape each underscore in your math code by entering `\_` instead of `_`. This can become quite tedious if the equations you are entering are full of subscripts.
-
-Another option is to tell Markdown to treat the MathJax code as verbatim code and not process it. One way to do this is to wrap the math expression inside a `<div>` `</div>` block. Markdown would ignore these sections and they would get passed directly on to MathJax and processed correctly. This works great for display style mathematics, but for inline math expressions the line break induced by the `<div>` is not acceptable. The syntax for instructing Markdown to treat inline text as verbatim is by wrapping it in backticks (`` ` ``). You might have noticed, however, that the text included in between backticks is rendered differently than standard text (on this site these are items highlighted in red). To get around this problem, we could create a new CSS entry that would apply standard styling to all inline verbatim text that includes MathJax code. Below I will show the HTML and CSS source that would accomplish this (note this solution was adapted from [this blog post](http://doswa.com/2011/07/20/mathjax-in-markdown.html)---all credit goes to the original author).
-
-{{< code file="mathjax-markdown-solution.html" >}}
-<script type="text/x-mathjax-config">
-MathJax.Hub.Config({
-  tex2jax: {
-    inlineMath: [['$','$'], ['\\(','\\)']],
-    displayMath: [['$$','$$'], ['\[','\]']],
-    processEscapes: true,
-    processEnvironments: true,
-    skipTags: ['script', 'noscript', 'style', 'textarea', 'pre'],
-    TeX: { equationNumbers: { autoNumber: "AMS" },
-         extensions: ["AMSmath.js", "AMSsymbols.js"] }
-  }
-});
-</script>
-{{< /code >}}
-
-As before, this content should be included in the HTML source of each page that will be using MathJax. The next code snippet contains the CSS that is used to have verbatim MathJax blocks render with the same font style as the body of the page.
-
-{{< code file="mathjax-style.css" >}}
-code.has-jax {
-    font: inherit;
-    font-size: 100%;
-    background: inherit;
-    border: inherit;
-    color: #515151;
-}
-{{< /code >}}
-
-In the CSS snippet, notice the line `color: #515151;`. `#515151` is the value assigned to the `color` attribute of the `body` class in my CSS. In order for the equations to fit in with the body of a web page, this value should be the same as the color of the body.
-
-### Usage
-
-With this setup, everything is in place for a natural usage of MathJax on pages generated using Hugo. In order to include inline mathematics, just put LaTeX code in between `` `$ TeX Code $` `` or `` `\( TeX Code \)` ``. To include display style mathematics, just put LaTeX code in between `<div>$$TeX Code$$</div>`. All the math will be properly typeset and displayed within your Hugo generated web page!
-
-## Additional Formats Through External Helpers
-
-Hugo has a new concept called _external helpers_. It means that you can write your content using [Asciidoc][ascii], [reStructuredText][rest], or [pandoc]. If you have files with associated extensions, Hugo will call external commands to generate the content. ([See the Hugo source code for external helpers][helperssource].)
-
-For example, for Asciidoc files, Hugo will try to call the `asciidoctor` or `asciidoc` command. This means that you will have to install the associated tool on your machine to be able to use these formats. ([See the Asciidoctor docs for installation instructions](https://asciidoctor.org/docs/install-toolchain/)).
-
-To use these formats, just use the standard extension and the front matter exactly as you would do with natively supported `.md` files.
 
 Hugo passes reasonable default arguments to these external helpers by default:
 

docs/content/en/content-management/syntax-highlighting.md

@@ -3,7 +3,7 @@ title: Syntax Highlighting
 description: Hugo comes with really fast syntax highlighting from Chroma.
 date: 2017-02-01
 publishdate: 2017-02-01
-keywords: [highlighting,pygments,chroma,code blocks,syntax]
+keywords: [highlighting,chroma,code blocks,syntax]
 categories: [content management]
 menu:
   docs:
@@ -16,17 +16,39 @@ aliases: [/extras/highlighting/,/extras/highlight/,/tools/syntax-highlighting/]
 toc: true
 ---
 
-From Hugo 0.28, the default syntax highlighter in Hugo is [Chroma](https://github.com/alecthomas/chroma); it is built in Go and is really, really fast -- and for the most important parts compatible with Pygments.
 
-If you want to continue to use Pygments (see below), set `pygmentsUseClassic=true` in your site config.
+Hugo uses [Chroma](https://github.com/alecthomas/chroma) as its code highlighter; it is built in Go and is really, really fast -- and for the most important parts compatible with Pygments we used before.
 
-The example below shows a simple code snippet from the Hugo source highlighted with the `highlight` shortcode. Note that the gohugo.io site is generated with `pygmentsUseClasses=true` (see [Generate Syntax Highlighter CSS](#generate-syntax-highlighter-css)).
+## Configure Syntax Highlighter
 
-* `linenos=inline` or `linenos=table` (`table` will give copy-and-paste friendly code blocks) turns on line numbers.
+See [Configure Highlight](/getting-started/configuration-markup#highlight).
-* `hl_lines` lists a set of line numbers or line number ranges to be highlighted. Note that the hyphen range syntax is only supported for Chroma.
+
+
+## Generate Syntax Highlighter CSS
+
+If you run with `pygmentsUseClasses=true` in your site config, you need a style sheet.
+
+You can generate one with Hugo:
+
+```bash
+hugo gen chromastyles --style=monokai > syntax.css
+```
+
+Run `hugo gen chromastyles -h` for more options. See https://xyproto.github.io/splash/docs/ for a gallery of available styles.
+
+
+## Highlight Shortcode
+
+Highlighting is carried out via the [built-in shortcode](/content-management/shortcodes/) `highlight`. `highlight` takes exactly one required parameter for the programming language to be highlighted and requires a closing shortcode. Note that `highlight` is *not* used for client-side javascript highlighting.
+
+Options:
+
+* `linenos`: Valid values are `true`, `false`, `table`, `inline`. `table` will give copy-and-paste friendly code blocks) turns on line numbers.
+* Setting `linenos` to `false` will turn off linenumbers if it's configured to be on in site config.{{< new-in "0.60.0" >}}
+* `hl_lines` lists a set of line numbers or line number ranges to be highlighted.
 * `linenostart=199` starts the line number count from 199.
 
-With that, this:
+### Example: Highlight Shortcode
 
 ```
 {{</* highlight go "linenos=table,hl_lines=8 15-17,linenostart=199" */>}}
@@ -62,134 +84,63 @@ func GetTitleFunc(style string) func(s string) string {
 {{< / highlight >}}
 
 
-## Configure Syntax Highlighter
-To make the transition from Pygments to Chroma seamless, they share a common set of configuration options:
-
-pygmentsOptions
-:  A comma separated list of options. See below for a full list.
-
-pygmentsCodeFences
-: Set to true to enable syntax highlighting in code fences with a language tag in markdown (see below for an example).
-
-pygmentsStyle
-: The style of code highlighting. Note that this option is not
-  relevant when `pygmentsUseClasses` is set.
-
-  Syntax highlighting galleries:
-  **Chroma** ([short snippets](https://xyproto.github.io/splash/docs/all.html),
-  [long snippets](https://xyproto.github.io/splash/docs/longer/all.html)),
-  [Pygments](https://help.farbox.com/pygments.html)
-
-pygmentsUseClasses
-: Set to `true` to use CSS classes to format your highlighted code. See [Generate Syntax Highlighter CSS](#generate-syntax-highlighter-css).
-
-pygmentsCodeFencesGuessSyntax
-: Set to `true` to try to do syntax highlighting on code fenced blocks in markdown without a language tag.
-
-pygmentsUseClassic
-: Set to true to use Pygments instead of the much faster Chroma.
-
-### Options
-
-`pygmentsOptions` can be set either in site config or overridden per code block in the Highlight shortcode or template func.
-
-noclasses
-: Use inline style.
-
-linenos
-: For Chroma, any value in this setting will print line numbers. Pygments has some more fine grained control.
-
-linenostart
-: Start the line numbers from this value (default is 1).
-
-
-hl_lines
-: Highlight a space separated list of line numbers. For Chroma, you can provide a list of ranges, i.e. "3-8 10-20".
-
-
-The full set of supported options for Pygments is: `encoding`, `outencoding`, `nowrap`, `full`, `title`, `style`, `noclasses`, `classprefix`, `cssclass`, `cssstyles`, `prestyles`, `linenos`, `hl_lines`, `linenostart`, `linenostep`, `linenospecial`, `nobackground`, `lineseparator`, `lineanchors`, `linespans`, `anchorlinenos`, `startinline`. See the [Pygments HTML Formatter Documentation](http://pygments.org/docs/formatters/#HtmlFormatter) for details.
-
-
-## Generate Syntax Highlighter CSS
-
-If you run with `pygmentsUseClasses=true` in your site config, you need a style sheet.
-
-You can generate one with Hugo:
-
-```bash
-hugo gen chromastyles --style=monokai > syntax.css
-```
-
-Run `hugo gen chromastyles -h` for more options. See https://xyproto.github.io/splash/docs/ for a gallery of available styles.
-
-
-## Highlight Shortcode
-
-Highlighting is carried out via the [built-in shortcode](/content-management/shortcodes/) `highlight`. `highlight` takes exactly one required parameter for the programming language to be highlighted and requires a closing shortcode. Note that `highlight` is *not* used for client-side javascript highlighting.
-
-### Example `highlight` Shortcode
-
-{{< code file="example-highlight-shortcode-input.md" >}}
-{{</* highlight html */>}}
-<section id="main">
-  <div>
-    <h1 id="title">{{ .Title }}</h1>
-    {{ range .Pages }}
-      {{ .Render "summary"}}
-    {{ end }}
-  </div>
-</section>
-{{</* /highlight */>}}
-{{< /code >}}
-
-
 
 ## Highlight Template Func
 
 See [Highlight](/functions/highlight/).
 
-## Highlight in Code Fences
+## Highlighting in Code Fences
 
-It is also possible to add syntax highlighting with GitHub flavored code fences. To enable this, set the `pygmentsCodeFences` to `true` in Hugo's [configuration file](/getting-started/configuration/);
+Highlighting in code fences is enabled by default.{{< new-in "0.60.0" >}}
 
 ````
-```go-html-template
+```go-html-template{hl_lines=[3,"5-6"],linenos=true}
-<section id="main">
-  <div>
-    <h1 id="title">{{ .Title }}</h1>
-    {{ range .Pages }}
-      {{ .Render "summary"}}
-    {{ end }}
-  </div>
-</section>
 ```
 ````
 
+````
+```go {linenos=table,hl_lines=[8,"15-17"],linenostart=199}
+// ... code
+````
+
+
+Gives this:
+
+```go {linenos=table,hl_lines=[8,"15-17"],linenostart=199}
+// GetTitleFunc returns a func that can be used to transform a string to
+// title case.
+//
+// The supported styles are
+//
+// - "Go" (strings.Title)
+// - "AP" (see https://www.apstylebook.com/)
+// - "Chicago" (see https://www.chicagomanualofstyle.org/home.html)
+//
+// If an unknown or empty style is provided, AP style is what you get.
+func GetTitleFunc(style string) func(s string) string {
+  switch strings.ToLower(style) {
+  case "go":
+    return strings.Title
+  case "chicago":
+    tc := transform.NewTitleConverter(transform.ChicagoStyle)
+    return tc.Title
+  default:
+    tc := transform.NewTitleConverter(transform.APStyle)
+    return tc.Title
+  }
+}
+```
+
+{{< new-in "0.60.0" >}}Note that only Goldmark supports passing attributes such as `hl_lines`, and it's important that it does not contain any spaces. See [goldmark-highlighting](https://github.com/yuin/goldmark-highlighting) for more information.
+
+The options are the same as in the [highlighting shortcode](/content-management/syntax-highlighting/#highlight-shortcode),including `linenos=false`, but note the slightly different Markdown attribute syntax.
+
 ## List of Chroma Highlighting Languages
 
 The full list of Chroma lexers and their aliases (which is the identifier used in the `highlight` template func or when doing highlighting in code fences):
 
 {{< chroma-lexers >}}
 
-## Highlight with Pygments Classic
-
-If you for some reason don't want to use the built-in Chroma highlighter, you can set `pygmentsUseClassic=true` in your config and add Pygments to your path.
-
-{{% note "Disclaimers on Pygments" %}}
-* Pygments is relatively slow and _causes a performance hit when building your site_, but Hugo has been designed to cache the results to disk.
-* The caching can be turned off by setting the `--ignoreCache` flag to `true`.
-* The languages available for highlighting depend on your Pygments installation.
-{{% /note %}}
-
-If you have never worked with Pygments before, here is a brief primer:
-
-+ Install Python from [python.org](https://www.python.org/downloads/). Version 2.7.x is already sufficient.
-+ Run `pip install Pygments` in order to install Pygments. Once installed, Pygments gives you a command `pygmentize`. Make sure it sits in your PATH; otherwise, Hugo will not be able to find and use it.
-
-On Debian and Ubuntu systems, you may also install Pygments by running `sudo apt-get install python3-pygments`.
-
-
-
 [Prism]: https://prismjs.com
 [prismdownload]: https://prismjs.com/download.html
 [Highlight.js]: https://highlightjs.org/

docs/content/en/contribute/documentation.md

@@ -104,6 +104,7 @@ Your options for languages are `xml`/`html`, `go`/`golang`, `md`/`markdown`/`mkd
 ```
 ````
 
+
 ### Code Block Shortcode
 
 The Hugo documentation comes with a very robust shortcode for adding interactive code blocks.

docs/content/en/functions/highlight.md

@@ -1,7 +1,7 @@
 ---
 title: highlight
 linktitle: highlight
-description: Takes a string of code and language declaration and uses Pygments to return syntax-highlighted HTML with inline-styles.
+description: Takes a string of code and language declaration and uses Chroma to return syntax-highlighted HTML.
 godocref:
 date: 2017-02-01
 publishdate: 2017-02-01
@@ -10,7 +10,7 @@ categories: [functions]
 menu:
   docs:
     parent: "functions"
-keywords: [highlighting,pygments,code blocks,syntax]
+keywords: [highlighting,code blocks,syntax]
 signature: ["highlight INPUT LANG OPTIONS"]
 workson: []
 hugoversion:
@@ -20,8 +20,6 @@ deprecated: false
 
 [`highlight` is used in Hugo's built-in `highlight` shortcode][highlight].
 
-See [Installing Hugo][installpygments] for more information on Pygments or [Syntax Highlighting][syntax] for more options on how to add syntax highlighting to your code blocks with Hugo.
-
 
 [highlight]: /content-management/shortcodes/#highlight
 [installpygments]: /getting-started/installing/#installing-pygments-optional

docs/content/en/getting-started/configuration-markup.md

@@ -0,0 +1,73 @@
+---
+title: Configure Markup
+description: How to handle Markdown and other markup related configuration.
+date: 2019-11-15
+categories: [getting started,fundamentals]
+keywords: [configuration,highlighting]
+weight: 65
+sections_weight: 65
+slug: configuration-markup
+toc: true
+---
+
+## Configure Markup
+
+{{< new-in "0.60.0" >}}
+
+See [Goldmark](#goldmark) for settings related to the default Markdown handler in Hugo.
+
+Below are all markup related configuration in Hugo with their default settings:
+
+{{< code-toggle config="markup" />}}
+
+**See each section below for details.**
+
+### Goldmark
+
+[Goldmark](https://github.com/yuin/goldmark/) is from Hugo 0.60 the default library used for Markdown. It's fast, it's [CommonMark](https://spec.commonmark.org/0.29/) compliant and it's very flexible. Note that the feature set of Goldmark vs Blackfriday isn't the same; you gain a lot but also lose some, but we will work to bridge any gap in the upcoming Hugo versions.
+
+This is the default configuration:
+
+{{< code-toggle config="markup.goldmark" />}}
+
+Some settings explained:
+
+unsafe
+: By default, Goldmark does not render raw HTMLs and potentially dangerous links. If you have lots of inline HTML and/or JavaScript, you may need to turn this on.
+
+typographer
+: This extension substitutes punctuations with typographic entities like [smartypants](https://daringfireball.net/projects/smartypants/).
+
+### Blackfriday
+
+
+[Blackfriday](https://github.com/russross/blackfriday) was Hugo's default Markdown rendering engine, now replaced with Goldmark. But you can still use it: Just set `defaultMarkdownHandler` to `blackfriday` in your top level `markup` config.
+
+This is the default config:
+
+{{< code-toggle config="markup.blackFriday" />}}
+
+### Highlight
+
+This is the default `highlight` configuration. Note that some of these settings can be set per code block, see [Syntax Highlighting](/content-management/syntax-highlighting/).
+
+{{< code-toggle config="markup.highlight" />}}
+
+For `style`, see these galleries:
+
+* [Short snippets](https://xyproto.github.io/splash/docs/all.html)
+* [Long snippets](https://xyproto.github.io/splash/docs/longer/all.html)
+
+For CSS, see [Generate Syntax Highlighter CSS](/content-management/syntax-highlighting/#generate-syntax-highlighter-css).
+
+### Table Of Contents
+
+{{< code-toggle config="markup.tableOfContents" />}}
+
+These settings only works for the Goldmark renderer:
+
+startLevel
+: The heading level, values starting at 1 (`h1`), to start render the table of contents.
+
+endLevel
+: The heading level, inclusive, to stop render the table of contents.

docs/content/en/getting-started/configuration.md

@@ -184,11 +184,14 @@ log (false)
 logFile ("")
 : Log File path (if set, logging enabled automatically).
 
+markup
+: See [Configure Markup](/getting-started/configuration-markup).{{< new-in "0.60.0" >}}
+
 menu
 : See [Add Non-content Entries to a Menu](/content-management/menus/#add-non-content-entries-to-a-menu).
 
 module
-: Module config see [Module Config](/hugo-modules/configuration/).
+: Module config see [Module Config](/hugo-modules/configuration/).{{< new-in "0.56.0" >}}
 
 newContentEditor ("")
 : The editor to use when creating new content.
@@ -214,26 +217,8 @@ pluralizeListTitles (true)
 publishDir ("public")
 : The directory to where Hugo will write the final static site (the HTML files etc.).
 
-pygmentsOptions ("")
-:  A comma separated list of options for syntax highlighting. See the [Syntax Highlighting Options](/content-management/syntax-highlighting/#options) for the full list of available options.
-
-pygmentsCodeFences (false)
-: Enables syntax highlighting in [code fences with a language tag](/content-management/syntax-highlighting/#highlight-in-code-fences) in markdown.
-
-pygmentsCodeFencesGuessSyntax (false)
-: Enable syntax guessing for code fences without specified language.
-
-pygmentsStyle ("monokai")
-: Color-theme or style for syntax highlighting. See [Pygments Color Themes](https://help.farbox.com/pygments.html).
-
-pygmentsUseClasses (false)
-: Enable using external CSS for syntax highlighting.
-
-pygmentsUseClassic (false)
-: Enable using Pygments instead of the much faster Chroma for syntax highlighting.
-
 related
-: See [Related Content](/content-management/related/#configure-related-content).
+: See [Related Content](/content-management/related/#configure-related-content).{{< new-in "0.27" >}}
 
 relativeURLs (false)
 : Enable this to make all relative URLs relative to content root. Note that this does not affect absolute URLs.
@@ -436,29 +421,6 @@ The above will try first to extract the value for `.Date` from the filename, the
 `:git`
 : This is the Git author date for the last revision of this content file. This will only be set if `--enableGitInfo` is set or `enableGitInfo = true` is set in site config.
 
-## Configure Blackfriday
-
-[Blackfriday](https://github.com/russross/blackfriday) is Hugo's built-in Markdown rendering engine.
-
-Hugo typically configures Blackfriday with sane default values that should fit most use cases reasonably well.
-
-However, if you have specific needs with respect to Markdown, Hugo exposes some of its Blackfriday behavior options for you to alter. The following table lists these Hugo options, paired with the corresponding flags from Blackfriday's source code ( [html.go](https://github.com/russross/blackfriday/blob/master/html.go) and [markdown.go](https://github.com/russross/blackfriday/blob/master/markdown.go)).
-
-{{< readfile file="/content/en/readfiles/bfconfig.md" markdown="true" >}}
-
-{{% note %}}
-1. Blackfriday flags are *case sensitive* as of Hugo v0.15.
-2. Blackfriday flags must be grouped under the `blackfriday` key and can be set on both the site level *and* the page level. Any setting on a page will override its respective site setting.
-{{% /note %}}
-
-{{< code-toggle file="config" >}}
-[blackfriday]
-  angledQuotes = true
-  fractions = false
-  plainIDAnchors = true
-  extensions = ["hardLineBreak"]
-{{< /code-toggle >}}
-
 ## Configure Additional Output Formats
 
 Hugo v0.20 introduced the ability to render your content to multiple output formats (e.g., to JSON, AMP html, or CSV). See [Output Formats][] for information on how to add these values to your Hugo project's configuration file.

docs/content/en/getting-started/installing.md

@@ -502,12 +502,6 @@ OpenBSD provides a package for Hugo via `pkg_add`:
 
 Upgrading Hugo is as easy as downloading and replacing the executable you’ve placed in your `PATH` or run `brew upgrade hugo` if using Homebrew.
 
-## Install Pygments (Optional)
-
-The Hugo executable has one *optional* external dependency for source code highlighting ([Pygments][pygments]).
-
-If you want to have source code highlighting using the [highlight shortcode][], you need to install the Python-based Pygments program. The procedure is outlined on the [Pygments homepage][pygments].
-
 ## Next Steps
 
 Now that you've installed Hugo, read the [Quick Start guide][quickstart] and explore the rest of the documentation. If you have questions, ask the Hugo community directly by visiting the [Hugo Discussion Forum][forum].

docs/content/en/readfiles/bfconfig.md

@@ -1,197 +0,0 @@
-## Blackfriday Options
-
-`taskLists`
-: default: **`true`**<br>
-    Blackfriday flag: <br>
-    Purpose: `false` turns off GitHub-style automatic task/TODO list generation.
-
-`smartypants`
-: default: **`true`** <br>
-    Blackfriday flag: **`HTML_USE_SMARTYPANTS`** <br>
-    Purpose: `false` disables smart punctuation substitutions, including smart quotes, smart dashes, smart fractions, etc. If `true`, it may be fine-tuned with the `angledQuotes`, `fractions`, `smartDashes`, and `latexDashes` flags (see below).
-
-`smartypantsQuotesNBSP`
-: default: **`false`** <br>
-    Blackfriday flag: **`HTML_SMARTYPANTS_QUOTES_NBSP`** <br>
-    Purpose: `true` enables French style Guillemets with non-breaking space inside the quotes.
-
-`angledQuotes`
-: default: **`false`**<br>
-    Blackfriday flag: **`HTML_SMARTYPANTS_ANGLED_QUOTES`**<br>
-    Purpose: `true` enables smart, angled double quotes. Example: "Hugo" renders to «Hugo» instead of “Hugo”.
-
-`fractions`
-: default: **`true`**<br>
-    Blackfriday flag: **`HTML_SMARTYPANTS_FRACTIONS`** <br>
-    Purpose: <code>false</code> disables smart fractions.<br>
-    Example: `5/12` renders to <sup>5</sup>&frasl;<sub>12</sub>(<code>&lt;sup&gt;5&lt;/sup&gt;&amp;frasl;&lt;sub&gt;12&lt;/sub&gt;</code>).<br> <small><strong>Caveat:</strong> Even with <code>fractions = false</code>, Blackfriday still converts `1/2`, `1/4`, and `3/4` respectively to ½ (<code>&amp;frac12;</code>), ¼ (<code>&amp;frac14;</code>) and ¾ (<code>&amp;frac34;</code>), but only these three.</small>
-
-`smartDashes`
-: default: **`true`** <br>
-    Blackfriday flag: **`HTML_SMARTY_DASHES`** <br>
-    Purpose: `false` disables smart dashes; i.e., the conversion of multiple hyphens into an en-dash or em-dash. If `true`, its behavior can be modified with the `latexDashes` flag below.
-
-`latexDashes`
-: default: **`true`** <br>
-    Blackfriday flag: **`HTML_SMARTYPANTS_LATEX_DASHES`** <br>
-    Purpose: `false` disables LaTeX-style smart dashes and selects conventional smart dashes. Assuming `smartDashes`: <br>
-    If `true`, `--` is translated into &ndash; (`&ndash;`), whereas `---` is translated into &mdash; (`&mdash;`). <br>
-    However, *spaced* single hyphen between two words is translated into an en&nbsp;dash&mdash; e.g., "`12 June - 3 July`" becomes `12 June &ndash; 3 July` upon rendering.
-
-`hrefTargetBlank`
-: default: **`false`** <br>
-    Blackfriday flag: **`HTML_HREF_TARGET_BLANK`** <br>
-    Purpose: `true` opens <s>external links</s> **absolute** links in a new window or tab. While the `target="_blank"` attribute is typically used for external links, Blackfriday does that for _all_ absolute links ([ref](https://discourse.gohugo.io/t/internal-links-in-same-tab-external-links-in-new-tab/11048/8)). One needs to make note of this if they use absolute links throughout, for internal links too (for example, by setting `canonifyURLs` to `true` or via `absURL`).
-
-`nofollowLinks`
-: default: **`false`** <br>
-    Blackfriday flag: **`HTML_NOFOLLOW_LINKS`** <br>
-    Purpose: `true` creates <s>external links</s> **absolute** links with `nofollow` being added to their `rel` attribute. Thereby crawlers are advised to not follow the link. While the `rel="nofollow"` attribute is typically used for external links, Blackfriday does that for _all_ absolute links. One needs to make note of this if they use absolute links throughout, for internal links too (for example, by setting `canonifyURLs` to `true` or via `absURL`).
-
-`noreferrerLinks`
-: default: **`false`** <br>
-    Blackfriday flag: **`HTML_NOREFERRER_LINKS`** <br>
-    Purpose: `true` creates <s>external links</s> **absolute** links with `noreferrer` being added to their `rel` attribute. Thus when following the link no referrer information will be leaked. While the `rel="noreferrer"` attribute is typically used for external links, Blackfriday does that for _all_ absolute links. One needs to make note of this if they use absolute links throughout, for internal links too (for example, by setting `canonifyURLs` to `true` or via `absURL`).
-
-`plainIDAnchors`
-: default **`true`** <br>
-    Blackfriday flag: **`FootnoteAnchorPrefix` and `HeaderIDSuffix`** <br>
-    Purpose: `true` renders any heading and footnote IDs without the document ID. <br>
-    Example: renders `#my-heading` instead of `#my-heading:bec3ed8ba720b970`
-
-`extensions`
-: default: **`[]`** <br>
-    Purpose: Enable one or more Blackfriday's Markdown extensions (**`EXTENSION_*`**). <br>
-    Example: Include `hardLineBreak` in the list to enable Blackfriday's `EXTENSION_HARD_LINE_BREAK`. <br>
-    *See [Blackfriday extensions](#blackfriday-extensions) section for information on all extensions.*
-
-`extensionsmask`
-: default: **`[]`** <br>
-    Purpose: Disable one or more of Blackfriday's Markdown extensions (**`EXTENSION_*`**). <br>
-    Example: Include `autoHeaderIds` as `false` in the list to disable Blackfriday's `EXTENSION_AUTO_HEADER_IDS`. <br>
-    *See [Blackfriday extensions](#blackfriday-extensions) section for information on all extensions.*
-
-`skipHTML`
-: default: **`false`** <br>
-    Blackfriday flag: **`HTML_SKIP_HTML`** <br>
-    Purpose: `true` causes any HTML in the markdown files to be skipped.
-
-## Blackfriday extensions
-
-`noIntraEmphasis`
-: default: *enabled* <br>
-    Purpose: The "\_" character is commonly used inside words when discussing
-    code, so having Markdown interpret it as an emphasis command is usually the
-    wrong thing.  When enabled, Blackfriday lets you treat all emphasis markers
-    as normal characters when they occur inside a word.
-
-`tables`
-: default: *enabled* <br>
-    Purpose: When enabled, tables can be created by drawing them in the input
-    using the below syntax:
-    Example:
-
-           Name | Age
-        --------|------
-            Bob | 27
-          Alice | 23
-
-`fencedCode`
-: default: *enabled* <br>
-    Purpose: When enabled, in addition to the normal 4-space indentation to mark
-    code blocks, you can explicitly mark them and supply a language (to make
-    syntax highlighting simple).
-
-    You can use 3 or more backticks to mark the beginning of the block, and the
-    same number to mark the end of the block.
-
-    Example:
-
-         ```md
-        # Heading Level 1
-        Some test
-        ## Heading Level 2
-        Some more test
-        ```
-
-`autolink`
-: default: *enabled* <br>
-    Purpose: When enabled, URLs that have not been explicitly marked as links
-    will be converted into links.
-
-`strikethrough`
-: default: *enabled* <br>
-    Purpose: When enabled, text wrapped with two tildes will be crossed out. <br>
-    Example: `~~crossed-out~~`
-
-`laxHtmlBlocks`
-: default: *disabled* <br>
-    Purpose: When enabled, loosen up HTML block parsing rules.
-
-`spaceHeaders`
-: default: *enabled* <br>
-    Purpose: When enabled, be strict about prefix header rules.
-
-`hardLineBreak`
-: default: *disabled* <br>
-    Purpose: When enabled, newlines in the input translate into line breaks in
-    the output.
-
-
-`tabSizeEight`
-: default: *disabled* <br>
-    Purpose: When enabled, expand tabs to eight spaces instead of four.
-
-`footnotes`
-: default: *enabled* <br>
-    Purpose: When enabled, Pandoc-style footnotes will be supported.  The
-    footnote marker in the text that will become a superscript text; the
-    footnote definition will be placed in a list of footnotes at the end of the
-    document. <br>
-    Example:
-
-        This is a footnote.[^1]
-
-        [^1]: the footnote text.
-
-`noEmptyLineBeforeBlock`
-: default: *disabled* <br>
-    Purpose: When enabled, no need to insert an empty line to start a (code,
-    quote, ordered list, unordered list) block.
-
-
-`headerIds`
-: default: *enabled* <br>
-    Purpose: When enabled, allow specifying header IDs with `{#id}`.
-
-`titleblock`
-: default: *disabled* <br>
-    Purpose: When enabled, support [Pandoc-style title blocks][1].
-
-`autoHeaderIds`
-: default: *enabled* <br>
-    Purpose: When enabled, auto-create the header ID's from the headline text.
-
-`backslashLineBreak`
-: default: *enabled* <br>
-    Purpose: When enabled, translate trailing backslashes into line breaks.
-
-`definitionLists`
-: default: *enabled* <br>
-    Purpose: When enabled, a simple definition list is made of a single-line
-    term followed by a colon and the definition for that term. <br>
-    Example:
-
-        Cat
-        : Fluffy animal everyone likes
-
-        Internet
-        : Vector of transmission for pictures of cats
-
-    Terms must be separated from the previous definition by a blank line.
-
-`joinLines`
-: default: *enabled* <br>
-    Purpose: When enabled, delete newlines and join the lines.
-
-[1]: http://pandoc.org/MANUAL.html#extension-pandoc_title_block

docs/content/en/templates/shortcode-templates.md

@@ -299,10 +299,6 @@ The rendered output of the HTML example code block will be as follows:
 </pre></div>
 {{< /code >}}
 
-{{% note %}}
-The preceding shortcode makes use of a Hugo-specific template function called `highlight`, which uses [Pygments](http://pygments.org) to add syntax highlighting to the example HTML code block. See the [developer tools page on syntax highlighting](/tools/syntax-highlighting/) for more information.
-{{% /note %}}
-
 ### Nested Shortcode: Image Gallery
 
 Hugo's [`.Parent` shortcode variable][parent] returns a boolean value depending on whether the shortcode in question is called within the context of a *parent* shortcode. This provides an inheritance model for common shortcode parameters.

docs/data/docs.json

@@ -1,6 +1,13 @@
 {
   "chroma": {
     "lexers": [
+      {
+        "Name": "ABAP",
+        "Aliases": [
+          "ABAP",
+          "abap"
+        ]
+      },
       {
         "Name": "ABNF",
         "Aliases": [
@@ -134,6 +141,13 @@
           "winbatch"
         ]
       },
+      {
+        "Name": "BibTeX",
+        "Aliases": [
+          "bib",
+          "bibtex"
+        ]
+      },
       {
         "Name": "BlitzBasic",
         "Aliases": [
@@ -405,7 +419,8 @@
         "Aliases": [
           "forth",
           "frt",
-          "fs"
+          "fs",
+          "fth"
         ]
       },
       {
@@ -579,12 +594,27 @@
           "idris"
         ]
       },
+      {
+        "Name": "Igor",
+        "Aliases": [
+          "igor",
+          "igorpro",
+          "ipf"
+        ]
+      },
       {
         "Name": "Io",
         "Aliases": [
           "io"
         ]
       },
+      {
+        "Name": "J",
+        "Aliases": [
+          "ijs",
+          "j"
+        ]
+      },
       {
         "Name": "JSON",
         "Aliases": [
@@ -1342,6 +1372,63 @@
       }
     ]
   },
+  "config": {
+    "markup": {
+      "defaultMarkdownHandler": "goldmark",
+      "highlight": {
+        "style": "monokai",
+        "codeFences": true,
+        "noClasses": true,
+        "lineNos": false,
+        "lineNumbersInTable": true,
+        "lineNoStart": 1,
+        "hl_Lines": "",
+        "tabWidth": 4
+      },
+      "tableOfContents": {
+        "startLevel": 2,
+        "endLevel": 3
+      },
+      "goldmark": {
+        "renderer": {
+          "hardWraps": false,
+          "xHTML": false,
+          "unsafe": false
+        },
+        "parser": {
+          "autoHeadingID": true,
+          "attribute": true
+        },
+        "extensions": {
+          "typographer": true,
+          "footnote": true,
+          "definitionList": true,
+          "table": true,
+          "strikethrough": true,
+          "linkify": true,
+          "taskList": true
+        }
+      },
+      "blackFriday": {
+        "smartypants": true,
+        "smartypantsQuotesNBSP": false,
+        "angledQuotes": false,
+        "fractions": true,
+        "hrefTargetBlank": false,
+        "nofollowLinks": false,
+        "noreferrerLinks": false,
+        "smartDashes": true,
+        "latexDashes": true,
+        "taskLists": true,
+        "plainIDAnchors": true,
+        "extensions": null,
+        "extensionsMask": null,
+        "skipHTML": false,
+        "footnoteAnchorPrefix": "",
+        "footnoteReturnLinkContents": ""
+      }
+    }
+  },
   "media": {
     "types": [
       {
@@ -1513,6 +1600,68 @@
         "suffixes": [
           "scss"
         ]
+      },
+      {
+        "type": "video/3gpp",
+        "string": "video/3gpp",
+        "mainType": "video",
+        "subType": "3gpp",
+        "delimiter": ".",
+        "suffixes": [
+          "3gpp",
+          "3gp"
+        ]
+      },
+      {
+        "type": "video/mp4",
+        "string": "video/mp4",
+        "mainType": "video",
+        "subType": "mp4",
+        "delimiter": ".",
+        "suffixes": [
+          "mp4"
+        ]
+      },
+      {
+        "type": "video/mpeg",
+        "string": "video/mpeg",
+        "mainType": "video",
+        "subType": "mpeg",
+        "delimiter": ".",
+        "suffixes": [
+          "mpg",
+          "mpeg"
+        ]
+      },
+      {
+        "type": "video/ogg",
+        "string": "video/ogg",
+        "mainType": "video",
+        "subType": "ogg",
+        "delimiter": ".",
+        "suffixes": [
+          "ogv"
+        ]
+      },
+      {
+        "type": "video/webm",
+        "string": "video/webm",
+        "mainType": "video",
+        "subType": "webm",
+        "delimiter": ".",
+        "suffixes": [
+          "webm"
+        ]
+      },
+      {
+        "type": "video/x-msvideo",
+        "string": "video/x-msvideo",
+        "mainType": "video",
+        "subType": "x-msvideo",
+        "delimiter": ".",
+        "suffixes": [
+          "avi"
+        ]
       }
     ]
   },
@@ -2290,10 +2439,10 @@
           ]
         },
         "Eq": {
-          "Description": "Eq returns the boolean truth of arg1 == arg2.",
+          "Description": "Eq returns the boolean truth of arg1 == arg2 || arg1 == arg3 || arg1 == arg4.",
           "Args": [
-            "x",
+            "first",
-            "y"
+            "others"
           ],
           "Aliases": [
             "eq"
@@ -2454,7 +2603,7 @@
           ]
         },
         "Dictionary": {
-          "Description": "Dictionary creates a map[string]interface{} from the given parameters by\nwalking the parameters and treating them as key-value pairs.  The number\nof parameters must be even.",
+          "Description": "Dictionary creates a map[string]interface{} from the given parameters by\nwalking the parameters and treating them as key-value pairs.  The number\nof parameters must be even.\nThe keys can be string slices, which will create the needed nested structure.",
           "Args": [
             "values"
           ],
@@ -2521,7 +2670,7 @@
           "Description": "Index returns the result of indexing its first argument by the following\narguments. Thus \"index x 1 2 3\" is, in Go syntax, x[1][2][3]. Each\nindexed item must be a map, slice, or array.\n\nCopied from Go stdlib src/text/template/funcs.go.\n\nWe deviate from the stdlib due to https://github.com/golang/go/issues/14751.\n\nTODO(moorereason): merge upstream changes.",
           "Args": [
             "item",
-            "indices"
+            "args"
           ],
           "Aliases": [
             "index"
@@ -2630,6 +2779,12 @@
             ]
           ]
         },
+        "Reverse": {
+          "Description": "",
+          "Args": null,
+          "Aliases": null,
+          "Examples": null
+        },
         "Seq": {
           "Description": "Seq creates a sequence of integers.  It's named and used as GNU's seq.\n\nExamples:\n    3 =\u003e 1, 2, 3\n    1 2 4 =\u003e 1, 3\n    -3 =\u003e -1, -2, -3\n    1 4 =\u003e 1, 2, 3, 4\n    1 -2 =\u003e 1, 0, -1, -2",
           "Args": [

docs/layouts/shortcodes/code-toggle.html

@@ -0,0 +1,34 @@
+{{ $file := .Get "file" }}
+{{ $code := "" }}
+{{ with .Get "config" }}
+{{ $file = $file | default "config" }}
+{{ $sections := (split . ".") }}
+{{ $configSection := index $.Site.Data.docs.config $sections }}
+{{ $code = dict $sections $configSection  }}
+{{ else }}
+{{ $code = $.Inner }}
+{{ end  }}
+{{ $langs := (slice "yaml" "toml" "json") }}
+<div class="code relative" {{ with $file }}id="{{ . | urlize}}"{{ end }}>
+	<div class="code-nav flex flex-nowrap items-stretch">
+		{{- with $file -}}
+			<div class="san-serif f6 dib lh-solid pl2 pv2 mr2">{{ . }}.</div>
+		{{- end -}}
+		{{ range $langs }}
+			<button data-toggle-tab="{{ . }}" class="tab-button {{ cond (eq . "yaml") "active" ""}} ba san-serif f6 dib lh-solid ph2 pv2">{{ . }}</button>&nbsp;
+		{{ end }}
+	</div>
+	<div class="tab-content">
+		{{ range $langs }}
+			<div data-pane="{{ . }}" class="code-copy-content nt3 tab-pane {{ cond (eq . "yaml") "active" ""}}">
+				{{ highlight ($code | transform.Remarshal . | safeHTML) . ""}}
+			</div>
+			{{ if ne ($.Get "copy") "false" }}
+				<button class="needs-js copy copy-toggle bg-accent-color-dark f6 absolute top-0 right-0 lh-solid hover-bg-primary-color-dark bn white ph3 pv2" title="Copy this code to your clipboard." data-clipboard-action="copy" aria-label="copy button">
+				</button>
+				{{/* Functionality located within filesaver.js The copy here is located in the css with .copy class so it can be replaced with JS on success */}}
+			{{end}}
+		{{ end }}
+	</div>
+
+</div>

docs/layouts/shortcodes/new-in.html

@@ -0,0 +1,8 @@
+{{ $version := .Get 0 }}
+{{ if not $version }}
+{{ errorf "Missing version in new-in shortcode "}}
+{{ end }}
+{{ $version = $version | strings.TrimPrefix "v" }}
+<button class="bg-white hover:bg-gray-100 text-gray-800 font-semibold py-2 mr2 ml2 px-4 border border-gray-400 rounded shadow">
+    <a href="{{ printf "https://gohugo.io/news/%s-relnotes/" $version }}" target="_blank">New in v{{$version}}</a>
+</button>

docs/requirements.txt

@@ -1 +0,0 @@
-Pygments==2.1.3

go.mod

@@ -5,7 +5,7 @@ require (
 	github.com/BurntSushi/toml v0.3.1
 	github.com/PuerkitoBio/purell v1.1.0
 	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
-	github.com/alecthomas/chroma v0.6.9
+	github.com/alecthomas/chroma v0.6.10-0.20191121231300-5921c52787e3
 	github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1 // indirect
 	github.com/armon/go-radix v1.0.0
 	github.com/aws/aws-sdk-go v1.19.40
@@ -54,6 +54,7 @@ require (
 	github.com/spf13/viper v1.4.0
 	github.com/tdewolff/minify/v2 v2.5.2
 	github.com/yosssi/ace v0.0.5
+	github.com/yuin/goldmark v1.1.4
 	go.opencensus.io v0.22.0 // indirect
 	gocloud.dev v0.15.0
 	golang.org/x/image v0.0.0-20190523035834-f03afa92d3ff

go.sum

@@ -40,8 +40,8 @@ github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMx
 github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c=
 github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38 h1:smF2tmSOzy2Mm+0dGI2AIUHY+w0BUc+4tn40djz7+6U=
 github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38/go.mod h1:r7bzyVFMNntcxPZXK3/+KdruV1H5KSlyVY0gc+NgInI=
-github.com/alecthomas/chroma v0.6.9 h1:afiCdwnNPo6fcyvoqqsXs78t7NbR9TuW4wDB7NJkcag=
+github.com/alecthomas/chroma v0.6.10-0.20191121231300-5921c52787e3 h1:7QpewsuR2wMuqAR9SADf+Cb043SlnmlA7LtDJGDx7UQ=
-github.com/alecthomas/chroma v0.6.9/go.mod h1:1U/PfCsTALWWYHDnsIQkxEBM0+6LLe0v8+RSVMOwxeY=
+github.com/alecthomas/chroma v0.6.10-0.20191121231300-5921c52787e3/go.mod h1:1U/PfCsTALWWYHDnsIQkxEBM0+6LLe0v8+RSVMOwxeY=
 github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721 h1:JHZL0hZKJ1VENNfmXvHbgYlbUOvpzYzvy2aZU5gXVeo=
 github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721/go.mod h1:QO9JBoKquHd+jz9nshCh40fOfO+JzsoXy8qTHF68zU0=
 github.com/alecthomas/kong v0.1.17-0.20190424132513-439c674f7ae0/go.mod h1:+inYUSluD+p4L8KdviBSgzcqEjUQOfC5fQDRFuc36lI=
@@ -116,8 +116,6 @@ github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/frankban/quicktest v1.4.1 h1:Wv2VwvNn73pAdFIVUQRXYDFp31lXKbqblIXo/Q5GPSg=
 github.com/frankban/quicktest v1.4.1/go.mod h1:36zfPVQyHxymz4cH7wlDmVwDrJuljRB60qkgn7rorfQ=
-github.com/frankban/quicktest v1.5.0 h1:Tb4jWdSpdjKzTUicPnY61PZxKbDoGa7ABbrReT3gQVY=
-github.com/frankban/quicktest v1.5.0/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o=
 github.com/frankban/quicktest v1.6.0 h1:Cd62nl66vQsx8Uv1t8M0eICyxIwZG7MxiAOrdnnUSW0=
 github.com/frankban/quicktest v1.6.0/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
@@ -349,6 +347,8 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/yosssi/ace v0.0.5 h1:tUkIP/BLdKqrlrPwcmH0shwEEhTRHoGnc1wFIWmaBUA=
 github.com/yosssi/ace v0.0.5/go.mod h1:ALfIzm2vT7t5ZE7uoIZqF3TQ7SAOyupFZnkrF5id+K0=
+github.com/yuin/goldmark v1.1.4 h1:Fj9vOhXMWRBITkIfa8OG/5j6PTKPkyPHxZbT1bvmjV8=
+github.com/yuin/goldmark v1.1.4/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.mongodb.org/mongo-driver v1.0.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
 go.opencensus.io v0.15.0/go.mod h1:UffZAU+4sDEINUGP/B7UfBBkq4fqLu9zXAX7ke6CHW0=

helpers/content.go

@@ -32,7 +32,6 @@ import (
 	bp "github.com/gohugoio/hugo/bufferpool"
 	"github.com/gohugoio/hugo/config"
 	"github.com/spf13/afero"
-	jww "github.com/spf13/jwalterweatherman"
 
 	"strings"
 )
@@ -58,9 +57,6 @@ type ContentSpec struct {
 	BuildExpired bool
 	BuildDrafts  bool
 
-	Highlight            func(code, lang, optsStr string) (string, error)
-	defatultPygmentsOpts map[string]string
-
 	Cfg config.Provider
 }
 
@@ -77,36 +73,10 @@ func NewContentSpec(cfg config.Provider, logger *loggers.Logger, contentFs afero
 		Cfg: cfg,
 	}
 
-	// Highlighting setup
-	options, err := parseDefaultPygmentsOpts(cfg)
-	if err != nil {
-		return nil, err
-	}
-	spec.defatultPygmentsOpts = options
-
-	// Use the Pygmentize on path if present
-	useClassic := false
-	h := newHiglighters(spec)
-
-	if cfg.GetBool("pygmentsUseClassic") {
-		if !hasPygments() {
-			jww.WARN.Println("Highlighting with pygmentsUseClassic set requires Pygments to be installed and in the path")
-		} else {
-			useClassic = true
-		}
-	}
-
-	if useClassic {
-		spec.Highlight = h.pygmentsHighlight
-	} else {
-		spec.Highlight = h.chromaHighlight
-	}
-
 	converterProvider, err := markup.NewConverterProvider(converter.ProviderConfig{
 		Cfg:       cfg,
 		ContentFs: contentFs,
 		Logger:    logger,
-		Highlight: spec.Highlight,
 	})
 	if err != nil {
 		return nil, err
@@ -220,6 +190,21 @@ func (c *ContentSpec) RenderMarkdown(src []byte) ([]byte, error) {
 	return b.Bytes(), nil
 }
 
+func (c *ContentSpec) ResolveMarkup(in string) string {
+	in = strings.ToLower(in)
+	switch in {
+	case "md", "markdown", "mdown":
+		return "markdown"
+	case "html", "htm":
+		return "html"
+	default:
+		if conv := c.Converters.Get(in); conv != nil {
+			return conv.Name()
+		}
+	}
+	return ""
+}
+
 // TotalWords counts instance of one or more consecutive white space
 // characters, as defined by unicode.IsSpace, in s.
 // This is a cheaper way of word counting than the obvious len(strings.Fields(s)).

helpers/general.go

@@ -44,11 +44,6 @@ import (
 // FilePathSeparator as defined by os.Separator.
 const FilePathSeparator = string(filepath.Separator)
 
-// Strips carriage returns from third-party / external processes (useful for Windows)
-func normalizeExternalHelperLineFeeds(content []byte) []byte {
-	return bytes.Replace(content, []byte("\r"), []byte(""), -1)
-}
-
 // FindAvailablePort returns an available and valid TCP port.
 func FindAvailablePort() (*net.TCPAddr, error) {
 	l, err := net.Listen("tcp", ":0")
@@ -74,28 +69,6 @@ func InStringArray(arr []string, el string) bool {
 	return false
 }
 
-// GuessType attempts to guess the type of file from a given string.
-func GuessType(in string) string {
-	switch strings.ToLower(in) {
-	case "md", "markdown", "mdown":
-		return "markdown"
-	case "asciidoc", "adoc", "ad":
-		return "asciidoc"
-	case "mmark":
-		return "mmark"
-	case "rst":
-		return "rst"
-	case "pandoc", "pdc":
-		return "pandoc"
-	case "html", "htm":
-		return "html"
-	case "org":
-		return "org"
-	}
-
-	return ""
-}
-
 // FirstUpper returns a string with the first character as upper case.
 func FirstUpper(s string) string {
 	if s == "" {

helpers/general_test.go

@@ -19,11 +19,20 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/spf13/viper"
+
+	"github.com/gohugoio/hugo/common/loggers"
+
 	qt "github.com/frankban/quicktest"
 	"github.com/spf13/afero"
 )
 
-func TestGuessType(t *testing.T) {
+func TestResolveMarkup(t *testing.T) {
+	c := qt.New(t)
+	cfg := viper.New()
+	spec, err := NewContentSpec(cfg, loggers.NewErrorLogger(), afero.NewMemMapFs())
+	c.Assert(err, qt.IsNil)
+
 	for i, this := range []struct {
 		in     string
 		expect string
@@ -43,7 +52,7 @@ func TestGuessType(t *testing.T) {
 		{"org", "org"},
 		{"excel", ""},
 	} {
-		result := GuessType(this.in)
+		result := spec.ResolveMarkup(this.in)
 		if result != this.expect {
 			t.Errorf("[%d] got %s but expected %s", i, result, this.expect)
 		}

helpers/pygments.go

@@ -1,402 +0,0 @@
-// Copyright 2019 The Hugo Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package helpers
-
-import (
-	"bytes"
-	"crypto/sha1"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"os/exec"
-	"path/filepath"
-	"regexp"
-	"sort"
-	"strconv"
-	"strings"
-
-	"github.com/alecthomas/chroma"
-	"github.com/alecthomas/chroma/formatters"
-	"github.com/alecthomas/chroma/formatters/html"
-	"github.com/alecthomas/chroma/lexers"
-	"github.com/alecthomas/chroma/styles"
-	bp "github.com/gohugoio/hugo/bufferpool"
-
-	"github.com/gohugoio/hugo/config"
-	"github.com/gohugoio/hugo/hugofs"
-	jww "github.com/spf13/jwalterweatherman"
-)
-
-const pygmentsBin = "pygmentize"
-
-// hasPygments checks to see if Pygments is installed and available
-// on the system.
-func hasPygments() bool {
-	if _, err := exec.LookPath(pygmentsBin); err != nil {
-		return false
-	}
-	return true
-}
-
-type highlighters struct {
-	cs          *ContentSpec
-	ignoreCache bool
-	cacheDir    string
-}
-
-func newHiglighters(cs *ContentSpec) highlighters {
-	return highlighters{cs: cs, ignoreCache: cs.Cfg.GetBool("ignoreCache"), cacheDir: cs.Cfg.GetString("cacheDir")}
-}
-
-func (h highlighters) chromaHighlight(code, lang, optsStr string) (string, error) {
-	opts, err := h.cs.parsePygmentsOpts(optsStr)
-	if err != nil {
-		jww.ERROR.Print(err.Error())
-		return code, err
-	}
-
-	style, found := opts["style"]
-	if !found || style == "" {
-		style = "friendly"
-	}
-
-	f, err := h.cs.chromaFormatterFromOptions(opts)
-	if err != nil {
-		jww.ERROR.Print(err.Error())
-		return code, err
-	}
-
-	b := bp.GetBuffer()
-	defer bp.PutBuffer(b)
-
-	err = chromaHighlight(b, code, lang, style, f)
-	if err != nil {
-		jww.ERROR.Printf("Highlight failed: %s\nLang: %q\nCode: \n%s", err, lang, code)
-		return code, err
-	}
-
-	return h.injectCodeTag(`<div class="highlight">`+b.String()+"</div>", lang), nil
-}
-
-func (h highlighters) pygmentsHighlight(code, lang, optsStr string) (string, error) {
-	options, err := h.cs.createPygmentsOptionsString(optsStr)
-
-	if err != nil {
-		jww.ERROR.Print(err.Error())
-		return code, nil
-	}
-
-	// Try to read from cache first
-	hash := sha1.New()
-	io.WriteString(hash, code)
-	io.WriteString(hash, lang)
-	io.WriteString(hash, options)
-
-	fs := hugofs.Os
-
-	var cachefile string
-
-	if !h.ignoreCache && h.cacheDir != "" {
-		cachefile = filepath.Join(h.cacheDir, fmt.Sprintf("pygments-%x", hash.Sum(nil)))
-
-		exists, err := Exists(cachefile, fs)
-		if err != nil {
-			jww.ERROR.Print(err.Error())
-			return code, nil
-		}
-		if exists {
-			f, err := fs.Open(cachefile)
-			if err != nil {
-				jww.ERROR.Print(err.Error())
-				return code, nil
-			}
-
-			s, err := ioutil.ReadAll(f)
-			if err != nil {
-				jww.ERROR.Print(err.Error())
-				return code, nil
-			}
-
-			return string(s), nil
-		}
-	}
-
-	// No cache file, render and cache it
-	var out bytes.Buffer
-	var stderr bytes.Buffer
-
-	var langOpt string
-	if lang == "" {
-		langOpt = "-g" // Try guessing the language
-	} else {
-		langOpt = "-l" + lang
-	}
-
-	cmd := exec.Command(pygmentsBin, langOpt, "-fhtml", "-O", options)
-	cmd.Stdin = strings.NewReader(code)
-	cmd.Stdout = &out
-	cmd.Stderr = &stderr
-
-	if err := cmd.Run(); err != nil {
-		jww.ERROR.Print(stderr.String())
-		return code, err
-	}
-
-	str := string(normalizeExternalHelperLineFeeds(out.Bytes()))
-
-	str = h.injectCodeTag(str, lang)
-
-	if !h.ignoreCache && cachefile != "" {
-		// Write cache file
-		if err := WriteToDisk(cachefile, strings.NewReader(str), fs); err != nil {
-			jww.ERROR.Print(stderr.String())
-		}
-	}
-
-	return str, nil
-}
-
-var preRe = regexp.MustCompile(`(?s)(.*?<pre.*?>)(.*?)(</pre>)`)
-
-func (h highlighters) injectCodeTag(code, lang string) string {
-	if lang == "" {
-		return code
-	}
-	codeTag := fmt.Sprintf(`<code class="language-%s" data-lang="%s">`, lang, lang)
-	return preRe.ReplaceAllString(code, fmt.Sprintf("$1%s$2</code>$3", codeTag))
-}
-
-func chromaHighlight(w io.Writer, source, lexer, style string, f chroma.Formatter) error {
-	l := lexers.Get(lexer)
-	if l == nil {
-		l = lexers.Analyse(source)
-	}
-	if l == nil {
-		l = lexers.Fallback
-	}
-	l = chroma.Coalesce(l)
-
-	if f == nil {
-		f = formatters.Fallback
-	}
-
-	s := styles.Get(style)
-	if s == nil {
-		s = styles.Fallback
-	}
-
-	it, err := l.Tokenise(nil, source)
-	if err != nil {
-		return err
-	}
-
-	return f.Format(w, s, it)
-}
-
-var pygmentsKeywords = make(map[string]bool)
-
-func init() {
-	pygmentsKeywords["encoding"] = true
-	pygmentsKeywords["outencoding"] = true
-	pygmentsKeywords["nowrap"] = true
-	pygmentsKeywords["full"] = true
-	pygmentsKeywords["title"] = true
-	pygmentsKeywords["style"] = true
-	pygmentsKeywords["noclasses"] = true
-	pygmentsKeywords["classprefix"] = true
-	pygmentsKeywords["cssclass"] = true
-	pygmentsKeywords["cssstyles"] = true
-	pygmentsKeywords["prestyles"] = true
-	pygmentsKeywords["linenos"] = true
-	pygmentsKeywords["hl_lines"] = true
-	pygmentsKeywords["linenostart"] = true
-	pygmentsKeywords["linenostep"] = true
-	pygmentsKeywords["linenospecial"] = true
-	pygmentsKeywords["nobackground"] = true
-	pygmentsKeywords["lineseparator"] = true
-	pygmentsKeywords["lineanchors"] = true
-	pygmentsKeywords["linespans"] = true
-	pygmentsKeywords["anchorlinenos"] = true
-	pygmentsKeywords["startinline"] = true
-}
-
-func parseOptions(defaults map[string]string, in string) (map[string]string, error) {
-	in = strings.Trim(in, " ")
-	opts := make(map[string]string)
-
-	for k, v := range defaults {
-		opts[k] = v
-	}
-
-	if in == "" {
-		return opts, nil
-	}
-
-	for _, v := range strings.Split(in, ",") {
-		keyVal := strings.Split(v, "=")
-		key := strings.ToLower(strings.Trim(keyVal[0], " "))
-		if len(keyVal) != 2 || !pygmentsKeywords[key] {
-			return opts, fmt.Errorf("invalid Pygments option: %s", key)
-		}
-		opts[key] = keyVal[1]
-	}
-
-	return opts, nil
-}
-
-func createOptionsString(options map[string]string) string {
-	var keys []string
-	for k := range options {
-		keys = append(keys, k)
-	}
-	sort.Strings(keys)
-
-	var optionsStr string
-	for i, k := range keys {
-		optionsStr += fmt.Sprintf("%s=%s", k, options[k])
-		if i < len(options)-1 {
-			optionsStr += ","
-		}
-	}
-
-	return optionsStr
-}
-
-func parseDefaultPygmentsOpts(cfg config.Provider) (map[string]string, error) {
-	options, err := parseOptions(nil, cfg.GetString("pygmentsOptions"))
-	if err != nil {
-		return nil, err
-	}
-
-	if cfg.IsSet("pygmentsStyle") {
-		options["style"] = cfg.GetString("pygmentsStyle")
-	}
-
-	if cfg.IsSet("pygmentsUseClasses") {
-		if cfg.GetBool("pygmentsUseClasses") {
-			options["noclasses"] = "false"
-		} else {
-			options["noclasses"] = "true"
-		}
-
-	}
-
-	if _, ok := options["encoding"]; !ok {
-		options["encoding"] = "utf8"
-	}
-
-	return options, nil
-}
-
-func (cs *ContentSpec) chromaFormatterFromOptions(pygmentsOpts map[string]string) (chroma.Formatter, error) {
-	var options = []html.Option{html.TabWidth(4)}
-
-	if pygmentsOpts["noclasses"] == "false" {
-		options = append(options, html.WithClasses())
-	}
-
-	lineNumbers := pygmentsOpts["linenos"]
-	if lineNumbers != "" {
-		options = append(options, html.WithLineNumbers())
-		if lineNumbers != "inline" {
-			options = append(options, html.LineNumbersInTable())
-		}
-	}
-
-	startLineStr := pygmentsOpts["linenostart"]
-	var startLine = 1
-	if startLineStr != "" {
-
-		line, err := strconv.Atoi(strings.TrimSpace(startLineStr))
-		if err == nil {
-			startLine = line
-			options = append(options, html.BaseLineNumber(startLine))
-		}
-	}
-
-	hlLines := pygmentsOpts["hl_lines"]
-
-	if hlLines != "" {
-		ranges, err := hlLinesToRanges(startLine, hlLines)
-
-		if err == nil {
-			options = append(options, html.HighlightLines(ranges))
-		}
-	}
-
-	return html.New(options...), nil
-}
-
-func (cs *ContentSpec) parsePygmentsOpts(in string) (map[string]string, error) {
-	opts, err := parseOptions(cs.defatultPygmentsOpts, in)
-	if err != nil {
-		return nil, err
-	}
-	return opts, nil
-
-}
-
-func (cs *ContentSpec) createPygmentsOptionsString(in string) (string, error) {
-	opts, err := cs.parsePygmentsOpts(in)
-	if err != nil {
-		return "", err
-	}
-	return createOptionsString(opts), nil
-}
-
-// startLine compansates for https://github.com/alecthomas/chroma/issues/30
-func hlLinesToRanges(startLine int, s string) ([][2]int, error) {
-	var ranges [][2]int
-	s = strings.TrimSpace(s)
-
-	if s == "" {
-		return ranges, nil
-	}
-
-	// Variants:
-	// 1 2 3 4
-	// 1-2 3-4
-	// 1-2 3
-	// 1 3-4
-	// 1    3-4
-	fields := strings.Split(s, " ")
-	for _, field := range fields {
-		field = strings.TrimSpace(field)
-		if field == "" {
-			continue
-		}
-		numbers := strings.Split(field, "-")
-		var r [2]int
-		first, err := strconv.Atoi(numbers[0])
-		if err != nil {
-			return ranges, err
-		}
-		first = first + startLine - 1
-		r[0] = first
-		if len(numbers) > 1 {
-			second, err := strconv.Atoi(numbers[1])
-			if err != nil {
-				return ranges, err
-			}
-			second = second + startLine - 1
-			r[1] = second
-		} else {
-			r[1] = first
-		}
-
-		ranges = append(ranges, r)
-	}
-	return ranges, nil
-
-}

helpers/pygments_test.go

@@ -1,300 +0,0 @@
-// Copyright 2015 The Hugo Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package helpers
-
-import (
-	"fmt"
-	"reflect"
-	"testing"
-
-	"github.com/alecthomas/chroma/formatters/html"
-
-	qt "github.com/frankban/quicktest"
-	"github.com/spf13/viper"
-)
-
-func TestParsePygmentsArgs(t *testing.T) {
-	c := qt.New(t)
-
-	for i, this := range []struct {
-		in                 string
-		pygmentsStyle      string
-		pygmentsUseClasses bool
-		expect1            interface{}
-	}{
-		{"", "foo", true, "encoding=utf8,noclasses=false,style=foo"},
-		{"style=boo,noclasses=true", "foo", true, "encoding=utf8,noclasses=true,style=boo"},
-		{"Style=boo, noClasses=true", "foo", true, "encoding=utf8,noclasses=true,style=boo"},
-		{"noclasses=true", "foo", true, "encoding=utf8,noclasses=true,style=foo"},
-		{"style=boo", "foo", true, "encoding=utf8,noclasses=false,style=boo"},
-		{"boo=invalid", "foo", false, false},
-		{"style", "foo", false, false},
-	} {
-
-		v := viper.New()
-		v.Set("pygmentsStyle", this.pygmentsStyle)
-		v.Set("pygmentsUseClasses", this.pygmentsUseClasses)
-		spec, err := NewContentSpec(v, nil, nil)
-		c.Assert(err, qt.IsNil)
-
-		result1, err := spec.createPygmentsOptionsString(this.in)
-		if b, ok := this.expect1.(bool); ok && !b {
-			if err == nil {
-				t.Errorf("[%d] parsePygmentArgs didn't return an expected error", i)
-			}
-		} else {
-			if err != nil {
-				t.Errorf("[%d] parsePygmentArgs failed: %s", i, err)
-				continue
-			}
-			if result1 != this.expect1 {
-				t.Errorf("[%d] parsePygmentArgs got %v but expected %v", i, result1, this.expect1)
-			}
-
-		}
-	}
-}
-
-func TestParseDefaultPygmentsArgs(t *testing.T) {
-	c := qt.New(t)
-
-	expect := "encoding=utf8,noclasses=false,style=foo"
-
-	for i, this := range []struct {
-		in                 string
-		pygmentsStyle      interface{}
-		pygmentsUseClasses interface{}
-		pygmentsOptions    string
-	}{
-		{"", "foo", true, "style=override,noclasses=override"},
-		{"", nil, nil, "style=foo,noclasses=false"},
-		{"style=foo,noclasses=false", nil, nil, "style=override,noclasses=override"},
-		{"style=foo,noclasses=false", "override", false, "style=override,noclasses=override"},
-	} {
-		v := viper.New()
-
-		v.Set("pygmentsOptions", this.pygmentsOptions)
-
-		if s, ok := this.pygmentsStyle.(string); ok {
-			v.Set("pygmentsStyle", s)
-		}
-
-		if b, ok := this.pygmentsUseClasses.(bool); ok {
-			v.Set("pygmentsUseClasses", b)
-		}
-
-		spec, err := NewContentSpec(v, nil, nil)
-		c.Assert(err, qt.IsNil)
-
-		result, err := spec.createPygmentsOptionsString(this.in)
-		if err != nil {
-			t.Errorf("[%d] parsePygmentArgs failed: %s", i, err)
-			continue
-		}
-		if result != expect {
-			t.Errorf("[%d] parsePygmentArgs got %v but expected %v", i, result, expect)
-		}
-	}
-}
-
-type chromaInfo struct {
-	classes            bool
-	lineNumbers        bool
-	lineNumbersInTable bool
-	highlightRangesLen int
-	highlightRangesStr string
-	baseLineNumber     int
-}
-
-func formatterChromaInfo(f *html.Formatter) chromaInfo {
-	v := reflect.ValueOf(f).Elem()
-	c := chromaInfo{}
-	// Hack:
-
-	c.classes = f.Classes
-	c.lineNumbers = v.FieldByName("lineNumbers").Bool()
-	c.lineNumbersInTable = v.FieldByName("lineNumbersInTable").Bool()
-	c.baseLineNumber = int(v.FieldByName("baseLineNumber").Int())
-	vv := v.FieldByName("highlightRanges")
-	c.highlightRangesLen = vv.Len()
-	c.highlightRangesStr = fmt.Sprint(vv)
-
-	return c
-}
-
-func TestChromaHTMLHighlight(t *testing.T) {
-	c := qt.New(t)
-
-	v := viper.New()
-	v.Set("pygmentsUseClasses", true)
-	spec, err := NewContentSpec(v, nil, nil)
-	c.Assert(err, qt.IsNil)
-
-	result, err := spec.Highlight(`echo "Hello"`, "bash", "")
-	c.Assert(err, qt.IsNil)
-
-	c.Assert(result, qt.Contains, `<div class="highlight"><pre class="chroma"><code class="language-bash" data-lang="bash"><span class="nb">echo</span> <span class="s2">&#34;Hello&#34;</span></code></pre></div>`)
-
-}
-
-func TestChromaHTMLFormatterFromOptions(t *testing.T) {
-	c := qt.New(t)
-
-	for i, this := range []struct {
-		in                 string
-		pygmentsStyle      interface{}
-		pygmentsUseClasses interface{}
-		pygmentsOptions    string
-		assert             func(c chromaInfo)
-	}{
-		{"", "monokai", true, "style=manni,noclasses=true", func(ci chromaInfo) {
-			c.Assert(ci.classes, qt.Equals, true)
-			c.Assert(ci.lineNumbers, qt.Equals, false)
-			c.Assert(ci.highlightRangesLen, qt.Equals, 0)
-
-		}},
-		{"", nil, nil, "style=monokai,noclasses=false", func(ci chromaInfo) {
-			c.Assert(ci.classes, qt.Equals, true)
-		}},
-		{"linenos=sure,hl_lines=1 2 3", nil, nil, "style=monokai,noclasses=false", func(ci chromaInfo) {
-			c.Assert(ci.classes, qt.Equals, true)
-			c.Assert(ci.lineNumbers, qt.Equals, true)
-			c.Assert(ci.highlightRangesLen, qt.Equals, 3)
-			c.Assert(ci.highlightRangesStr, qt.Equals, "[[1 1] [2 2] [3 3]]")
-			c.Assert(ci.baseLineNumber, qt.Equals, 1)
-		}},
-		{"linenos=inline,hl_lines=1,linenostart=4", nil, nil, "style=monokai,noclasses=false", func(ci chromaInfo) {
-			c.Assert(ci.classes, qt.Equals, true)
-			c.Assert(ci.lineNumbers, qt.Equals, true)
-			c.Assert(ci.lineNumbersInTable, qt.Equals, false)
-			c.Assert(ci.highlightRangesLen, qt.Equals, 1)
-			// This compansates for https://github.com/alecthomas/chroma/issues/30
-			c.Assert(ci.highlightRangesStr, qt.Equals, "[[4 4]]")
-			c.Assert(ci.baseLineNumber, qt.Equals, 4)
-		}},
-		{"linenos=table", nil, nil, "style=monokai", func(ci chromaInfo) {
-			c.Assert(ci.lineNumbers, qt.Equals, true)
-			c.Assert(ci.lineNumbersInTable, qt.Equals, true)
-		}},
-		{"style=monokai,noclasses=false", nil, nil, "style=manni,noclasses=true", func(ci chromaInfo) {
-			c.Assert(ci.classes, qt.Equals, true)
-		}},
-		{"style=monokai,noclasses=true", "friendly", false, "style=manni,noclasses=false", func(ci chromaInfo) {
-			c.Assert(ci.classes, qt.Equals, false)
-		}},
-	} {
-		v := viper.New()
-
-		v.Set("pygmentsOptions", this.pygmentsOptions)
-
-		if s, ok := this.pygmentsStyle.(string); ok {
-			v.Set("pygmentsStyle", s)
-		}
-
-		if b, ok := this.pygmentsUseClasses.(bool); ok {
-			v.Set("pygmentsUseClasses", b)
-		}
-
-		spec, err := NewContentSpec(v, nil, nil)
-		c.Assert(err, qt.IsNil)
-
-		opts, err := spec.parsePygmentsOpts(this.in)
-		if err != nil {
-			t.Fatalf("[%d] parsePygmentsOpts failed: %s", i, err)
-		}
-
-		chromaFormatter, err := spec.chromaFormatterFromOptions(opts)
-		if err != nil {
-			t.Fatalf("[%d] chromaFormatterFromOptions failed: %s", i, err)
-		}
-
-		this.assert(formatterChromaInfo(chromaFormatter.(*html.Formatter)))
-	}
-}
-
-func TestHlLinesToRanges(t *testing.T) {
-	var zero [][2]int
-
-	for _, this := range []struct {
-		in        string
-		startLine int
-		expected  interface{}
-	}{
-		{"", 1, zero},
-		{"1 4", 1, [][2]int{{1, 1}, {4, 4}}},
-		{"1 4", 2, [][2]int{{2, 2}, {5, 5}}},
-		{"1-4 5-8", 1, [][2]int{{1, 4}, {5, 8}}},
-		{" 1   4 ", 1, [][2]int{{1, 1}, {4, 4}}},
-		{"1-4    5-8 ", 1, [][2]int{{1, 4}, {5, 8}}},
-		{"1-4 5", 1, [][2]int{{1, 4}, {5, 5}}},
-		{"4 5-9", 1, [][2]int{{4, 4}, {5, 9}}},
-		{" 1 -4 5 - 8  ", 1, true},
-		{"a b", 1, true},
-	} {
-		got, err := hlLinesToRanges(this.startLine, this.in)
-
-		if expectErr, ok := this.expected.(bool); ok && expectErr {
-			if err == nil {
-				t.Fatal("No error")
-			}
-		} else if err != nil {
-			t.Fatalf("Got error: %s", err)
-		} else if !reflect.DeepEqual(this.expected, got) {
-			t.Fatalf("Expected\n%v but got\n%v", this.expected, got)
-		}
-	}
-}
-
-func BenchmarkChromaHighlight(b *testing.B) {
-	c := qt.New(b)
-	v := viper.New()
-
-	v.Set("pygmentsstyle", "trac")
-	v.Set("pygmentsuseclasses", false)
-	v.Set("pygmentsuseclassic", false)
-
-	code := `// GetTitleFunc returns a func that can be used to transform a string to
-// title case.
-//
-// The supported styles are
-//
-// - "Go" (strings.Title)
-// - "AP" (see https://www.apstylebook.com/)
-// - "Chicago" (see http://www.chicagomanualofstyle.org/home.html)
-//
-// If an unknown or empty style is provided, AP style is what you get.
-func GetTitleFunc(style string) func(s string) string {
-  switch strings.ToLower(style) {
-  case "go":
-    return strings.Title
-  case "chicago":
-    tc := transform.NewTitleConverter(transform.ChicagoStyle)
-    return tc.Title
-  default:
-    tc := transform.NewTitleConverter(transform.APStyle)
-    return tc.Title
-  }
-}
-`
-
-	spec, err := NewContentSpec(v, nil, nil)
-	c.Assert(err, qt.IsNil)
-
-	for i := 0; i < b.N; i++ {
-		_, err := spec.Highlight(code, "go", "linenos=inline,hl_lines=8 15-17")
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}

hugolib/case_insensitive_test.go

@@ -208,7 +208,6 @@ Page2: {{ $page2.Params.ColoR }}
 		"Partial Site Global: green|yellow",
 		"Page Title: Side 1",
 		"Site Title: Nynorsk title",
-		"«Hi»", // angled quotes
 		"Page2: black ",
 	)
 

hugolib/config.go

@@ -597,11 +597,6 @@ func loadDefaultSettingsFor(v *viper.Viper) error {
 	v.SetDefault("taxonomies", map[string]string{"tag": "tags", "category": "categories"})
 	v.SetDefault("permalinks", make(map[string]string))
 	v.SetDefault("sitemap", config.Sitemap{Priority: -1, Filename: "sitemap.xml"})
-	v.SetDefault("pygmentsStyle", "monokai")
-	v.SetDefault("pygmentsUseClasses", false)
-	v.SetDefault("pygmentsCodeFences", false)
-	v.SetDefault("pygmentsUseClassic", false)
-	v.SetDefault("pygmentsOptions", "")
 	v.SetDefault("disableLiveReload", false)
 	v.SetDefault("pluralizeListTitles", true)
 	v.SetDefault("forceSyncStatic", false)

hugolib/hugo_sites_build_test.go

@@ -671,7 +671,7 @@ END
 
 	b.CreateSites().Build(BuildCfg{})
 
-	contentMatchers := []string{"<h2 id=\"another-header\">Another header</h2>", "<h2 id=\"another-header-99\">Another header</h2>", "<p>The End.</p>"}
+	contentMatchers := []string{"<h2 id=\"another-header\">Another header</h2>", "<h2 id=\"another-header99\">Another header</h2>", "<p>The End.</p>"}
 
 	for i := 1; i <= numPages; i++ {
 		if i%3 != 0 {
@@ -691,13 +691,13 @@ END
 		checkContent(b, fmt.Sprintf("public/%s/page%d/index.json", section, i), contentMatchers...)
 	}
 
-	checkContent(b, "public/s1/index.html", "P: s1/_index.md\nList: 10|List Content: 8335\n\n\nL1: 500 L2: 5\n\nRender 0: View: 8335\n\nRender 1: View: 8335\n\nRender 2: View: 8335\n\nRender 3: View: 8335\n\nRender 4: View: 8335\n\nEND\n")
+	checkContent(b, "public/s1/index.html", "P: s1/_index.md\nList: 10|List Content: 8033\n\n\nL1: 500 L2: 5\n\nRender 0: View: 8033\n\nRender 1: View: 8033\n\nRender 2: View: 8033\n\nRender 3: View: 8033\n\nRender 4: View: 8033\n\nEND\n")
-	checkContent(b, "public/s2/index.html", "P: s2/_index.md\nList: 10|List Content: 8335", "Render 4: View: 8335\n\nEND")
+	checkContent(b, "public/s2/index.html", "P: s2/_index.md\nList: 10|List Content: 8033", "Render 4: View: 8033\n\nEND")
-	checkContent(b, "public/index.html", "P: _index.md\nList: 10|List Content: 8335", "4: View: 8335\n\nEND")
+	checkContent(b, "public/index.html", "P: _index.md\nList: 10|List Content: 8033", "4: View: 8033\n\nEND")
 
 	// Check paginated pages
 	for i := 2; i <= 9; i++ {
-		checkContent(b, fmt.Sprintf("public/page/%d/index.html", i), fmt.Sprintf("Page: %d", i), "Content: 8335\n\n\nL1: 500 L2: 5\n\nRender 0: View: 8335", "Render 4: View: 8335\n\nEND")
+		checkContent(b, fmt.Sprintf("public/page/%d/index.html", i), fmt.Sprintf("Page: %d", i), "Content: 8033\n\n\nL1: 500 L2: 5\n\nRender 0: View: 8033", "Render 4: View: 8033\n\nEND")
 	}
 }
 
@@ -977,7 +977,10 @@ enableRobotsTXT = true
 [permalinks]
 other = "/somewhere/else/:filename"
 
-[blackfriday]
+# TODO(bep)
+[markup]
+  defaultMarkdownHandler = "blackfriday"
+[markup.blackfriday]
 angledQuotes = true
 
 [Taxonomies]
@@ -1035,7 +1038,10 @@ enableRobotsTXT: true
 permalinks:
     other: "/somewhere/else/:filename"
 
-blackfriday:
+# TODO(bep)
+markup:
+  defaultMarkdownHandler: blackfriday
+  blackFriday:
     angledQuotes: true
 
 Taxonomies:
@@ -1093,9 +1099,12 @@ var multiSiteJSONConfigTemplate = `
   "permalinks": {
     "other": "/somewhere/else/:filename"
   },
-  "blackfriday": {
+  "markup": {
-    "angledQuotes": true
+		"defaultMarkdownHandler": "blackfriday",
-  },
+		"blackfriday": {
+	    "angledQuotes": true
+	  }
+   },
   "Taxonomies": {
     "tag": "tags"
   },

hugolib/page__meta.go

@@ -565,7 +565,7 @@ func (pm *pageMeta) setMetadata(bucket *pagesMapBucket, p *pageState, frontmatte
 		pm.sitemap = p.s.siteCfg.sitemap
 	}
 
-	pm.markup = helpers.GuessType(pm.markup)
+	pm.markup = p.s.ContentSpec.ResolveMarkup(pm.markup)
 
 	if draft != nil && published != nil {
 		pm.draft = *draft
@@ -596,7 +596,7 @@ func (p *pageMeta) applyDefaultValues() error {
 	if p.markup == "" {
 		if !p.File().IsZero() {
 			// Fall back to file extension
-			p.markup = helpers.GuessType(p.File().Ext())
+			p.markup = p.s.ContentSpec.ResolveMarkup(p.File().Ext())
 		}
 		if p.markup == "" {
 			p.markup = "markdown"
@@ -638,14 +638,20 @@ func (p *pageMeta) applyDefaultValues() error {
 		}
 	}
 
-	if !p.f.IsZero() && p.markup != "html" {
+	if !p.f.IsZero() {
 		var renderingConfigOverrides map[string]interface{}
 		bfParam := getParamToLower(p, "blackfriday")
 		if bfParam != nil {
 			renderingConfigOverrides = maps.ToStringMap(bfParam)
 		}
 
-		cp := p.s.ContentSpec.Converters.Get(p.markup)
+		markup := p.markup
+		if markup == "html" {
+			// Only used for shortcode inner content.
+			markup = "markdown"
+		}
+
+		cp := p.s.ContentSpec.Converters.Get(markup)
 		if cp == nil {
 			return errors.Errorf("no content renderer found for markup %q", p.markup)
 		}

hugolib/page__per_output.go

@@ -77,7 +77,7 @@ func newPageContentOutput(p *pageState) func(f output.Format) (*pageContentOutpu
 				// See https://github.com/gohugoio/hugo/issues/6210
 				if r := recover(); r != nil {
 					err = fmt.Errorf("%s", r)
-					p.s.Log.ERROR.Println("[BUG] Got panic:\n", string(debug.Stack()))
+					p.s.Log.ERROR.Printf("[BUG] Got panic:\n%s\n%s", r, string(debug.Stack()))
 				}
 			}()
 
@@ -103,11 +103,14 @@ func newPageContentOutput(p *pageState) func(f output.Format) (*pageContentOutpu
 					if err != nil {
 						return err
 					}
+					cp.convertedResult = r
 					cp.workContent = r.Bytes()
 
-					tmpContent, tmpTableOfContents := helpers.ExtractTOC(cp.workContent)
+					if _, ok := r.(converter.TableOfContentsProvider); !ok {
-					cp.tableOfContents = helpers.BytesToHTML(tmpTableOfContents)
+						tmpContent, tmpTableOfContents := helpers.ExtractTOC(cp.workContent)
-					cp.workContent = tmpContent
+						cp.tableOfContents = helpers.BytesToHTML(tmpTableOfContents)
+						cp.workContent = tmpContent
+					}
 				}
 
 				if cp.placeholdersEnabled {
@@ -223,7 +226,8 @@ type pageContentOutput struct {
 
 	// Content state
 
-	workContent []byte
+	workContent     []byte
+	convertedResult converter.Result
 
 	// Temporary storage of placeholders mapped to their content.
 	// These are shortcodes etc. Some of these will need to be replaced
@@ -284,6 +288,10 @@ func (p *pageContentOutput) Summary() template.HTML {
 
 func (p *pageContentOutput) TableOfContents() template.HTML {
 	p.p.s.initInit(p.initMain, p.p)
+	if tocProvider, ok := p.convertedResult.(converter.TableOfContentsProvider); ok {
+		cfg := p.p.s.ContentSpec.Converters.GetMarkupConfig()
+		return template.HTML(tocProvider.TableOfContents().ToHTML(cfg.TableOfContents.StartLevel, cfg.TableOfContents.EndLevel))
+	}
 	return p.tableOfContents
 }
 

hugolib/page_test.go

@@ -326,7 +326,7 @@ func normalizeContent(c string) string {
 
 func checkPageTOC(t *testing.T, page page.Page, toc string) {
 	if page.TableOfContents() != template.HTML(toc) {
-		t.Fatalf("Page TableOfContents is: %q.\nExpected %q", page.TableOfContents(), toc)
+		t.Fatalf("Page TableOfContents is:\n%q.\nExpected %q", page.TableOfContents(), toc)
 	}
 }
 
@@ -442,6 +442,7 @@ func testAllMarkdownEnginesForPages(t *testing.T,
 func TestPageWithDelimiterForMarkdownThatCrossesBorder(t *testing.T) {
 	t.Parallel()
 	cfg, fs := newTestCfg()
+
 	c := qt.New(t)
 
 	writeSource(t, fs, filepath.Join("content", "simple.md"), simplePageWithSummaryDelimiterAndMarkdownThatCrossesBorder)
@@ -453,12 +454,12 @@ func TestPageWithDelimiterForMarkdownThatCrossesBorder(t *testing.T) {
 	p := s.RegularPages()[0]
 
 	if p.Summary() != template.HTML(
-		"<p>The <a href=\"http://gohugo.io/\">best static site generator</a>.<sup class=\"footnote-ref\" id=\"fnref:1\"><a href=\"#fn:1\">1</a></sup></p>") {
+		"<p>The <a href=\"http://gohugo.io/\">best static site generator</a>.<sup id=\"fnref:1\"><a href=\"#fn:1\" class=\"footnote-ref\" role=\"doc-noteref\">1</a></sup></p>") {
 		t.Fatalf("Got summary:\n%q", p.Summary())
 	}
 
 	cnt := content(p)
-	if cnt != "<p>The <a href=\"http://gohugo.io/\">best static site generator</a>.<sup class=\"footnote-ref\" id=\"fnref:1\"><a href=\"#fn:1\">1</a></sup></p>\n\n<div class=\"footnotes\">\n\n<hr />\n\n<ol>\n<li id=\"fn:1\">Many people say so.\n <a class=\"footnote-return\" href=\"#fnref:1\"><sup>[return]</sup></a></li>\n</ol>\n</div>" {
+	if cnt != "<p>The <a href=\"http://gohugo.io/\">best static site generator</a>.<sup id=\"fnref:1\"><a href=\"#fn:1\" class=\"footnote-ref\" role=\"doc-noteref\">1</a></sup></p>\n<section class=\"footnotes\" role=\"doc-endnotes\">\n<hr>\n<ol>\n<li id=\"fn:1\" role=\"doc-endnote\">\n<p>Many people say so.</p>\n</li>\n</ol>\n</section>" {
 		t.Fatalf("Got content:\n%q", cnt)
 	}
 }
@@ -673,23 +674,13 @@ func TestPageWithShortCodeInSummary(t *testing.T) {
 	testAllMarkdownEnginesForPages(t, assertFunc, nil, simplePageWithShortcodeInSummary)
 }
 
-func TestPageWithEmbeddedScriptTag(t *testing.T) {
-	t.Parallel()
-	assertFunc := func(t *testing.T, ext string, pages page.Pages) {
-		p := pages[0]
-		if ext == "ad" || ext == "rst" {
-			// TOD(bep)
-			return
-		}
-		checkPageContent(t, p, "<script type='text/javascript'>alert('the script tags are still there, right?');</script>\n", ext)
-	}
-
-	testAllMarkdownEnginesForPages(t, assertFunc, nil, simplePageWithEmbeddedScript)
-}
-
 func TestPageWithAdditionalExtension(t *testing.T) {
 	t.Parallel()
 	cfg, fs := newTestCfg()
+	cfg.Set("markup", map[string]interface{}{
+		"defaultMarkdownHandler": "blackfriday", // TODO(bep)
+	})
+
 	c := qt.New(t)
 
 	writeSource(t, fs, filepath.Join("content", "simple.md"), simplePageWithAdditionalExtension)
@@ -716,8 +707,8 @@ func TestTableOfContents(t *testing.T) {
 
 	p := s.RegularPages()[0]
 
-	checkPageContent(t, p, "\n\n<p>For some moments the old man did not reply. He stood with bowed head, buried in deep thought. But at last he spoke.</p>\n\n<h2 id=\"aa\">AA</h2>\n\n<p>I have no idea, of course, how long it took me to reach the limit of the plain,\nbut at last I entered the foothills, following a pretty little canyon upward\ntoward the mountains. Beside me frolicked a laughing brooklet, hurrying upon\nits noisy way down to the silent sea. In its quieter pools I discovered many\nsmall fish, of four-or five-pound weight I should imagine. In appearance,\nexcept as to size and color, they were not unlike the whale of our own seas. As\nI watched them playing about I discovered, not only that they suckled their\nyoung, but that at intervals they rose to the surface to breathe as well as to\nfeed upon certain grasses and a strange, scarlet lichen which grew upon the\nrocks just above the water line.</p>\n\n<h3 id=\"aaa\">AAA</h3>\n\n<p>I remember I felt an extraordinary persuasion that I was being played with,\nthat presently, when I was upon the very verge of safety, this mysterious\ndeath&ndash;as swift as the passage of light&ndash;would leap after me from the pit about\nthe cylinder and strike me down. ## BB</p>\n\n<h3 id=\"bbb\">BBB</h3>\n\n<p>&ldquo;You&rsquo;re a great Granser,&rdquo; he cried delightedly, &ldquo;always making believe them little marks mean something.&rdquo;</p>\n")
+	checkPageContent(t, p, "<p>For some moments the old man did not reply. He stood with bowed head, buried in deep thought. But at last he spoke.</p><h2 id=\"aa\">AA</h2> <p>I have no idea, of course, how long it took me to reach the limit of the plain, but at last I entered the foothills, following a pretty little canyon upward toward the mountains. Beside me frolicked a laughing brooklet, hurrying upon its noisy way down to the silent sea. In its quieter pools I discovered many small fish, of four-or five-pound weight I should imagine. In appearance, except as to size and color, they were not unlike the whale of our own seas. As I watched them playing about I discovered, not only that they suckled their young, but that at intervals they rose to the surface to breathe as well as to feed upon certain grasses and a strange, scarlet lichen which grew upon the rocks just above the water line.</p><h3 id=\"aaa\">AAA</h3> <p>I remember I felt an extraordinary persuasion that I was being played with, that presently, when I was upon the very verge of safety, this mysterious death&ndash;as swift as the passage of light&ndash;would leap after me from the pit about the cylinder and strike me down. ## BB</p><h3 id=\"bbb\">BBB</h3> <p>&ldquo;You're a great Granser,&rdquo; he cried delightedly, &ldquo;always making believe them little marks mean something.&rdquo;</p>")
-	checkPageTOC(t, p, "<nav id=\"TableOfContents\">\n<ul>\n<li>\n<ul>\n<li><a href=\"#aa\">AA</a>\n<ul>\n<li><a href=\"#aaa\">AAA</a></li>\n<li><a href=\"#bbb\">BBB</a></li>\n</ul></li>\n</ul></li>\n</ul>\n</nav>")
+	checkPageTOC(t, p, "<nav id=\"TableOfContents\">\n  <ul>\n    <li><a href=\"#aa\">AA</a>\n      <ul>\n        <li><a href=\"#aaa\">AAA</a></li>\n        <li><a href=\"#bbb\">BBB</a></li>\n      </ul>\n    </li>\n  </ul>\n</nav>")
 }
 
 func TestPageWithMoreTag(t *testing.T) {
@@ -1518,12 +1509,12 @@ Summary: In Chinese, 好 means good.
 
 	b.AssertFileContent("public/p1/index.html", "WordCount: 510\nFuzzyWordCount: 600\nReadingTime: 3\nLen Plain: 2550\nLen PlainWords: 510\nTruncated: false\nLen Summary: 2549\nLen Content: 2557")
 
-	b.AssertFileContent("public/p2/index.html", "WordCount: 314\nFuzzyWordCount: 400\nReadingTime: 2\nLen Plain: 1569\nLen PlainWords: 314\nTruncated: true\nLen Summary: 25\nLen Content: 1583")
+	b.AssertFileContent("public/p2/index.html", "WordCount: 314\nFuzzyWordCount: 400\nReadingTime: 2\nLen Plain: 1569\nLen PlainWords: 314\nTruncated: true\nLen Summary: 25\nLen Content: 1582")
 
-	b.AssertFileContent("public/p3/index.html", "WordCount: 206\nFuzzyWordCount: 300\nReadingTime: 1\nLen Plain: 638\nLen PlainWords: 7\nTruncated: true\nLen Summary: 43\nLen Content: 652")
+	b.AssertFileContent("public/p3/index.html", "WordCount: 206\nFuzzyWordCount: 300\nReadingTime: 1\nLen Plain: 638\nLen PlainWords: 7\nTruncated: true\nLen Summary: 43\nLen Content: 651")
-	b.AssertFileContent("public/p4/index.html", "WordCount: 7\nFuzzyWordCount: 100\nReadingTime: 1\nLen Plain: 638\nLen PlainWords: 7\nTruncated: true\nLen Summary: 43\nLen Content: 652")
+	b.AssertFileContent("public/p4/index.html", "WordCount: 7\nFuzzyWordCount: 100\nReadingTime: 1\nLen Plain: 638\nLen PlainWords: 7\nTruncated: true\nLen Summary: 43\nLen Content: 651")
-	b.AssertFileContent("public/p5/index.html", "WordCount: 206\nFuzzyWordCount: 300\nReadingTime: 1\nLen Plain: 638\nLen PlainWords: 7\nTruncated: true\nLen Summary: 229\nLen Content: 653")
+	b.AssertFileContent("public/p5/index.html", "WordCount: 206\nFuzzyWordCount: 300\nReadingTime: 1\nLen Plain: 638\nLen PlainWords: 7\nTruncated: true\nLen Summary: 229\nLen Content: 652")
-	b.AssertFileContent("public/p6/index.html", "WordCount: 7\nFuzzyWordCount: 100\nReadingTime: 1\nLen Plain: 638\nLen PlainWords: 7\nTruncated: false\nLen Summary: 637\nLen Content: 653")
+	b.AssertFileContent("public/p6/index.html", "WordCount: 7\nFuzzyWordCount: 100\nReadingTime: 1\nLen Plain: 638\nLen PlainWords: 7\nTruncated: false\nLen Summary: 637\nLen Content: 652")
 
 }
 
@@ -1611,3 +1602,132 @@ author = "Jo Nesbø"
 		"Author site config:  Kurt Vonnegut")
 
 }
+
+func TestGoldmark(t *testing.T) {
+	t.Parallel()
+
+	b := newTestSitesBuilder(t).WithConfigFile("toml", `
+baseURL = "https://example.org"
+
+[markup]
+defaultMarkdownHandler="goldmark"
+[markup.goldmark]
+[markup.goldmark.renderer]
+unsafe = false
+[markup.highlight]
+noClasses=false
+
+
+`)
+	b.WithTemplatesAdded("_default/single.html", `
+Title: {{ .Title }}
+ToC: {{ .TableOfContents }}
+Content: {{ .Content }}
+
+`, "shortcodes/t.html", `T-SHORT`, "shortcodes/s.html", `## Code
+{{ .Inner }}
+`)
+
+	content := `
++++
+title = "A Page!"
++++
+
+## Shortcode {{% t %}} in header
+
+## Code Fense in Shortcode
+
+{{% s %}}
+$$$bash {hl_lines=[1]}
+SHORT
+$$$
+{{% /s %}}
+
+## Code Fence
+
+$$$bash {hl_lines=[1]}
+MARKDOWN
+$$$
+
+`
+	content = strings.ReplaceAll(content, "$$$", "```")
+
+	b.WithContent("page.md", content)
+
+	b.Build(BuildCfg{})
+
+	b.AssertFileContent("public/page/index.html",
+		`<nav id="TableOfContents">`,
+		`<li><a href="#shortcode-tshort-in-header">Shortcode T-SHORT in header</a></li>`,
+		`<code class="language-bash" data-lang="bash"><span class="hl">SHORT`,
+		`<code class="language-bash" data-lang="bash"><span class="hl">MARKDOWN`)
+}
+
+func TestBlackfridayDefault(t *testing.T) {
+	t.Parallel()
+
+	b := newTestSitesBuilder(t).WithConfigFile("toml", `
+baseURL = "https://example.org"
+
+[markup]
+defaultMarkdownHandler="blackfriday"
+[markup.highlight]
+noClasses=false
+[markup.goldmark]
+[markup.goldmark.renderer]
+unsafe=true
+
+
+`)
+	// Use the new attribute syntax to make sure it's not Goldmark.
+	b.WithTemplatesAdded("_default/single.html", `
+Title: {{ .Title }}
+Content: {{ .Content }}
+
+`, "shortcodes/s.html", `## Code
+{{ .Inner }}
+`)
+
+	content := `
++++
+title = "A Page!"
++++
+
+
+## Code Fense in Shortcode
+
+{{% s %}}
+S:
+{{% s %}}
+$$$bash {hl_lines=[1]}
+SHORT
+$$$
+{{% /s %}}
+{{% /s %}}
+
+## Code Fence
+
+$$$bash {hl_lines=[1]}
+MARKDOWN
+$$$
+
+`
+	content = strings.ReplaceAll(content, "$$$", "```")
+
+	for i, ext := range []string{"md", "html"} {
+		b.WithContent(fmt.Sprintf("page%d.%s", i+1, ext), content)
+
+	}
+
+	b.Build(BuildCfg{})
+
+	// Blackfriday does not support this extended attribute syntax.
+	b.AssertFileContent("public/page1/index.html",
+		`<pre><code class="language-bash {hl_lines=[1]}" data-lang="bash {hl_lines=[1]}">SHORT</code></pre>`,
+		`<pre><code class="language-bash {hl_lines=[1]}" data-lang="bash {hl_lines=[1]}">MARKDOWN`,
+	)
+
+	b.AssertFileContent("public/page2/index.html",
+		`<pre><code class="language-bash {hl_lines=[1]}" data-lang="bash {hl_lines=[1]}">SHORT`,
+	)
+}

hugolib/shortcode_test.go

@@ -44,6 +44,11 @@ func CheckShortCodeMatch(t *testing.T, input, expected string, withTemplate func
 func CheckShortCodeMatchAndError(t *testing.T, input, expected string, withTemplate func(templ tpl.TemplateHandler) error, expectError bool) {
 	t.Helper()
 	cfg, fs := newTestCfg()
+
+	cfg.Set("markup", map[string]interface{}{
+		"defaultMarkdownHandler": "blackfriday", // TODO(bep)
+	})
+
 	c := qt.New(t)
 
 	// Need some front matter, see https://github.com/gohugoio/hugo/issues/2337
@@ -584,6 +589,9 @@ title: "Foo"
 
 	cfg.Set("pygmentsUseClasses", true)
 	cfg.Set("pygmentsCodefences", true)
+	cfg.Set("markup", map[string]interface{}{
+		"defaultMarkdownHandler": "blackfriday", // TODO(bep)
+	})
 
 	writeSourcesToSource(t, "content", fs, sources...)
 
@@ -597,6 +605,7 @@ title: "Foo"
 			th := newTestHelper(s.Cfg, s.Fs, t)
 
 			expected := cast.ToStringSlice(test.expected)
+
 			th.assertFileContent(filepath.FromSlash(test.outFile), expected...)
 		})
 
@@ -1245,6 +1254,9 @@ func TestShortcodeRef(t *testing.T) {
 			v.Set("blackfriday", map[string]interface{}{
 				"plainIDAnchors": plainIDAnchors,
 			})
+			v.Set("markup", map[string]interface{}{
+				"defaultMarkdownHandler": "blackfriday", // TODO(bep)
+			})
 
 			builder := newTestSitesBuilder(t).WithViper(v)
 

hugolib/site_benchmark_new_test.go

@@ -181,7 +181,7 @@ pygmentsCodeFences = true
 			return sb
 		},
 			func(s *sitesBuilder) {
-				s.AssertFileContent("public/page8/index.html", `<div class="highlight"><pre style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-bash" data-lang="bash">echo <span style="color:#e6db74">&#34;Hugo Rocks!&#34;</span></code></pre></div>`)
+				s.AssertFileContent("public/page8/index.html", `<div class="highlight"><pre style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-bash" data-lang="bash">echo <span style="color:#e6db74">&#34;Hugo Rocks!&#34;</span>`)
 			},
 		},
 		{"Deep content tree", func(b testing.TB) *sitesBuilder {

hugolib/site_test.go

@@ -231,12 +231,12 @@ THE END.`, refShortcode),
 		// Issue #1753: Should not add a trailing newline after shortcode.
 		{
 			filepath.FromSlash("sect/doc3.md"),
-			fmt.Sprintf(`**Ref 1:**{{< %s "sect/doc3.md" >}}.`, refShortcode),
+			fmt.Sprintf(`**Ref 1:** {{< %s "sect/doc3.md" >}}.`, refShortcode),
 		},
 		// Issue #3703
 		{
 			filepath.FromSlash("sect/doc4.md"),
-			fmt.Sprintf(`**Ref 1:**{{< %s "%s" >}}.`, refShortcode, doc3Slashed),
+			fmt.Sprintf(`**Ref 1:** {{< %s "%s" >}}.`, refShortcode, doc3Slashed),
 		},
 	}
 
@@ -267,9 +267,9 @@ THE END.`, refShortcode),
 		expected string
 	}{
 		{filepath.FromSlash(fmt.Sprintf("public/sect/doc1%s", expectedPathSuffix)), fmt.Sprintf("<p>Ref 2: %s/sect/doc2%s</p>\n", expectedBase, expectedURLSuffix)},
-		{filepath.FromSlash(fmt.Sprintf("public/sect/doc2%s", expectedPathSuffix)), fmt.Sprintf("<p><strong>Ref 1:</strong></p>\n\n%s/sect/doc1%s\n\n<p>THE END.</p>\n", expectedBase, expectedURLSuffix)},
+		{filepath.FromSlash(fmt.Sprintf("public/sect/doc2%s", expectedPathSuffix)), fmt.Sprintf("<p><strong>Ref 1:</strong></p>\n%s/sect/doc1%s\n<p>THE END.</p>\n", expectedBase, expectedURLSuffix)},
-		{filepath.FromSlash(fmt.Sprintf("public/sect/doc3%s", expectedPathSuffix)), fmt.Sprintf("<p><strong>Ref 1:</strong>%s/sect/doc3%s.</p>\n", expectedBase, expectedURLSuffix)},
+		{filepath.FromSlash(fmt.Sprintf("public/sect/doc3%s", expectedPathSuffix)), fmt.Sprintf("<p><strong>Ref 1:</strong> %s/sect/doc3%s.</p>\n", expectedBase, expectedURLSuffix)},
-		{filepath.FromSlash(fmt.Sprintf("public/sect/doc4%s", expectedPathSuffix)), fmt.Sprintf("<p><strong>Ref 1:</strong>%s/sect/doc3%s.</p>\n", expectedBase, expectedURLSuffix)},
+		{filepath.FromSlash(fmt.Sprintf("public/sect/doc4%s", expectedPathSuffix)), fmt.Sprintf("<p><strong>Ref 1:</strong> %s/sect/doc3%s.</p>\n", expectedBase, expectedURLSuffix)},
 	}
 
 	for _, test := range tests {
@@ -330,12 +330,12 @@ func doTestShouldAlwaysHaveUglyURLs(t *testing.T, uglyURLs bool) {
 		expected string
 	}{
 		{filepath.FromSlash("public/index.html"), "Home Sweet Home."},
-		{filepath.FromSlash(expectedPagePath), "\n\n<h1 id=\"title\">title</h1>\n\n<p>some <em>content</em></p>\n"},
+		{filepath.FromSlash(expectedPagePath), "<h1 id=\"title\">title</h1>\n<p>some <em>content</em></p>\n"},
 		{filepath.FromSlash("public/404.html"), "Page Not Found."},
 		{filepath.FromSlash("public/index.xml"), "<root>RSS</root>"},
 		{filepath.FromSlash("public/sitemap.xml"), "<root>SITEMAP</root>"},
 		// Issue #1923
-		{filepath.FromSlash("public/ugly.html"), "\n\n<h1 id=\"title\">title</h1>\n\n<p>doc2 <em>content</em></p>\n"},
+		{filepath.FromSlash("public/ugly.html"), "<h1 id=\"title\">title</h1>\n<p>doc2 <em>content</em></p>\n"},
 	}
 
 	for _, p := range s.RegularPages() {
@@ -540,14 +540,14 @@ func TestSkipRender(t *testing.T) {
 		doc      string
 		expected string
 	}{
-		{filepath.FromSlash("public/sect/doc1.html"), "\n\n<h1 id=\"title\">title</h1>\n\n<p>some <em>content</em></p>\n"},
+		{filepath.FromSlash("public/sect/doc1.html"), "<h1 id=\"title\">title</h1>\n<p>some <em>content</em></p>\n"},
 		{filepath.FromSlash("public/sect/doc2.html"), "<!doctype html><html><body>more content</body></html>"},
-		{filepath.FromSlash("public/sect/doc3.html"), "\n\n<h1 id=\"doc3\">doc3</h1>\n\n<p><em>some</em> content</p>\n"},
+		{filepath.FromSlash("public/sect/doc3.html"), "<h1 id=\"doc3\">doc3</h1>\n<p><em>some</em> content</p>\n"},
-		{filepath.FromSlash("public/sect/doc4.html"), "\n\n<h1 id=\"doc4\">doc4</h1>\n\n<p><em>some content</em></p>\n"},
+		{filepath.FromSlash("public/sect/doc4.html"), "<h1 id=\"doc4\">doc4</h1>\n<p><em>some content</em></p>\n"},
 		{filepath.FromSlash("public/sect/doc5.html"), "<!doctype html><html><head><script src=\"script.js\"></script></head><body>body5</body></html>"},
 		{filepath.FromSlash("public/sect/doc6.html"), "<!doctype html><html><head><script src=\"http://auth/bub/script.js\"></script></head><body>body5</body></html>"},
 		{filepath.FromSlash("public/doc7.html"), "<html><body>doc7 content</body></html>"},
-		{filepath.FromSlash("public/sect/doc8.html"), "\n\n<h1 id=\"title\">title</h1>\n\n<p>some <em>content</em></p>\n"},
+		{filepath.FromSlash("public/sect/doc8.html"), "<h1 id=\"title\">title</h1>\n<p>some <em>content</em></p>\n"},
 		{filepath.FromSlash("public/doc9.html"), "<html><body>doc9: SHORT</body></html>"},
 	}
 

hugolib/testhelpers_test.go

@@ -743,7 +743,7 @@ func (th testHelper) assertFileContent(filename string, matches ...string) {
 	content := readDestination(th, th.Fs, filename)
 	for _, match := range matches {
 		match = th.replaceDefaultContentLanguageValue(match)
-		th.Assert(strings.Contains(content, match), qt.Equals, true, qt.Commentf(content))
+		th.Assert(strings.Contains(content, match), qt.Equals, true, qt.Commentf(match+" not in: \n"+content))
 	}
 }
 
@@ -755,7 +755,7 @@ func (th testHelper) assertFileContentRegexp(filename string, matches ...string)
 		r := regexp.MustCompile(match)
 		matches := r.MatchString(content)
 		if !matches {
-			fmt.Println(content)
+			fmt.Println(match+":\n", content)
 		}
 		th.Assert(matches, qt.Equals, true)
 	}

markup/asciidoc/convert.go

@@ -24,19 +24,18 @@ import (
 )
 
 // Provider is the package entry point.
-var Provider converter.NewProvider = provider{}
+var Provider converter.ProviderProvider = provider{}
 
 type provider struct {
 }
 
 func (p provider) New(cfg converter.ProviderConfig) (converter.Provider, error) {
-	var n converter.NewConverter = func(ctx converter.DocumentContext) (converter.Converter, error) {
+	return converter.NewProvider("asciidoc", func(ctx converter.DocumentContext) (converter.Converter, error) {
 		return &asciidocConverter{
 			ctx: ctx,
 			cfg: cfg,
 		}, nil
-	}
+	}), nil
-	return n, nil
 }
 
 type asciidocConverter struct {

markup/blackfriday/blackfriday_config/config.go

@@ -0,0 +1,70 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package helpers implements general utility functions that work with
+// and on content.  The helper functions defined here lay down the
+// foundation of how Hugo works with files and filepaths, and perform
+// string operations on content.
+
+package blackfriday_config
+
+import (
+	"github.com/mitchellh/mapstructure"
+	"github.com/pkg/errors"
+)
+
+// Default holds the default BlackFriday config.
+// Do not change!
+var Default = Config{
+	Smartypants:           true,
+	AngledQuotes:          false,
+	SmartypantsQuotesNBSP: false,
+	Fractions:             true,
+	HrefTargetBlank:       false,
+	NofollowLinks:         false,
+	NoreferrerLinks:       false,
+	SmartDashes:           true,
+	LatexDashes:           true,
+	PlainIDAnchors:        true,
+	TaskLists:             true,
+	SkipHTML:              false,
+}
+
+// Config holds configuration values for BlackFriday rendering.
+// It is kept here because it's used in several packages.
+type Config struct {
+	Smartypants           bool
+	SmartypantsQuotesNBSP bool
+	AngledQuotes          bool
+	Fractions             bool
+	HrefTargetBlank       bool
+	NofollowLinks         bool
+	NoreferrerLinks       bool
+	SmartDashes           bool
+	LatexDashes           bool
+	TaskLists             bool
+	PlainIDAnchors        bool
+	Extensions            []string
+	ExtensionsMask        []string
+	SkipHTML              bool
+
+	FootnoteAnchorPrefix       string
+	FootnoteReturnLinkContents string
+}
+
+func UpdateConfig(b Config, m map[string]interface{}) (Config, error) {
+	if err := mapstructure.Decode(m, &b); err != nil {
+		return b, errors.WithMessage(err, "failed to decode rendering config")
+	}
+	return b, nil
+}

markup/blackfriday/convert.go

@@ -15,36 +15,27 @@
 package blackfriday
 
 import (
+	"github.com/gohugoio/hugo/markup/blackfriday/blackfriday_config"
 	"github.com/gohugoio/hugo/markup/converter"
-	"github.com/gohugoio/hugo/markup/internal"
 	"github.com/russross/blackfriday"
 )
 
 // Provider is the package entry point.
-var Provider converter.NewProvider = provider{}
+var Provider converter.ProviderProvider = provider{}
 
 type provider struct {
 }
 
 func (p provider) New(cfg converter.ProviderConfig) (converter.Provider, error) {
-	defaultBlackFriday, err := internal.NewBlackfriday(cfg)
+	defaultExtensions := getMarkdownExtensions(cfg.MarkupConfig.BlackFriday)
-	if err != nil {
-		return nil, err
-	}
 
-	defaultExtensions := getMarkdownExtensions(defaultBlackFriday)
+	return converter.NewProvider("blackfriday", func(ctx converter.DocumentContext) (converter.Converter, error) {
-
+		b := cfg.MarkupConfig.BlackFriday
-	pygmentsCodeFences := cfg.Cfg.GetBool("pygmentsCodeFences")
-	pygmentsCodeFencesGuessSyntax := cfg.Cfg.GetBool("pygmentsCodeFencesGuessSyntax")
-	pygmentsOptions := cfg.Cfg.GetString("pygmentsOptions")
-
-	var n converter.NewConverter = func(ctx converter.DocumentContext) (converter.Converter, error) {
-		b := defaultBlackFriday
 		extensions := defaultExtensions
 
 		if ctx.ConfigOverrides != nil {
 			var err error
-			b, err = internal.UpdateBlackFriday(b, ctx.ConfigOverrides)
+			b, err = blackfriday_config.UpdateConfig(b, ctx.ConfigOverrides)
 			if err != nil {
 				return nil, err
 			}
@@ -56,27 +47,16 @@ func (p provider) New(cfg converter.ProviderConfig) (converter.Provider, error)
 			bf:         b,
 			extensions: extensions,
 			cfg:        cfg,
-
-			pygmentsCodeFences:            pygmentsCodeFences,
-			pygmentsCodeFencesGuessSyntax: pygmentsCodeFencesGuessSyntax,
-			pygmentsOptions:               pygmentsOptions,
 		}, nil
-	}
+	}), nil
-
-	return n, nil
 
 }
 
 type blackfridayConverter struct {
 	ctx        converter.DocumentContext
-	bf         *internal.BlackFriday
+	bf         blackfriday_config.Config
 	extensions int
-
+	cfg        converter.ProviderConfig
-	pygmentsCodeFences            bool
-	pygmentsCodeFencesGuessSyntax bool
-	pygmentsOptions               string
-
-	cfg converter.ProviderConfig
 }
 
 func (c *blackfridayConverter) AnchorSuffix() string {
@@ -90,7 +70,6 @@ func (c *blackfridayConverter) Convert(ctx converter.RenderContext) (converter.R
 	r := c.getHTMLRenderer(ctx.RenderTOC)
 
 	return converter.Bytes(blackfriday.Markdown(ctx.Src, r, c.extensions)), nil
-
 }
 
 func (c *blackfridayConverter) getHTMLRenderer(renderTOC bool) blackfriday.Renderer {
@@ -114,7 +93,7 @@ func (c *blackfridayConverter) getHTMLRenderer(renderTOC bool) blackfriday.Rende
 	}
 }
 
-func getFlags(renderTOC bool, cfg *internal.BlackFriday) int {
+func getFlags(renderTOC bool, cfg blackfriday_config.Config) int {
 
 	var flags int
 
@@ -168,7 +147,7 @@ func getFlags(renderTOC bool, cfg *internal.BlackFriday) int {
 	return flags
 }
 
-func getMarkdownExtensions(cfg *internal.BlackFriday) int {
+func getMarkdownExtensions(cfg blackfriday_config.Config) int {
 	// Default Blackfriday common extensions
 	commonExtensions := 0 |
 		blackfriday.EXTENSION_NO_INTRA_EMPHASIS |

markup/blackfriday/convert_test.go

@@ -18,19 +18,15 @@ import (
 
 	"github.com/spf13/viper"
 
-	"github.com/gohugoio/hugo/markup/internal"
-
 	"github.com/gohugoio/hugo/markup/converter"
 
 	qt "github.com/frankban/quicktest"
+	"github.com/gohugoio/hugo/markup/blackfriday/blackfriday_config"
 	"github.com/russross/blackfriday"
 )
 
 func TestGetMarkdownExtensionsMasksAreRemovedFromExtensions(t *testing.T) {
-	c := qt.New(t)
+	b := blackfriday_config.Default
-	b, err := internal.NewBlackfriday(converter.ProviderConfig{Cfg: viper.New()})
-	c.Assert(err, qt.IsNil)
-
 	b.Extensions = []string{"headerId"}
 	b.ExtensionsMask = []string{"noIntraEmphasis"}
 
@@ -45,9 +41,7 @@ func TestGetMarkdownExtensionsByDefaultAllExtensionsAreEnabled(t *testing.T) {
 		testFlag int
 	}
 
-	c := qt.New(t)
+	b := blackfriday_config.Default
-	b, err := internal.NewBlackfriday(converter.ProviderConfig{Cfg: viper.New()})
-	c.Assert(err, qt.IsNil)
 
 	b.Extensions = []string{""}
 	b.ExtensionsMask = []string{""}
@@ -79,9 +73,7 @@ func TestGetMarkdownExtensionsByDefaultAllExtensionsAreEnabled(t *testing.T) {
 }
 
 func TestGetMarkdownExtensionsAddingFlagsThroughRenderingContext(t *testing.T) {
-	c := qt.New(t)
+	b := blackfriday_config.Default
-	b, err := internal.NewBlackfriday(converter.ProviderConfig{Cfg: viper.New()})
-	c.Assert(err, qt.IsNil)
 
 	b.Extensions = []string{"definitionLists"}
 	b.ExtensionsMask = []string{""}
@@ -93,10 +85,7 @@ func TestGetMarkdownExtensionsAddingFlagsThroughRenderingContext(t *testing.T) {
 }
 
 func TestGetFlags(t *testing.T) {
-	c := qt.New(t)
+	b := blackfriday_config.Default
-	cfg := converter.ProviderConfig{Cfg: viper.New()}
-	b, err := internal.NewBlackfriday(cfg)
-	c.Assert(err, qt.IsNil)
 	flags := getFlags(false, b)
 	if flags&blackfriday.HTML_USE_XHTML != blackfriday.HTML_USE_XHTML {
 		t.Errorf("Test flag: %d was not found amongs set flags:%d; Result: %d", blackfriday.HTML_USE_XHTML, flags, flags&blackfriday.HTML_USE_XHTML)
@@ -105,9 +94,8 @@ func TestGetFlags(t *testing.T) {
 
 func TestGetAllFlags(t *testing.T) {
 	c := qt.New(t)
-	cfg := converter.ProviderConfig{Cfg: viper.New()}
+
-	b, err := internal.NewBlackfriday(cfg)
+	b := blackfriday_config.Default
-	c.Assert(err, qt.IsNil)
 
 	type data struct {
 		testFlag int
@@ -145,9 +133,8 @@ func TestGetAllFlags(t *testing.T) {
 	for _, d := range allFlags {
 		expectedFlags |= d.testFlag
 	}
-	if expectedFlags != actualFlags {
+
-		t.Errorf("Expected flags (%d) did not equal actual (%d) flags.", expectedFlags, actualFlags)
+	c.Assert(actualFlags, qt.Equals, expectedFlags)
-	}
 }
 
 func TestConvert(t *testing.T) {

markup/blackfriday/renderer.go

@@ -30,10 +30,9 @@ type hugoHTMLRenderer struct {
 // BlockCode renders a given text as a block of code.
 // Pygments is used if it is setup to handle code fences.
 func (r *hugoHTMLRenderer) BlockCode(out *bytes.Buffer, text []byte, lang string) {
-	if r.c.pygmentsCodeFences && (lang != "" || r.c.pygmentsCodeFencesGuessSyntax) {
+	if r.c.cfg.MarkupConfig.Highlight.CodeFences {
-		opts := r.c.pygmentsOptions
 		str := strings.Trim(string(text), "\n\r")
-		highlighted, _ := r.c.cfg.Highlight(str, lang, opts)
+		highlighted, _ := r.c.cfg.Highlight(str, lang, "")
 		out.WriteString(highlighted)
 	} else {
 		r.Renderer.BlockCode(out, text, lang)

markup/converter/converter.go

@@ -16,33 +16,51 @@ package converter
 import (
 	"github.com/gohugoio/hugo/common/loggers"
 	"github.com/gohugoio/hugo/config"
+	"github.com/gohugoio/hugo/markup/markup_config"
+	"github.com/gohugoio/hugo/markup/tableofcontents"
 	"github.com/spf13/afero"
 )
 
 // ProviderConfig configures a new Provider.
 type ProviderConfig struct {
+	MarkupConfig markup_config.Config
+
 	Cfg       config.Provider // Site config
 	ContentFs afero.Fs
 	Logger    *loggers.Logger
 	Highlight func(code, lang, optsStr string) (string, error)
 }
 
-// NewProvider creates converter providers.
+// ProviderProvider creates converter providers.
-type NewProvider interface {
+type ProviderProvider interface {
 	New(cfg ProviderConfig) (Provider, error)
 }
 
 // Provider creates converters.
 type Provider interface {
 	New(ctx DocumentContext) (Converter, error)
+	Name() string
 }
 
-// NewConverter is an adapter that can be used as a ConverterProvider.
+// NewProvider creates a new Provider with the given name.
-type NewConverter func(ctx DocumentContext) (Converter, error)
+func NewProvider(name string, create func(ctx DocumentContext) (Converter, error)) Provider {
+	return newConverter{
+		name:   name,
+		create: create,
+	}
+}
 
-// New creates a new Converter for the given ctx.
+type newConverter struct {
-func (n NewConverter) New(ctx DocumentContext) (Converter, error) {
+	name   string
-	return n(ctx)
+	create func(ctx DocumentContext) (Converter, error)
+}
+
+func (n newConverter) New(ctx DocumentContext) (Converter, error) {
+	return n.create(ctx)
+}
+
+func (n newConverter) Name() string {
+	return n.name
 }
 
 // Converter wraps the Convert method that converts some markup into
@@ -61,6 +79,11 @@ type DocumentInfo interface {
 	AnchorSuffix() string
 }
 
+// TableOfContentsProvider provides the content as a ToC structure.
+type TableOfContentsProvider interface {
+	TableOfContents() tableofcontents.Root
+}
+
 // Bytes holds a byte slice and implements the Result interface.
 type Bytes []byte
 

markup/goldmark/convert.go

@@ -0,0 +1,233 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package goldmark converts Markdown to HTML using Goldmark.
+package goldmark
+
+import (
+	"bytes"
+	"fmt"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+
+	"github.com/spf13/afero"
+
+	"github.com/gohugoio/hugo/hugofs"
+
+	"github.com/alecthomas/chroma/styles"
+	"github.com/gohugoio/hugo/markup/converter"
+	"github.com/gohugoio/hugo/markup/highlight"
+	hl "github.com/gohugoio/hugo/markup/highlight/temphighlighting"
+	"github.com/gohugoio/hugo/markup/markup_config"
+	"github.com/gohugoio/hugo/markup/tableofcontents"
+	"github.com/yuin/goldmark"
+	"github.com/yuin/goldmark/extension"
+	"github.com/yuin/goldmark/parser"
+	"github.com/yuin/goldmark/renderer"
+	"github.com/yuin/goldmark/renderer/html"
+	"github.com/yuin/goldmark/text"
+	"github.com/yuin/goldmark/util"
+)
+
+// Provider is the package entry point.
+var Provider converter.ProviderProvider = provide{}
+
+type provide struct {
+}
+
+func (p provide) New(cfg converter.ProviderConfig) (converter.Provider, error) {
+	md := newMarkdown(cfg.MarkupConfig)
+	return converter.NewProvider("goldmark", func(ctx converter.DocumentContext) (converter.Converter, error) {
+		return &goldmarkConverter{
+			ctx: ctx,
+			cfg: cfg,
+			md:  md,
+		}, nil
+	}), nil
+}
+
+type goldmarkConverter struct {
+	md  goldmark.Markdown
+	ctx converter.DocumentContext
+	cfg converter.ProviderConfig
+}
+
+func newMarkdown(mcfg markup_config.Config) goldmark.Markdown {
+	cfg := mcfg.Goldmark
+
+	var (
+		extensions = []goldmark.Extender{
+			newTocExtension(),
+		}
+		rendererOptions []renderer.Option
+		parserOptions   []parser.Option
+	)
+
+	if cfg.Renderer.HardWraps {
+		rendererOptions = append(rendererOptions, html.WithHardWraps())
+	}
+
+	if cfg.Renderer.XHTML {
+		rendererOptions = append(rendererOptions, html.WithXHTML())
+	}
+
+	if cfg.Renderer.Unsafe {
+		rendererOptions = append(rendererOptions, html.WithUnsafe())
+	}
+
+	if mcfg.Highlight.CodeFences {
+		extensions = append(extensions, newHighlighting(mcfg.Highlight))
+	}
+
+	if cfg.Extensions.Table {
+		extensions = append(extensions, extension.Table)
+	}
+
+	if cfg.Extensions.Strikethrough {
+		extensions = append(extensions, extension.Strikethrough)
+	}
+
+	if cfg.Extensions.Linkify {
+		extensions = append(extensions, extension.Linkify)
+	}
+
+	if cfg.Extensions.TaskList {
+		extensions = append(extensions, extension.TaskList)
+	}
+
+	if cfg.Extensions.Typographer {
+		extensions = append(extensions, extension.Typographer)
+	}
+
+	if cfg.Extensions.DefinitionList {
+		extensions = append(extensions, extension.DefinitionList)
+	}
+
+	if cfg.Extensions.Footnote {
+		extensions = append(extensions, extension.Footnote)
+	}
+
+	if cfg.Parser.AutoHeadingID {
+		parserOptions = append(parserOptions, parser.WithAutoHeadingID())
+	}
+
+	if cfg.Parser.Attribute {
+		parserOptions = append(parserOptions, parser.WithAttribute())
+	}
+
+	md := goldmark.New(
+		goldmark.WithExtensions(
+			extensions...,
+		),
+		goldmark.WithParserOptions(
+			parserOptions...,
+		),
+		goldmark.WithRendererOptions(
+			rendererOptions...,
+		),
+	)
+
+	return md
+
+}
+
+type converterResult struct {
+	converter.Result
+	toc tableofcontents.Root
+}
+
+func (c converterResult) TableOfContents() tableofcontents.Root {
+	return c.toc
+}
+
+func (c *goldmarkConverter) Convert(ctx converter.RenderContext) (result converter.Result, err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			dir := afero.GetTempDir(hugofs.Os, "hugo_bugs")
+			name := fmt.Sprintf("goldmark_%s.txt", c.ctx.DocumentID)
+			filename := filepath.Join(dir, name)
+			afero.WriteFile(hugofs.Os, filename, ctx.Src, 07555)
+			err = errors.Errorf("[BUG] goldmark: create an issue on GitHub attaching the file in: %s", filename)
+
+		}
+	}()
+
+	buf := &bytes.Buffer{}
+	result = buf
+	pctx := parser.NewContext()
+	pctx.Set(tocEnableKey, ctx.RenderTOC)
+
+	reader := text.NewReader(ctx.Src)
+
+	doc := c.md.Parser().Parse(
+		reader,
+		parser.WithContext(pctx),
+	)
+
+	if err := c.md.Renderer().Render(buf, ctx.Src, doc); err != nil {
+		return nil, err
+	}
+
+	if toc, ok := pctx.Get(tocResultKey).(tableofcontents.Root); ok {
+		return converterResult{
+			Result: buf,
+			toc:    toc,
+		}, nil
+	}
+
+	return buf, nil
+}
+
+func newHighlighting(cfg highlight.Config) goldmark.Extender {
+	style := styles.Get(cfg.Style)
+	if style == nil {
+		style = styles.Fallback
+	}
+
+	e := hl.NewHighlighting(
+		hl.WithStyle(cfg.Style),
+		hl.WithCodeBlockOptions(highlight.GetCodeBlockOptions()),
+		hl.WithFormatOptions(
+			cfg.ToHTMLOptions()...,
+		),
+
+		hl.WithWrapperRenderer(func(w util.BufWriter, ctx hl.CodeBlockContext, entering bool) {
+			l, hasLang := ctx.Language()
+			var language string
+			if hasLang {
+				language = string(l)
+			}
+
+			if entering {
+				if !ctx.Highlighted() {
+					w.WriteString(`<pre>`)
+					highlight.WriteCodeTag(w, language)
+					return
+				}
+				w.WriteString(`<div class="highlight">`)
+				return
+			}
+
+			if !ctx.Highlighted() {
+				w.WriteString(`</code></pre>`)
+				return
+			}
+
+			w.WriteString("</div>")
+
+		}),
+	)
+
+	return e
+}

markup/goldmark/convert_test.go

@@ -0,0 +1,219 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package goldmark
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/gohugoio/hugo/markup/highlight"
+
+	"github.com/gohugoio/hugo/markup/markup_config"
+
+	"github.com/gohugoio/hugo/common/loggers"
+
+	"github.com/gohugoio/hugo/markup/converter"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestConvert(t *testing.T) {
+	c := qt.New(t)
+
+	// Smoke test of the default configuration.
+	content := `
+## Code Fences
+
+§§§bash
+LINE1
+§§§
+
+## Code Fences No Lexer
+
+§§§moo
+LINE1
+§§§
+
+## Custom ID {#custom}
+
+## Auto ID
+
+* Autolink: https://gohugo.io/
+* Strikethrough:~~Hi~~ Hello, world!
+ 
+## Table
+
+| foo | bar |
+| --- | --- |
+| baz | bim |
+
+## Task Lists (default on)
+
+- [x] Finish my changes[^1]
+- [ ] Push my commits to GitHub
+- [ ] Open a pull request
+
+
+## Smartypants (default on)
+
+* Straight double "quotes" and single 'quotes' into “curly” quote HTML entities
+* Dashes (“--” and “---”) into en- and em-dash entities
+* Three consecutive dots (“...”) into an ellipsis entity
+
+## Footnotes
+
+That's some text with a footnote.[^1]
+
+## Definition Lists
+
+date
+: the datetime assigned to this page. 
+
+description
+: the description for the content.
+
+
+[^1]: And that's the footnote.
+
+`
+
+	// Code fences
+	content = strings.Replace(content, "§§§", "```", -1)
+
+	mconf := markup_config.Default
+	mconf.Highlight.NoClasses = false
+
+	p, err := Provider.New(
+		converter.ProviderConfig{
+			MarkupConfig: mconf,
+			Logger:       loggers.NewErrorLogger(),
+		},
+	)
+	c.Assert(err, qt.IsNil)
+	conv, err := p.New(converter.DocumentContext{})
+	c.Assert(err, qt.IsNil)
+	b, err := conv.Convert(converter.RenderContext{Src: []byte(content)})
+	c.Assert(err, qt.IsNil)
+
+	got := string(b.Bytes())
+
+	// Header IDs
+	c.Assert(got, qt.Contains, `<h2 id="custom">Custom ID</h2>`, qt.Commentf(got))
+	c.Assert(got, qt.Contains, `<h2 id="auto-id">Auto ID</h2>`, qt.Commentf(got))
+
+	// Code fences
+	c.Assert(got, qt.Contains, "<div class=\"highlight\"><pre class=\"chroma\"><code class=\"language-bash\" data-lang=\"bash\">LINE1\n</code></pre></div>")
+	c.Assert(got, qt.Contains, "Code Fences No Lexer</h2>\n<pre><code class=\"language-moo\" data-lang=\"moo\">LINE1\n</code></pre>")
+
+	// Extensions
+	c.Assert(got, qt.Contains, `Autolink: <a href="https://gohugo.io/">https://gohugo.io/</a>`)
+	c.Assert(got, qt.Contains, `Strikethrough:<del>Hi</del> Hello, world`)
+	c.Assert(got, qt.Contains, `<th>foo</th>`)
+	c.Assert(got, qt.Contains, `<li><input disabled="" type="checkbox">Push my commits to GitHub</li>`)
+
+	c.Assert(got, qt.Contains, `Straight double &ldquo;quotes&rdquo; and single &lsquo;quotes&rsquo;`)
+	c.Assert(got, qt.Contains, `Dashes (“&ndash;” and “&mdash;”) `)
+	c.Assert(got, qt.Contains, `Three consecutive dots (“&hellip;”)`)
+	c.Assert(got, qt.Contains, `footnote.<sup id="fnref:1"><a href="#fn:1" class="footnote-ref" role="doc-noteref">1</a></sup>`)
+	c.Assert(got, qt.Contains, `<section class="footnotes" role="doc-endnotes">`)
+	c.Assert(got, qt.Contains, `<dt>date</dt>`)
+
+}
+
+func TestCodeFence(t *testing.T) {
+	c := qt.New(t)
+
+	lines := `LINE1
+LINE2
+LINE3
+LINE4
+LINE5
+`
+
+	convertForConfig := func(c *qt.C, conf highlight.Config, code, language string) string {
+		mconf := markup_config.Default
+		mconf.Highlight = conf
+
+		p, err := Provider.New(
+			converter.ProviderConfig{
+				MarkupConfig: mconf,
+				Logger:       loggers.NewErrorLogger(),
+			},
+		)
+
+		content := "```" + language + "\n" + code + "\n```"
+
+		c.Assert(err, qt.IsNil)
+		conv, err := p.New(converter.DocumentContext{})
+		c.Assert(err, qt.IsNil)
+		b, err := conv.Convert(converter.RenderContext{Src: []byte(content)})
+		c.Assert(err, qt.IsNil)
+
+		return string(b.Bytes())
+	}
+
+	c.Run("Basic", func(c *qt.C) {
+		cfg := highlight.DefaultConfig
+		cfg.NoClasses = false
+
+		result := convertForConfig(c, cfg, `echo "Hugo Rocks!"`, "bash")
+		// TODO(bep) there is a whitespace mismatch (\n) between this and the highlight template func.
+		c.Assert(result, qt.Equals, `<div class="highlight"><pre class="chroma"><code class="language-bash" data-lang="bash"><span class="nb">echo</span> <span class="s2">&#34;Hugo Rocks!&#34;</span>
+</code></pre></div>`)
+		result = convertForConfig(c, cfg, `echo "Hugo Rocks!"`, "unknown")
+		c.Assert(result, qt.Equals, "<pre><code class=\"language-unknown\" data-lang=\"unknown\">echo &quot;Hugo Rocks!&quot;\n</code></pre>")
+
+	})
+
+	c.Run("Highlight lines, default config", func(c *qt.C) {
+		cfg := highlight.DefaultConfig
+		cfg.NoClasses = false
+
+		result := convertForConfig(c, cfg, lines, `bash {linenos=table,hl_lines=[2 "4-5"],linenostart=3}`)
+		c.Assert(result, qt.Contains, "<div class=\"highlight\"><div class=\"chroma\">\n<table class=\"lntable\"><tr><td class=\"lntd\">\n<pre class=\"chroma\"><code><span class")
+		c.Assert(result, qt.Contains, "<span class=\"hl\"><span class=\"lnt\">4")
+
+		result = convertForConfig(c, cfg, lines, "bash {linenos=inline,hl_lines=[2]}")
+		c.Assert(result, qt.Contains, "<span class=\"ln\">2</span>LINE2\n</span>")
+		c.Assert(result, qt.Not(qt.Contains), "<table")
+
+		result = convertForConfig(c, cfg, lines, "bash {linenos=true,hl_lines=[2]}")
+		c.Assert(result, qt.Contains, "<table")
+		c.Assert(result, qt.Contains, "<span class=\"hl\"><span class=\"lnt\">2\n</span>")
+	})
+
+	c.Run("Highlight lines, linenumbers default on", func(c *qt.C) {
+		cfg := highlight.DefaultConfig
+		cfg.NoClasses = false
+		cfg.LineNos = true
+
+		result := convertForConfig(c, cfg, lines, "bash")
+		c.Assert(result, qt.Contains, "<span class=\"lnt\">2\n</span>")
+
+		result = convertForConfig(c, cfg, lines, "bash {linenos=false,hl_lines=[2]}")
+		c.Assert(result, qt.Not(qt.Contains), "class=\"lnt\"")
+	})
+
+	c.Run("Highlight lines, linenumbers default on, linenumbers in table default off", func(c *qt.C) {
+		cfg := highlight.DefaultConfig
+		cfg.NoClasses = false
+		cfg.LineNos = true
+		cfg.LineNumbersInTable = false
+
+		result := convertForConfig(c, cfg, lines, "bash")
+		c.Assert(result, qt.Contains, "<span class=\"ln\">2</span>LINE2\n<")
+		result = convertForConfig(c, cfg, lines, "bash {linenos=table}")
+		c.Assert(result, qt.Contains, "<span class=\"lnt\">1\n</span>")
+	})
+}

markup/goldmark/goldmark_config/config.go

@@ -0,0 +1,74 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package goldmark_config holds Goldmark related configuration.
+package goldmark_config
+
+// DefaultConfig holds the default Goldmark configuration.
+var Default = Config{
+	Extensions: Extensions{
+		Typographer:    true,
+		Footnote:       true,
+		DefinitionList: true,
+		Table:          true,
+		Strikethrough:  true,
+		Linkify:        true,
+		TaskList:       true,
+	},
+	Renderer: Renderer{
+		Unsafe: false,
+	},
+	Parser: Parser{
+		AutoHeadingID: true,
+		Attribute:     true,
+	},
+}
+
+// Config configures Goldmark.
+type Config struct {
+	Renderer   Renderer
+	Parser     Parser
+	Extensions Extensions
+}
+
+type Extensions struct {
+	Typographer    bool
+	Footnote       bool
+	DefinitionList bool
+
+	// GitHub flavored markdown
+	Table         bool
+	Strikethrough bool
+	Linkify       bool
+	TaskList      bool
+}
+
+type Renderer struct {
+	// Whether softline breaks should be rendered as '<br>'
+	HardWraps bool
+
+	// XHTML instead of HTML5.
+	XHTML bool
+
+	// Allow raw HTML etc.
+	Unsafe bool
+}
+
+type Parser struct {
+	// Enables custom heading ids and
+	// auto generated heading ids.
+	AutoHeadingID bool
+
+	// Enables custom attributes.
+	Attribute bool
+}

markup/goldmark/toc.go

@@ -0,0 +1,102 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package goldmark
+
+import (
+	"bytes"
+
+	"github.com/gohugoio/hugo/markup/tableofcontents"
+
+	"github.com/yuin/goldmark"
+	"github.com/yuin/goldmark/ast"
+	"github.com/yuin/goldmark/parser"
+	"github.com/yuin/goldmark/text"
+	"github.com/yuin/goldmark/util"
+)
+
+var (
+	tocResultKey = parser.NewContextKey()
+	tocEnableKey = parser.NewContextKey()
+)
+
+type tocTransformer struct {
+}
+
+func (t *tocTransformer) Transform(n *ast.Document, reader text.Reader, pc parser.Context) {
+	if b, ok := pc.Get(tocEnableKey).(bool); !ok || !b {
+		return
+	}
+
+	var (
+		toc         tableofcontents.Root
+		header      tableofcontents.Header
+		level       int
+		row         = -1
+		inHeading   bool
+		headingText bytes.Buffer
+	)
+
+	ast.Walk(n, func(n ast.Node, entering bool) (ast.WalkStatus, error) {
+		s := ast.WalkStatus(ast.WalkContinue)
+		if n.Kind() == ast.KindHeading {
+			if inHeading && !entering {
+				header.Text = headingText.String()
+				headingText.Reset()
+				toc.AddAt(header, row, level-1)
+				header = tableofcontents.Header{}
+				inHeading = false
+				return s, nil
+			}
+
+			inHeading = true
+		}
+
+		if !(inHeading && entering) {
+			return s, nil
+		}
+
+		switch n.Kind() {
+		case ast.KindHeading:
+			heading := n.(*ast.Heading)
+			level = heading.Level
+
+			if level == 1 || row == -1 {
+				row++
+			}
+
+			id, found := heading.AttributeString("id")
+			if found {
+				header.ID = string(id.([]byte))
+			}
+		case ast.KindText:
+			textNode := n.(*ast.Text)
+			headingText.Write(textNode.Text(reader.Source()))
+		}
+
+		return s, nil
+	})
+
+	pc.Set(tocResultKey, toc)
+}
+
+type tocExtension struct {
+}
+
+func newTocExtension() goldmark.Extender {
+	return &tocExtension{}
+}
+
+func (e *tocExtension) Extend(m goldmark.Markdown) {
+	m.Parser().AddOptions(parser.WithASTTransformers(util.Prioritized(&tocTransformer{}, 10)))
+}

markup/goldmark/toc_test.go

@@ -0,0 +1,76 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package goldmark converts Markdown to HTML using Goldmark.
+package goldmark
+
+import (
+	"testing"
+
+	"github.com/gohugoio/hugo/markup/markup_config"
+
+	"github.com/gohugoio/hugo/common/loggers"
+
+	"github.com/gohugoio/hugo/markup/converter"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestToc(t *testing.T) {
+	c := qt.New(t)
+
+	content := `
+# Header 1
+
+## First h2
+
+Some text.
+
+### H3
+
+Some more text.
+
+## Second h2
+
+And then some.
+
+### Second H3
+
+#### First H4
+
+`
+	p, err := Provider.New(
+		converter.ProviderConfig{
+			MarkupConfig: markup_config.Default,
+			Logger:       loggers.NewErrorLogger()})
+	c.Assert(err, qt.IsNil)
+	conv, err := p.New(converter.DocumentContext{})
+	c.Assert(err, qt.IsNil)
+	b, err := conv.Convert(converter.RenderContext{Src: []byte(content), RenderTOC: true})
+	c.Assert(err, qt.IsNil)
+	got := b.(converter.TableOfContentsProvider).TableOfContents().ToHTML(2, 3)
+	c.Assert(got, qt.Equals, `<nav id="TableOfContents">
+  <ul>
+    <li><a href="#first-h2">First h2</a>
+      <ul>
+        <li><a href="#h3">H3</a></li>
+      </ul>
+    </li>
+    <li><a href="#second-h2">Second h2</a>
+      <ul>
+        <li><a href="#second-h3">Second H3</a></li>
+      </ul>
+    </li>
+  </ul>
+</nav>`, qt.Commentf(got))
+}

markup/highlight/config.go

@@ -0,0 +1,188 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package highlight provides code highlighting.
+package highlight
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/alecthomas/chroma/formatters/html"
+
+	"github.com/gohugoio/hugo/config"
+
+	"github.com/mitchellh/mapstructure"
+)
+
+var DefaultConfig = Config{
+	// The highlighter style to use.
+	// See https://xyproto.github.io/splash/docs/all.html
+	Style:              "monokai",
+	LineNoStart:        1,
+	CodeFences:         true,
+	NoClasses:          true,
+	LineNumbersInTable: true,
+	TabWidth:           4,
+}
+
+//
+type Config struct {
+	Style string
+
+	CodeFences bool
+
+	// Use inline CSS styles.
+	NoClasses bool
+
+	// When set, line numbers will be printed.
+	LineNos            bool
+	LineNumbersInTable bool
+
+	// Start the line numbers from this value (default is 1).
+	LineNoStart int
+
+	// A space separated list of line numbers, e.g. “3-8 10-20”.
+	Hl_Lines string
+
+	// TabWidth sets the number of characters for a tab. Defaults to 4.
+	TabWidth int
+}
+
+func (cfg Config) ToHTMLOptions() []html.Option {
+	var options = []html.Option{
+		html.TabWidth(cfg.TabWidth),
+		html.WithLineNumbers(cfg.LineNos),
+		html.BaseLineNumber(cfg.LineNoStart),
+		html.LineNumbersInTable(cfg.LineNumbersInTable),
+		html.WithClasses(!cfg.NoClasses),
+	}
+
+	if cfg.Hl_Lines != "" {
+		ranges, err := hlLinesToRanges(cfg.LineNoStart, cfg.Hl_Lines)
+		if err == nil {
+			options = append(options, html.HighlightLines(ranges))
+		}
+	}
+
+	return options
+}
+
+func applyOptionsFromString(opts string, cfg *Config) error {
+	optsm, err := parseOptions(opts)
+	if err != nil {
+		return err
+	}
+	return mapstructure.WeakDecode(optsm, cfg)
+}
+
+// ApplyLegacyConfig applies legacy config from back when we had
+// Pygments.
+func ApplyLegacyConfig(cfg config.Provider, conf *Config) error {
+	if conf.Style == DefaultConfig.Style {
+		if s := cfg.GetString("pygmentsStyle"); s != "" {
+			conf.Style = s
+		}
+	}
+
+	if conf.NoClasses == DefaultConfig.NoClasses && cfg.IsSet("pygmentsUseClasses") {
+		conf.NoClasses = !cfg.GetBool("pygmentsUseClasses")
+	}
+
+	if conf.CodeFences == DefaultConfig.CodeFences && cfg.IsSet("pygmentsCodeFences") {
+		conf.CodeFences = cfg.GetBool("pygmentsCodeFences")
+	}
+
+	if cfg.IsSet("pygmentsOptions") {
+		if err := applyOptionsFromString(cfg.GetString("pygmentsOptions"), conf); err != nil {
+			return err
+		}
+	}
+
+	return nil
+
+}
+
+func parseOptions(in string) (map[string]interface{}, error) {
+	in = strings.Trim(in, " ")
+	opts := make(map[string]interface{})
+
+	if in == "" {
+		return opts, nil
+	}
+
+	for _, v := range strings.Split(in, ",") {
+		keyVal := strings.Split(v, "=")
+		key := strings.ToLower(strings.Trim(keyVal[0], " "))
+		if len(keyVal) != 2 {
+			return opts, fmt.Errorf("invalid Highlight option: %s", key)
+		}
+		if key == "linenos" {
+			opts[key] = keyVal[1] != "false"
+			if keyVal[1] == "table" || keyVal[1] == "inline" {
+				opts["lineNumbersInTable"] = keyVal[1] == "table"
+			}
+		} else {
+			opts[key] = keyVal[1]
+		}
+	}
+
+	return opts, nil
+}
+
+// startLine compansates for https://github.com/alecthomas/chroma/issues/30
+func hlLinesToRanges(startLine int, s string) ([][2]int, error) {
+	var ranges [][2]int
+	s = strings.TrimSpace(s)
+
+	if s == "" {
+		return ranges, nil
+	}
+
+	// Variants:
+	// 1 2 3 4
+	// 1-2 3-4
+	// 1-2 3
+	// 1 3-4
+	// 1    3-4
+	fields := strings.Split(s, " ")
+	for _, field := range fields {
+		field = strings.TrimSpace(field)
+		if field == "" {
+			continue
+		}
+		numbers := strings.Split(field, "-")
+		var r [2]int
+		first, err := strconv.Atoi(numbers[0])
+		if err != nil {
+			return ranges, err
+		}
+		first = first + startLine - 1
+		r[0] = first
+		if len(numbers) > 1 {
+			second, err := strconv.Atoi(numbers[1])
+			if err != nil {
+				return ranges, err
+			}
+			second = second + startLine - 1
+			r[1] = second
+		} else {
+			r[1] = first
+		}
+
+		ranges = append(ranges, r)
+	}
+	return ranges, nil
+
+}

markup/highlight/config_test.go

@@ -0,0 +1,59 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package highlight provides code highlighting.
+package highlight
+
+import (
+	"testing"
+
+	"github.com/spf13/viper"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestConfig(t *testing.T) {
+	c := qt.New(t)
+
+	c.Run("applyLegacyConfig", func(c *qt.C) {
+		v := viper.New()
+		v.Set("pygmentsStyle", "hugo")
+		v.Set("pygmentsUseClasses", false)
+		v.Set("pygmentsCodeFences", false)
+		v.Set("pygmentsOptions", "linenos=inline")
+
+		cfg := DefaultConfig
+		err := ApplyLegacyConfig(v, &cfg)
+		c.Assert(err, qt.IsNil)
+		c.Assert(cfg.Style, qt.Equals, "hugo")
+		c.Assert(cfg.NoClasses, qt.Equals, true)
+		c.Assert(cfg.CodeFences, qt.Equals, false)
+		c.Assert(cfg.LineNos, qt.Equals, true)
+		c.Assert(cfg.LineNumbersInTable, qt.Equals, false)
+
+	})
+
+	c.Run("parseOptions", func(c *qt.C) {
+		cfg := DefaultConfig
+		opts := "noclasses=true,linenos=inline,linenostart=32,hl_lines=3-8 10-20"
+		err := applyOptionsFromString(opts, &cfg)
+
+		c.Assert(err, qt.IsNil)
+		c.Assert(cfg.NoClasses, qt.Equals, true)
+		c.Assert(cfg.LineNos, qt.Equals, true)
+		c.Assert(cfg.LineNumbersInTable, qt.Equals, false)
+		c.Assert(cfg.LineNoStart, qt.Equals, 32)
+		c.Assert(cfg.Hl_Lines, qt.Equals, "3-8 10-20")
+
+	})
+}

markup/highlight/highlight.go

@@ -0,0 +1,132 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package highlight
+
+import (
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/alecthomas/chroma"
+	"github.com/alecthomas/chroma/formatters/html"
+	"github.com/alecthomas/chroma/lexers"
+	"github.com/alecthomas/chroma/styles"
+
+	hl "github.com/gohugoio/hugo/markup/highlight/temphighlighting"
+)
+
+func New(cfg Config) Highlighter {
+	return Highlighter{
+		cfg: cfg,
+	}
+}
+
+type Highlighter struct {
+	cfg Config
+}
+
+func (h Highlighter) Highlight(code, lang, optsStr string) (string, error) {
+	cfg := h.cfg
+	if optsStr != "" {
+		if err := applyOptionsFromString(optsStr, &cfg); err != nil {
+			return "", err
+		}
+	}
+	return highlight(code, lang, cfg)
+}
+
+func highlight(code, lang string, cfg Config) (string, error) {
+	w := &strings.Builder{}
+	var lexer chroma.Lexer
+	if lang != "" {
+		lexer = lexers.Get(lang)
+	}
+
+	if lexer == nil {
+		wrapper := getPreWrapper(lang)
+		fmt.Fprint(w, wrapper.Start(true, ""))
+		fmt.Fprint(w, code)
+		fmt.Fprint(w, wrapper.End(true))
+		return w.String(), nil
+	}
+
+	style := styles.Get(cfg.Style)
+	if style == nil {
+		style = styles.Fallback
+	}
+
+	iterator, err := lexer.Tokenise(nil, code)
+	if err != nil {
+		return "", err
+	}
+
+	options := cfg.ToHTMLOptions()
+	options = append(options, getHtmlPreWrapper(lang))
+
+	formatter := html.New(options...)
+
+	fmt.Fprintf(w, `<div class="highlight">`)
+	if err := formatter.Format(w, style, iterator); err != nil {
+		return "", err
+	}
+	fmt.Fprintf(w, `</div>`)
+
+	return w.String(), nil
+}
+func GetCodeBlockOptions() func(ctx hl.CodeBlockContext) []html.Option {
+	return func(ctx hl.CodeBlockContext) []html.Option {
+		var language string
+		if l, ok := ctx.Language(); ok {
+			language = string(l)
+		}
+		return []html.Option{
+			getHtmlPreWrapper(language),
+		}
+	}
+}
+
+func getPreWrapper(language string) preWrapper {
+	return preWrapper{language: language}
+}
+func getHtmlPreWrapper(language string) html.Option {
+	return html.WithPreWrapper(getPreWrapper(language))
+}
+
+type preWrapper struct {
+	language string
+}
+
+func (p preWrapper) Start(code bool, styleAttr string) string {
+	w := &strings.Builder{}
+	fmt.Fprintf(w, "<pre%s>", styleAttr)
+	var language string
+	if code {
+		language = p.language
+	}
+	WriteCodeTag(w, language)
+	return w.String()
+}
+
+func WriteCodeTag(w io.Writer, language string) {
+	fmt.Fprint(w, "<code")
+	if language != "" {
+		fmt.Fprintf(w, " class=\"language-"+language+"\"")
+		fmt.Fprintf(w, " data-lang=\""+language+"\"")
+	}
+	fmt.Fprint(w, ">")
+}
+
+func (p preWrapper) End(code bool) string {
+	return "</code></pre>"
+}

markup/highlight/highlight_test.go

@@ -0,0 +1,87 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package highlight provides code highlighting.
+package highlight
+
+import (
+	"testing"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestHighlight(t *testing.T) {
+	c := qt.New(t)
+
+	lines := `LINE1
+LINE2
+LINE3
+LINE4
+LINE5
+`
+
+	c.Run("Basic", func(c *qt.C) {
+		cfg := DefaultConfig
+		cfg.NoClasses = false
+		h := New(cfg)
+
+		result, _ := h.Highlight(`echo "Hugo Rocks!"`, "bash", "")
+		c.Assert(result, qt.Equals, `<div class="highlight"><pre class="chroma"><code class="language-bash" data-lang="bash"><span class="nb">echo</span> <span class="s2">&#34;Hugo Rocks!&#34;</span></code></pre></div>`)
+		result, _ = h.Highlight(`echo "Hugo Rocks!"`, "unknown", "")
+		c.Assert(result, qt.Equals, `<pre><code class="language-unknown" data-lang="unknown">echo "Hugo Rocks!"</code></pre>`)
+
+	})
+
+	c.Run("Highlight lines, default config", func(c *qt.C) {
+		cfg := DefaultConfig
+		cfg.NoClasses = false
+		h := New(cfg)
+
+		result, _ := h.Highlight(lines, "bash", "linenos=table,hl_lines=2 4-5,linenostart=3")
+		c.Assert(result, qt.Contains, "<div class=\"highlight\"><div class=\"chroma\">\n<table class=\"lntable\"><tr><td class=\"lntd\">\n<pre class=\"chroma\"><code><span class")
+		c.Assert(result, qt.Contains, "<span class=\"hl\"><span class=\"lnt\">4")
+
+		result, _ = h.Highlight(lines, "bash", "linenos=inline,hl_lines=2")
+		c.Assert(result, qt.Contains, "<span class=\"ln\">2</span>LINE2\n</span>")
+		c.Assert(result, qt.Not(qt.Contains), "<table")
+
+		result, _ = h.Highlight(lines, "bash", "linenos=true,hl_lines=2")
+		c.Assert(result, qt.Contains, "<table")
+		c.Assert(result, qt.Contains, "<span class=\"hl\"><span class=\"lnt\">2\n</span>")
+	})
+
+	c.Run("Highlight lines, linenumbers default on", func(c *qt.C) {
+		cfg := DefaultConfig
+		cfg.NoClasses = false
+		cfg.LineNos = true
+		h := New(cfg)
+
+		result, _ := h.Highlight(lines, "bash", "")
+		c.Assert(result, qt.Contains, "<span class=\"lnt\">2\n</span>")
+		result, _ = h.Highlight(lines, "bash", "linenos=false,hl_lines=2")
+		c.Assert(result, qt.Not(qt.Contains), "class=\"lnt\"")
+	})
+
+	c.Run("Highlight lines, linenumbers default on, linenumbers in table default off", func(c *qt.C) {
+		cfg := DefaultConfig
+		cfg.NoClasses = false
+		cfg.LineNos = true
+		cfg.LineNumbersInTable = false
+		h := New(cfg)
+
+		result, _ := h.Highlight(lines, "bash", "")
+		c.Assert(result, qt.Contains, "<span class=\"ln\">2</span>LINE2\n<")
+		result, _ = h.Highlight(lines, "bash", "linenos=table")
+		c.Assert(result, qt.Contains, "<span class=\"lnt\">1\n</span>")
+	})
+}

markup/highlight/temphighlighting/highlighting.go

@@ -0,0 +1,512 @@
+// package highlighting is a extension for the goldmark(http://github.com/yuin/goldmark).
+//
+// This extension adds syntax-highlighting to the fenced code blocks using
+// chroma(https://github.com/alecthomas/chroma).
+//
+// TODO(bep) this is a very temporary fork based on https://github.com/yuin/goldmark-highlighting/pull/10
+// MIT Licensed, Copyright Yusuke Inuzuka
+package temphighlighting
+
+import (
+	"bytes"
+	"io"
+	"strconv"
+	"strings"
+
+	"github.com/yuin/goldmark"
+	"github.com/yuin/goldmark/ast"
+	"github.com/yuin/goldmark/parser"
+	"github.com/yuin/goldmark/renderer"
+	"github.com/yuin/goldmark/renderer/html"
+	"github.com/yuin/goldmark/text"
+	"github.com/yuin/goldmark/util"
+
+	"github.com/alecthomas/chroma"
+	chromahtml "github.com/alecthomas/chroma/formatters/html"
+	"github.com/alecthomas/chroma/lexers"
+	"github.com/alecthomas/chroma/styles"
+)
+
+// ImmutableAttributes is a read-only interface for ast.Attributes.
+type ImmutableAttributes interface {
+	// Get returns (value, true) if an attribute associated with given
+	// name exists, otherwise (nil, false)
+	Get(name []byte) (interface{}, bool)
+
+	// GetString returns (value, true) if an attribute associated with given
+	// name exists, otherwise (nil, false)
+	GetString(name string) (interface{}, bool)
+
+	// All returns all attributes.
+	All() []ast.Attribute
+}
+
+type immutableAttributes struct {
+	n ast.Node
+}
+
+func (a *immutableAttributes) Get(name []byte) (interface{}, bool) {
+	return a.n.Attribute(name)
+}
+
+func (a *immutableAttributes) GetString(name string) (interface{}, bool) {
+	return a.n.AttributeString(name)
+}
+
+func (a *immutableAttributes) All() []ast.Attribute {
+	if a.n.Attributes() == nil {
+		return []ast.Attribute{}
+	}
+	return a.n.Attributes()
+}
+
+// CodeBlockContext holds contextual information of code highlighting.
+type CodeBlockContext interface {
+	// Language returns (language, true) if specified, otherwise (nil, false).
+	Language() ([]byte, bool)
+
+	// Highlighted returns true if this code block can be highlighted, otherwise false.
+	Highlighted() bool
+
+	// Attributes return attributes of the code block.
+	Attributes() ImmutableAttributes
+}
+
+type codeBlockContext struct {
+	language    []byte
+	highlighted bool
+	attributes  ImmutableAttributes
+}
+
+func newCodeBlockContext(language []byte, highlighted bool, attrs ImmutableAttributes) CodeBlockContext {
+	return &codeBlockContext{
+		language:    language,
+		highlighted: highlighted,
+		attributes:  attrs,
+	}
+}
+
+func (c *codeBlockContext) Language() ([]byte, bool) {
+	if c.language != nil {
+		return c.language, true
+	}
+	return nil, false
+}
+
+func (c *codeBlockContext) Highlighted() bool {
+	return c.highlighted
+}
+
+func (c *codeBlockContext) Attributes() ImmutableAttributes {
+	return c.attributes
+}
+
+// WrapperRenderer renders wrapper elements like div, pre, etc.
+type WrapperRenderer func(w util.BufWriter, context CodeBlockContext, entering bool)
+
+// CodeBlockOptions creates Chroma options per code block.
+type CodeBlockOptions func(ctx CodeBlockContext) []chromahtml.Option
+
+// Config struct holds options for the extension.
+type Config struct {
+	html.Config
+
+	// Style is a highlighting style.
+	// Supported styles are defined under https://github.com/alecthomas/chroma/tree/master/formatters.
+	Style string
+
+	// FormatOptions is a option related to output formats.
+	// See https://github.com/alecthomas/chroma#the-html-formatter for details.
+	FormatOptions []chromahtml.Option
+
+	// CSSWriter is an io.Writer that will be used as CSS data output buffer.
+	// If WithClasses() is enabled, you can get CSS data corresponds to the style.
+	CSSWriter io.Writer
+
+	// CodeBlockOptions allows set Chroma options per code block.
+	CodeBlockOptions CodeBlockOptions
+
+	// WrapperRendererCodeBlockOptions allows you to change wrapper elements.
+	WrapperRenderer WrapperRenderer
+}
+
+// NewConfig returns a new Config with defaults.
+func NewConfig() Config {
+	return Config{
+		Config:           html.NewConfig(),
+		Style:            "github",
+		FormatOptions:    []chromahtml.Option{},
+		CSSWriter:        nil,
+		WrapperRenderer:  nil,
+		CodeBlockOptions: nil,
+	}
+}
+
+// SetOption implements renderer.SetOptioner.
+func (c *Config) SetOption(name renderer.OptionName, value interface{}) {
+	switch name {
+	case optStyle:
+		c.Style = value.(string)
+	case optFormatOptions:
+		if value != nil {
+			c.FormatOptions = value.([]chromahtml.Option)
+		}
+	case optCSSWriter:
+		c.CSSWriter = value.(io.Writer)
+	case optWrapperRenderer:
+		c.WrapperRenderer = value.(WrapperRenderer)
+	case optCodeBlockOptions:
+		c.CodeBlockOptions = value.(CodeBlockOptions)
+	default:
+		c.Config.SetOption(name, value)
+	}
+}
+
+// Option interface is a functional option interface for the extension.
+type Option interface {
+	renderer.Option
+	// SetHighlightingOption sets given option to the extension.
+	SetHighlightingOption(*Config)
+}
+
+type withHTMLOptions struct {
+	value []html.Option
+}
+
+func (o *withHTMLOptions) SetConfig(c *renderer.Config) {
+	if o.value != nil {
+		for _, v := range o.value {
+			v.(renderer.Option).SetConfig(c)
+		}
+	}
+}
+
+func (o *withHTMLOptions) SetHighlightingOption(c *Config) {
+	if o.value != nil {
+		for _, v := range o.value {
+			v.SetHTMLOption(&c.Config)
+		}
+	}
+}
+
+// WithHTMLOptions is functional option that wraps goldmark HTMLRenderer options.
+func WithHTMLOptions(opts ...html.Option) Option {
+	return &withHTMLOptions{opts}
+}
+
+const optStyle renderer.OptionName = "HighlightingStyle"
+
+var highlightLinesAttrName = []byte("hl_lines")
+
+var styleAttrName = []byte("hl_style")
+var nohlAttrName = []byte("nohl")
+var linenosAttrName = []byte("linenos")
+var linenosTableAttrValue = []byte("table")
+var linenosInlineAttrValue = []byte("inline")
+var linenostartAttrName = []byte("linenostart")
+
+type withStyle struct {
+	value string
+}
+
+func (o *withStyle) SetConfig(c *renderer.Config) {
+	c.Options[optStyle] = o.value
+}
+
+func (o *withStyle) SetHighlightingOption(c *Config) {
+	c.Style = o.value
+}
+
+// WithStyle is a functional option that changes highlighting style.
+func WithStyle(style string) Option {
+	return &withStyle{style}
+}
+
+const optCSSWriter renderer.OptionName = "HighlightingCSSWriter"
+
+type withCSSWriter struct {
+	value io.Writer
+}
+
+func (o *withCSSWriter) SetConfig(c *renderer.Config) {
+	c.Options[optCSSWriter] = o.value
+}
+
+func (o *withCSSWriter) SetHighlightingOption(c *Config) {
+	c.CSSWriter = o.value
+}
+
+// WithCSSWriter is a functional option that sets io.Writer for CSS data.
+func WithCSSWriter(w io.Writer) Option {
+	return &withCSSWriter{w}
+}
+
+const optWrapperRenderer renderer.OptionName = "HighlightingWrapperRenderer"
+
+type withWrapperRenderer struct {
+	value WrapperRenderer
+}
+
+func (o *withWrapperRenderer) SetConfig(c *renderer.Config) {
+	c.Options[optWrapperRenderer] = o.value
+}
+
+func (o *withWrapperRenderer) SetHighlightingOption(c *Config) {
+	c.WrapperRenderer = o.value
+}
+
+// WithWrapperRenderer is a functional option that sets WrapperRenderer that
+// renders wrapper elements like div, pre, etc.
+func WithWrapperRenderer(w WrapperRenderer) Option {
+	return &withWrapperRenderer{w}
+}
+
+const optCodeBlockOptions renderer.OptionName = "HighlightingCodeBlockOptions"
+
+type withCodeBlockOptions struct {
+	value CodeBlockOptions
+}
+
+func (o *withCodeBlockOptions) SetConfig(c *renderer.Config) {
+	c.Options[optWrapperRenderer] = o.value
+}
+
+func (o *withCodeBlockOptions) SetHighlightingOption(c *Config) {
+	c.CodeBlockOptions = o.value
+}
+
+// WithCodeBlockOptions is a functional option that sets CodeBlockOptions that
+// allows setting Chroma options per code block.
+func WithCodeBlockOptions(c CodeBlockOptions) Option {
+	return &withCodeBlockOptions{value: c}
+}
+
+const optFormatOptions renderer.OptionName = "HighlightingFormatOptions"
+
+type withFormatOptions struct {
+	value []chromahtml.Option
+}
+
+func (o *withFormatOptions) SetConfig(c *renderer.Config) {
+	if _, ok := c.Options[optFormatOptions]; !ok {
+		c.Options[optFormatOptions] = []chromahtml.Option{}
+	}
+	c.Options[optStyle] = append(c.Options[optFormatOptions].([]chromahtml.Option), o.value...)
+}
+
+func (o *withFormatOptions) SetHighlightingOption(c *Config) {
+	c.FormatOptions = append(c.FormatOptions, o.value...)
+}
+
+// WithFormatOptions is a functional option that wraps chroma HTML formatter options.
+func WithFormatOptions(opts ...chromahtml.Option) Option {
+	return &withFormatOptions{opts}
+}
+
+// HTMLRenderer struct is a renderer.NodeRenderer implementation for the extension.
+type HTMLRenderer struct {
+	Config
+}
+
+// NewHTMLRenderer builds a new HTMLRenderer with given options and returns it.
+func NewHTMLRenderer(opts ...Option) renderer.NodeRenderer {
+	r := &HTMLRenderer{
+		Config: NewConfig(),
+	}
+	for _, opt := range opts {
+		opt.SetHighlightingOption(&r.Config)
+	}
+	return r
+}
+
+// RegisterFuncs implements NodeRenderer.RegisterFuncs.
+func (r *HTMLRenderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) {
+	reg.Register(ast.KindFencedCodeBlock, r.renderFencedCodeBlock)
+}
+
+func getAttributes(node *ast.FencedCodeBlock, infostr []byte) ImmutableAttributes {
+	if node.Attributes() != nil {
+		return &immutableAttributes{node}
+	}
+	if infostr != nil {
+		attrStartIdx := -1
+
+		for idx, char := range infostr {
+			if char == '{' {
+				attrStartIdx = idx
+				break
+			}
+		}
+		if attrStartIdx > 0 {
+			n := ast.NewTextBlock() // dummy node for storing attributes
+			attrStr := infostr[attrStartIdx:]
+			if attrs, hasAttr := parser.ParseAttributes(text.NewReader(attrStr)); hasAttr {
+				for _, attr := range attrs {
+					n.SetAttribute(attr.Name, attr.Value)
+				}
+				return &immutableAttributes{n}
+			}
+		}
+	}
+	return nil
+}
+
+func (r *HTMLRenderer) renderFencedCodeBlock(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+	n := node.(*ast.FencedCodeBlock)
+	if !entering {
+		return ast.WalkContinue, nil
+	}
+	language := n.Language(source)
+
+	chromaFormatterOptions := make([]chromahtml.Option, len(r.FormatOptions))
+	copy(chromaFormatterOptions, r.FormatOptions)
+	style := styles.Get(r.Style)
+	nohl := false
+
+	var info []byte
+	if n.Info != nil {
+		info = n.Info.Segment.Value(source)
+	}
+	attrs := getAttributes(n, info)
+	if attrs != nil {
+		baseLineNumber := 1
+		if linenostartAttr, ok := attrs.Get(linenostartAttrName); ok {
+			baseLineNumber = int(linenostartAttr.(float64))
+			chromaFormatterOptions = append(chromaFormatterOptions, chromahtml.BaseLineNumber(baseLineNumber))
+		}
+		if linesAttr, hasLinesAttr := attrs.Get(highlightLinesAttrName); hasLinesAttr {
+			if lines, ok := linesAttr.([]interface{}); ok {
+				var hlRanges [][2]int
+				for _, l := range lines {
+					if ln, ok := l.(float64); ok {
+						hlRanges = append(hlRanges, [2]int{int(ln) + baseLineNumber - 1, int(ln) + baseLineNumber - 1})
+					}
+					if rng, ok := l.([]uint8); ok {
+						slices := strings.Split(string([]byte(rng)), "-")
+						lhs, err := strconv.Atoi(slices[0])
+						if err != nil {
+							continue
+						}
+						rhs := lhs
+						if len(slices) > 1 {
+							rhs, err = strconv.Atoi(slices[1])
+							if err != nil {
+								continue
+							}
+						}
+						hlRanges = append(hlRanges, [2]int{lhs + baseLineNumber - 1, rhs + baseLineNumber - 1})
+					}
+				}
+				chromaFormatterOptions = append(chromaFormatterOptions, chromahtml.HighlightLines(hlRanges))
+			}
+		}
+		if styleAttr, hasStyleAttr := attrs.Get(styleAttrName); hasStyleAttr {
+			styleStr := string([]byte(styleAttr.([]uint8)))
+			style = styles.Get(styleStr)
+		}
+		if _, hasNohlAttr := attrs.Get(nohlAttrName); hasNohlAttr {
+			nohl = true
+		}
+
+		if linenosAttr, ok := attrs.Get(linenosAttrName); ok {
+			switch v := linenosAttr.(type) {
+			case bool:
+				chromaFormatterOptions = append(chromaFormatterOptions, chromahtml.WithLineNumbers(v))
+			case []uint8:
+				if v != nil {
+					chromaFormatterOptions = append(chromaFormatterOptions, chromahtml.WithLineNumbers(true))
+				}
+				if bytes.Equal(v, linenosTableAttrValue) {
+					chromaFormatterOptions = append(chromaFormatterOptions, chromahtml.LineNumbersInTable(true))
+				} else if bytes.Equal(v, linenosInlineAttrValue) {
+					chromaFormatterOptions = append(chromaFormatterOptions, chromahtml.LineNumbersInTable(false))
+				}
+			}
+		}
+	}
+
+	var lexer chroma.Lexer
+	if language != nil {
+		lexer = lexers.Get(string(language))
+	}
+	if !nohl && lexer != nil {
+		if style == nil {
+			style = styles.Fallback
+		}
+		var buffer bytes.Buffer
+		l := n.Lines().Len()
+		for i := 0; i < l; i++ {
+			line := n.Lines().At(i)
+			buffer.Write(line.Value(source))
+		}
+		iterator, err := lexer.Tokenise(nil, buffer.String())
+		if err == nil {
+			c := newCodeBlockContext(language, true, attrs)
+
+			if r.CodeBlockOptions != nil {
+				chromaFormatterOptions = append(chromaFormatterOptions, r.CodeBlockOptions(c)...)
+			}
+			formatter := chromahtml.New(chromaFormatterOptions...)
+			if r.WrapperRenderer != nil {
+				r.WrapperRenderer(w, c, true)
+			}
+			_ = formatter.Format(w, style, iterator) == nil
+			if r.WrapperRenderer != nil {
+				r.WrapperRenderer(w, c, false)
+			}
+			if r.CSSWriter != nil {
+				_ = formatter.WriteCSS(r.CSSWriter, style)
+			}
+			return ast.WalkContinue, nil
+		}
+	}
+
+	var c CodeBlockContext
+	if r.WrapperRenderer != nil {
+		c = newCodeBlockContext(language, false, attrs)
+		r.WrapperRenderer(w, c, true)
+	} else {
+		_, _ = w.WriteString("<pre><code")
+		language := n.Language(source)
+		if language != nil {
+			_, _ = w.WriteString(" class=\"language-")
+			r.Writer.Write(w, language)
+			_, _ = w.WriteString("\"")
+		}
+		_ = w.WriteByte('>')
+	}
+	l := n.Lines().Len()
+	for i := 0; i < l; i++ {
+		line := n.Lines().At(i)
+		r.Writer.RawWrite(w, line.Value(source))
+	}
+	if r.WrapperRenderer != nil {
+		r.WrapperRenderer(w, c, false)
+	} else {
+		_, _ = w.WriteString("</code></pre>\n")
+	}
+	return ast.WalkContinue, nil
+}
+
+type highlighting struct {
+	options []Option
+}
+
+// Highlighting is a goldmark.Extender implementation.
+var Highlighting = &highlighting{
+	options: []Option{},
+}
+
+// NewHighlighting returns a new extension with given options.
+func NewHighlighting(opts ...Option) goldmark.Extender {
+	return &highlighting{
+		options: opts,
+	}
+}
+
+// Extend implements goldmark.Extender.
+func (e *highlighting) Extend(m goldmark.Markdown) {
+	m.Renderer().AddOptions(renderer.WithNodeRenderers(
+		util.Prioritized(NewHTMLRenderer(e.options...), 200),
+	))
+}

markup/highlight/temphighlighting/highlighting_test.go

@@ -0,0 +1,335 @@
+// TODO(bep) this is a very temporary fork based on https://github.com/yuin/goldmark-highlighting/pull/10
+// MIT Licensed, Copyright Yusuke Inuzuka
+package temphighlighting
+
+import (
+	"bytes"
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/yuin/goldmark/util"
+
+	chromahtml "github.com/alecthomas/chroma/formatters/html"
+	"github.com/yuin/goldmark"
+)
+
+type preWrapper struct {
+	language string
+}
+
+func (p preWrapper) Start(code bool, styleAttr string) string {
+	w := &strings.Builder{}
+	fmt.Fprintf(w, "<pre%s><code", styleAttr)
+	if p.language != "" {
+		fmt.Fprintf(w, " class=\"language-"+p.language)
+	}
+	fmt.Fprint(w, ">")
+	return w.String()
+}
+
+func (p preWrapper) End(code bool) string {
+	return "</code></pre>"
+}
+
+func TestHighlighting(t *testing.T) {
+	var css bytes.Buffer
+	markdown := goldmark.New(
+		goldmark.WithExtensions(
+			NewHighlighting(
+				WithStyle("monokai"),
+				WithCSSWriter(&css),
+				WithFormatOptions(
+					chromahtml.WithClasses(true),
+					chromahtml.WithLineNumbers(false),
+				),
+				WithWrapperRenderer(func(w util.BufWriter, c CodeBlockContext, entering bool) {
+					_, ok := c.Language()
+					if entering {
+						if !ok {
+							w.WriteString("<pre><code>")
+							return
+						}
+						w.WriteString(`<div class="highlight">`)
+					} else {
+						if !ok {
+							w.WriteString("</pre></code>")
+							return
+						}
+						w.WriteString(`</div>`)
+					}
+				}),
+				WithCodeBlockOptions(func(c CodeBlockContext) []chromahtml.Option {
+					if language, ok := c.Language(); ok {
+						// Turn on line numbers for Go only.
+						if string(language) == "go" {
+							return []chromahtml.Option{
+								chromahtml.WithLineNumbers(true),
+							}
+						}
+					}
+					return nil
+				}),
+			),
+		),
+	)
+	var buffer bytes.Buffer
+	if err := markdown.Convert([]byte(`
+Title
+=======
+`+"``` go\n"+`func main() {
+    fmt.Println("ok")
+}
+`+"```"+`
+`), &buffer); err != nil {
+		t.Fatal(err)
+	}
+
+	if strings.TrimSpace(buffer.String()) != strings.TrimSpace(`
+<h1>Title</h1>
+<div class="highlight"><pre class="chroma"><span class="ln">1</span><span class="kd">func</span> <span class="nf">main</span><span class="p">(</span><span class="p">)</span> <span class="p">{</span>
+<span class="ln">2</span>    <span class="nx">fmt</span><span class="p">.</span><span class="nf">Println</span><span class="p">(</span><span class="s">&#34;ok&#34;</span><span class="p">)</span>
+<span class="ln">3</span><span class="p">}</span>
+</pre></div>
+`) {
+		t.Error("failed to render HTML")
+	}
+
+	if strings.TrimSpace(css.String()) != strings.TrimSpace(`/* Background */ .chroma { color: #f8f8f2; background-color: #272822 }
+/* Error */ .chroma .err { color: #960050; background-color: #1e0010 }
+/* LineTableTD */ .chroma .lntd { vertical-align: top; padding: 0; margin: 0; border: 0; }
+/* LineTable */ .chroma .lntable { border-spacing: 0; padding: 0; margin: 0; border: 0; width: auto; overflow: auto; display: block; }
+/* LineHighlight */ .chroma .hl { display: block; width: 100%;background-color: #3c3d38 }
+/* LineNumbersTable */ .chroma .lnt { margin-right: 0.4em; padding: 0 0.4em 0 0.4em;color: #7f7f7f }
+/* LineNumbers */ .chroma .ln { margin-right: 0.4em; padding: 0 0.4em 0 0.4em;color: #7f7f7f }
+/* Keyword */ .chroma .k { color: #66d9ef }
+/* KeywordConstant */ .chroma .kc { color: #66d9ef }
+/* KeywordDeclaration */ .chroma .kd { color: #66d9ef }
+/* KeywordNamespace */ .chroma .kn { color: #f92672 }
+/* KeywordPseudo */ .chroma .kp { color: #66d9ef }
+/* KeywordReserved */ .chroma .kr { color: #66d9ef }
+/* KeywordType */ .chroma .kt { color: #66d9ef }
+/* NameAttribute */ .chroma .na { color: #a6e22e }
+/* NameClass */ .chroma .nc { color: #a6e22e }
+/* NameConstant */ .chroma .no { color: #66d9ef }
+/* NameDecorator */ .chroma .nd { color: #a6e22e }
+/* NameException */ .chroma .ne { color: #a6e22e }
+/* NameFunction */ .chroma .nf { color: #a6e22e }
+/* NameOther */ .chroma .nx { color: #a6e22e }
+/* NameTag */ .chroma .nt { color: #f92672 }
+/* Literal */ .chroma .l { color: #ae81ff }
+/* LiteralDate */ .chroma .ld { color: #e6db74 }
+/* LiteralString */ .chroma .s { color: #e6db74 }
+/* LiteralStringAffix */ .chroma .sa { color: #e6db74 }
+/* LiteralStringBacktick */ .chroma .sb { color: #e6db74 }
+/* LiteralStringChar */ .chroma .sc { color: #e6db74 }
+/* LiteralStringDelimiter */ .chroma .dl { color: #e6db74 }
+/* LiteralStringDoc */ .chroma .sd { color: #e6db74 }
+/* LiteralStringDouble */ .chroma .s2 { color: #e6db74 }
+/* LiteralStringEscape */ .chroma .se { color: #ae81ff }
+/* LiteralStringHeredoc */ .chroma .sh { color: #e6db74 }
+/* LiteralStringInterpol */ .chroma .si { color: #e6db74 }
+/* LiteralStringOther */ .chroma .sx { color: #e6db74 }
+/* LiteralStringRegex */ .chroma .sr { color: #e6db74 }
+/* LiteralStringSingle */ .chroma .s1 { color: #e6db74 }
+/* LiteralStringSymbol */ .chroma .ss { color: #e6db74 }
+/* LiteralNumber */ .chroma .m { color: #ae81ff }
+/* LiteralNumberBin */ .chroma .mb { color: #ae81ff }
+/* LiteralNumberFloat */ .chroma .mf { color: #ae81ff }
+/* LiteralNumberHex */ .chroma .mh { color: #ae81ff }
+/* LiteralNumberInteger */ .chroma .mi { color: #ae81ff }
+/* LiteralNumberIntegerLong */ .chroma .il { color: #ae81ff }
+/* LiteralNumberOct */ .chroma .mo { color: #ae81ff }
+/* Operator */ .chroma .o { color: #f92672 }
+/* OperatorWord */ .chroma .ow { color: #f92672 }
+/* Comment */ .chroma .c { color: #75715e }
+/* CommentHashbang */ .chroma .ch { color: #75715e }
+/* CommentMultiline */ .chroma .cm { color: #75715e }
+/* CommentSingle */ .chroma .c1 { color: #75715e }
+/* CommentSpecial */ .chroma .cs { color: #75715e }
+/* CommentPreproc */ .chroma .cp { color: #75715e }
+/* CommentPreprocFile */ .chroma .cpf { color: #75715e }
+/* GenericDeleted */ .chroma .gd { color: #f92672 }
+/* GenericEmph */ .chroma .ge { font-style: italic }
+/* GenericInserted */ .chroma .gi { color: #a6e22e }
+/* GenericStrong */ .chroma .gs { font-weight: bold }
+/* GenericSubheading */ .chroma .gu { color: #75715e }`) {
+		t.Error("failed to render CSS")
+	}
+
+}
+
+func TestHighlighting2(t *testing.T) {
+	markdown := goldmark.New(
+		goldmark.WithExtensions(
+			Highlighting,
+		),
+	)
+	var buffer bytes.Buffer
+	if err := markdown.Convert([]byte(`
+Title
+=======
+`+"```"+`
+func main() {
+    fmt.Println("ok")
+}
+`+"```"+`
+`), &buffer); err != nil {
+		t.Fatal(err)
+	}
+
+	if strings.TrimSpace(buffer.String()) != strings.TrimSpace(`
+<h1>Title</h1>
+<pre><code>func main() {
+    fmt.Println(&quot;ok&quot;)
+}
+</code></pre>
+`) {
+		t.Error("failed to render HTML")
+	}
+}
+
+func TestHighlighting3(t *testing.T) {
+	markdown := goldmark.New(
+		goldmark.WithExtensions(
+			Highlighting,
+		),
+	)
+	var buffer bytes.Buffer
+	if err := markdown.Convert([]byte(`
+Title
+=======
+
+`+"```"+`cpp {hl_lines=[1,2]}
+#include <iostream>
+int main() {
+    std::cout<< "hello" << std::endl;
+}
+`+"```"+`
+`), &buffer); err != nil {
+		t.Fatal(err)
+	}
+	if strings.TrimSpace(buffer.String()) != strings.TrimSpace(`
+<h1>Title</h1>
+<pre style="background-color:#fff"><span style="display:block;width:100%;background-color:#e5e5e5"><span style="color:#999;font-weight:bold;font-style:italic">#</span><span style="color:#999;font-weight:bold;font-style:italic">include</span> <span style="color:#999;font-weight:bold;font-style:italic">&lt;iostream&gt;</span><span style="color:#999;font-weight:bold;font-style:italic">
+</span></span><span style="display:block;width:100%;background-color:#e5e5e5"><span style="color:#999;font-weight:bold;font-style:italic"></span><span style="color:#458;font-weight:bold">int</span> <span style="color:#900;font-weight:bold">main</span>() {
+</span>    std<span style="color:#000;font-weight:bold">:</span><span style="color:#000;font-weight:bold">:</span>cout<span style="color:#000;font-weight:bold">&lt;</span><span style="color:#000;font-weight:bold">&lt;</span> <span style="color:#d14"></span><span style="color:#d14">&#34;</span><span style="color:#d14">hello</span><span style="color:#d14">&#34;</span> <span style="color:#000;font-weight:bold">&lt;</span><span style="color:#000;font-weight:bold">&lt;</span> std<span style="color:#000;font-weight:bold">:</span><span style="color:#000;font-weight:bold">:</span>endl;
+}
+</pre>
+`) {
+		t.Error("failed to render HTML")
+	}
+}
+
+func TestHighlightingHlLines(t *testing.T) {
+	markdown := goldmark.New(
+		goldmark.WithExtensions(
+			NewHighlighting(
+				WithFormatOptions(
+					chromahtml.WithClasses(true),
+				),
+			),
+		),
+	)
+
+	for i, test := range []struct {
+		attributes string
+		expect     []int
+	}{
+		{`hl_lines=["2"]`, []int{2}},
+		{`hl_lines=["2-3",5],linenostart=5`, []int{2, 3, 5}},
+		{`hl_lines=["2-3"]`, []int{2, 3}},
+	} {
+
+		t.Run(fmt.Sprint(i), func(t *testing.T) {
+
+			var buffer bytes.Buffer
+			codeBlock := fmt.Sprintf(`bash {%s}
+LINE1
+LINE2
+LINE3
+LINE4
+LINE5
+LINE6
+LINE7
+LINE8
+`, test.attributes)
+
+			if err := markdown.Convert([]byte(`
+`+"```"+codeBlock+"```"+`
+`), &buffer); err != nil {
+				t.Fatal(err)
+			}
+
+			for _, line := range test.expect {
+				expectStr := fmt.Sprintf("<span class=\"hl\">LINE%d\n</span>", line)
+				if !strings.Contains(buffer.String(), expectStr) {
+					t.Fatal("got\n", buffer.String(), "\nexpected\n", expectStr)
+				}
+			}
+		})
+	}
+
+}
+
+func TestHighlightingLinenos(t *testing.T) {
+
+	outputLineNumbersInTable := `<div class="chroma">
+<table class="lntable"><tr><td class="lntd">
+<span class="lnt">1
+</span></td>
+<td class="lntd">
+LINE1
+</td></tr></table>
+</div>`
+
+	for i, test := range []struct {
+		attributes         string
+		lineNumbers        bool
+		lineNumbersInTable bool
+		expect             string
+	}{
+		{`linenos=true`, false, false, `<span class="ln">1</span>LINE1`},
+		{`linenos=false`, false, false, `LINE1`},
+		{``, true, false, `<span class="ln">1</span>LINE1`},
+		{``, true, true, outputLineNumbersInTable},
+		{`linenos=inline`, true, true, `<span class="ln">1</span>LINE1`},
+		{`linenos=foo`, false, false, `<span class="ln">1</span>LINE1`},
+		{`linenos=table`, false, false, outputLineNumbersInTable},
+	} {
+
+		t.Run(fmt.Sprint(i), func(t *testing.T) {
+			markdown := goldmark.New(
+				goldmark.WithExtensions(
+					NewHighlighting(
+						WithFormatOptions(
+							chromahtml.WithLineNumbers(test.lineNumbers),
+							chromahtml.LineNumbersInTable(test.lineNumbersInTable),
+							chromahtml.PreventSurroundingPre(true),
+							chromahtml.WithClasses(true),
+						),
+					),
+				),
+			)
+
+			var buffer bytes.Buffer
+			codeBlock := fmt.Sprintf(`bash {%s}
+LINE1
+`, test.attributes)
+
+			content := "```" + codeBlock + "```"
+
+			if err := markdown.Convert([]byte(content), &buffer); err != nil {
+				t.Fatal(err)
+			}
+
+			s := strings.TrimSpace(buffer.String())
+
+			if s != test.expect {
+				t.Fatal("got\n", s, "\nexpected\n", test.expect)
+			}
+
+		})
+	}
+
+}

markup/internal/blackfriday.go

@@ -1,108 +0,0 @@
-// Copyright 2019 The Hugo Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Package helpers implements general utility functions that work with
-// and on content.  The helper functions defined here lay down the
-// foundation of how Hugo works with files and filepaths, and perform
-// string operations on content.
-
-package internal
-
-import (
-	"github.com/gohugoio/hugo/common/maps"
-	"github.com/gohugoio/hugo/markup/converter"
-	"github.com/mitchellh/mapstructure"
-	"github.com/pkg/errors"
-)
-
-// BlackFriday holds configuration values for BlackFriday rendering.
-// It is kept here because it's used in several packages.
-type BlackFriday struct {
-	Smartypants           bool
-	SmartypantsQuotesNBSP bool
-	AngledQuotes          bool
-	Fractions             bool
-	HrefTargetBlank       bool
-	NofollowLinks         bool
-	NoreferrerLinks       bool
-	SmartDashes           bool
-	LatexDashes           bool
-	TaskLists             bool
-	PlainIDAnchors        bool
-	Extensions            []string
-	ExtensionsMask        []string
-	SkipHTML              bool
-
-	FootnoteAnchorPrefix       string
-	FootnoteReturnLinkContents string
-}
-
-func UpdateBlackFriday(old *BlackFriday, m map[string]interface{}) (*BlackFriday, error) {
-	// Create a copy so we can modify it.
-	bf := *old
-	if err := mapstructure.Decode(m, &bf); err != nil {
-		return nil, errors.WithMessage(err, "failed to decode rendering config")
-	}
-	return &bf, nil
-}
-
-// NewBlackfriday creates a new Blackfriday filled with site config or some sane defaults.
-func NewBlackfriday(cfg converter.ProviderConfig) (*BlackFriday, error) {
-	var siteConfig map[string]interface{}
-	if cfg.Cfg != nil {
-		siteConfig = cfg.Cfg.GetStringMap("blackfriday")
-	}
-
-	defaultParam := map[string]interface{}{
-		"smartypants":           true,
-		"angledQuotes":          false,
-		"smartypantsQuotesNBSP": false,
-		"fractions":             true,
-		"hrefTargetBlank":       false,
-		"nofollowLinks":         false,
-		"noreferrerLinks":       false,
-		"smartDashes":           true,
-		"latexDashes":           true,
-		"plainIDAnchors":        true,
-		"taskLists":             true,
-		"skipHTML":              false,
-	}
-
-	maps.ToLower(defaultParam)
-
-	config := make(map[string]interface{})
-
-	for k, v := range defaultParam {
-		config[k] = v
-	}
-
-	for k, v := range siteConfig {
-		config[k] = v
-	}
-
-	combinedConfig := &BlackFriday{}
-	if err := mapstructure.Decode(config, combinedConfig); err != nil {
-		return nil, errors.Errorf("failed to decode Blackfriday config: %s", err)
-	}
-
-	// TODO(bep) update/consolidate docs
-	if combinedConfig.FootnoteAnchorPrefix == "" {
-		combinedConfig.FootnoteAnchorPrefix = cfg.Cfg.GetString("footnoteAnchorPrefix")
-	}
-
-	if combinedConfig.FootnoteReturnLinkContents == "" {
-		combinedConfig.FootnoteReturnLinkContents = cfg.Cfg.GetString("footnoteReturnLinkContents")
-	}
-
-	return combinedConfig, nil
-}

markup/markup.go

@@ -16,6 +16,12 @@ package markup
 import (
 	"strings"
 
+	"github.com/gohugoio/hugo/markup/highlight"
+
+	"github.com/gohugoio/hugo/markup/markup_config"
+
+	"github.com/gohugoio/hugo/markup/goldmark"
+
 	"github.com/gohugoio/hugo/markup/org"
 
 	"github.com/gohugoio/hugo/markup/asciidoc"
@@ -29,39 +35,71 @@ import (
 func NewConverterProvider(cfg converter.ProviderConfig) (ConverterProvider, error) {
 	converters := make(map[string]converter.Provider)
 
-	add := func(p converter.NewProvider, aliases ...string) error {
+	markupConfig, err := markup_config.Decode(cfg.Cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	if cfg.Highlight == nil {
+		h := highlight.New(markupConfig.Highlight)
+		cfg.Highlight = func(code, lang, optsStr string) (string, error) {
+			return h.Highlight(code, lang, optsStr)
+		}
+	}
+
+	cfg.MarkupConfig = markupConfig
+
+	add := func(p converter.ProviderProvider, aliases ...string) error {
 		c, err := p.New(cfg)
 		if err != nil {
 			return err
 		}
+
+		name := c.Name()
+
+		aliases = append(aliases, name)
+
+		if strings.EqualFold(name, cfg.MarkupConfig.DefaultMarkdownHandler) {
+			aliases = append(aliases, "markdown")
+		}
+
 		addConverter(converters, c, aliases...)
 		return nil
 	}
 
-	if err := add(blackfriday.Provider, "md", "markdown", "blackfriday"); err != nil {
+	if err := add(goldmark.Provider); err != nil {
 		return nil, err
 	}
-	if err := add(mmark.Provider, "mmark"); err != nil {
+	if err := add(blackfriday.Provider); err != nil {
 		return nil, err
 	}
-	if err := add(asciidoc.Provider, "asciidoc"); err != nil {
+	if err := add(mmark.Provider); err != nil {
 		return nil, err
 	}
-	if err := add(rst.Provider, "rst"); err != nil {
+	if err := add(asciidoc.Provider, "ad", "adoc"); err != nil {
 		return nil, err
 	}
-	if err := add(pandoc.Provider, "pandoc"); err != nil {
+	if err := add(rst.Provider); err != nil {
 		return nil, err
 	}
-	if err := add(org.Provider, "org"); err != nil {
+	if err := add(pandoc.Provider, "pdc"); err != nil {
+		return nil, err
+	}
+	if err := add(org.Provider); err != nil {
 		return nil, err
 	}
 
-	return &converterRegistry{converters: converters}, nil
+	return &converterRegistry{
+		config:     cfg,
+		converters: converters,
+	}, nil
 }
 
 type ConverterProvider interface {
 	Get(name string) converter.Provider
+	//Default() converter.Provider
+	GetMarkupConfig() markup_config.Config
+	Highlight(code, lang, optsStr string) (string, error)
 }
 
 type converterRegistry struct {
@@ -70,12 +108,22 @@ type converterRegistry struct {
 	// may be registered multiple times.
 	// All names are lower case.
 	converters map[string]converter.Provider
+
+	config converter.ProviderConfig
 }
 
 func (r *converterRegistry) Get(name string) converter.Provider {
 	return r.converters[strings.ToLower(name)]
 }
 
+func (r *converterRegistry) Highlight(code, lang, optsStr string) (string, error) {
+	return r.config.Highlight(code, lang, optsStr)
+}
+
+func (r *converterRegistry) GetMarkupConfig() markup_config.Config {
+	return r.config.MarkupConfig
+}
+
 func addConverter(m map[string]converter.Provider, c converter.Provider, aliases ...string) {
 	for _, alias := range aliases {
 		m[alias] = c

markup/markup_config/config.go

@@ -0,0 +1,105 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package markup_config
+
+import (
+	"github.com/gohugoio/hugo/config"
+	"github.com/gohugoio/hugo/docshelper"
+	"github.com/gohugoio/hugo/markup/blackfriday/blackfriday_config"
+	"github.com/gohugoio/hugo/markup/goldmark/goldmark_config"
+	"github.com/gohugoio/hugo/markup/highlight"
+	"github.com/gohugoio/hugo/markup/tableofcontents"
+	"github.com/gohugoio/hugo/parser"
+	"github.com/mitchellh/mapstructure"
+)
+
+type Config struct {
+	// Default markdown handler for md/markdown extensions.
+	// Default is "goldmark".
+	// Before Hugo 0.60 this was "blackfriday".
+	DefaultMarkdownHandler string
+
+	Highlight       highlight.Config
+	TableOfContents tableofcontents.Config
+
+	// Content renderers
+	Goldmark    goldmark_config.Config
+	BlackFriday blackfriday_config.Config
+}
+
+func Decode(cfg config.Provider) (conf Config, err error) {
+	conf = Default
+
+	m := cfg.GetStringMap("markup")
+	if m == nil {
+		return
+	}
+
+	err = mapstructure.WeakDecode(m, &conf)
+	if err != nil {
+		return
+	}
+
+	if err = applyLegacyConfig(cfg, &conf); err != nil {
+		return
+	}
+
+	if err = highlight.ApplyLegacyConfig(cfg, &conf.Highlight); err != nil {
+		return
+	}
+
+	return
+}
+
+func applyLegacyConfig(cfg config.Provider, conf *Config) error {
+	if bm := cfg.GetStringMap("blackfriday"); bm != nil {
+		// Legacy top level blackfriday config.
+		err := mapstructure.WeakDecode(bm, &conf.BlackFriday)
+		if err != nil {
+			return err
+		}
+	}
+
+	if conf.BlackFriday.FootnoteAnchorPrefix == "" {
+		conf.BlackFriday.FootnoteAnchorPrefix = cfg.GetString("footnoteAnchorPrefix")
+	}
+
+	if conf.BlackFriday.FootnoteReturnLinkContents == "" {
+		conf.BlackFriday.FootnoteReturnLinkContents = cfg.GetString("footnoteReturnLinkContents")
+	}
+
+	return nil
+
+}
+
+var Default = Config{
+	DefaultMarkdownHandler: "goldmark",
+
+	TableOfContents: tableofcontents.DefaultConfig,
+	Highlight:       highlight.DefaultConfig,
+
+	Goldmark:    goldmark_config.Default,
+	BlackFriday: blackfriday_config.Default,
+}
+
+func init() {
+	docsProvider := func() map[string]interface{} {
+		docs := make(map[string]interface{})
+		docs["markup"] = parser.LowerCaseCamelJSONMarshaller{Value: Default}
+		return docs
+
+	}
+	// TODO(bep) merge maps
+	docshelper.AddDocProvider("config", docsProvider)
+}

markup/markup_config/config_test.go

@@ -0,0 +1,69 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package markup_config
+
+import (
+	"testing"
+
+	"github.com/spf13/viper"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestConfig(t *testing.T) {
+	c := qt.New(t)
+
+	c.Run("Decode", func(c *qt.C) {
+		c.Parallel()
+		v := viper.New()
+
+		v.Set("markup", map[string]interface{}{
+			"goldmark": map[string]interface{}{
+				"renderer": map[string]interface{}{
+					"unsafe": true,
+				},
+			},
+		})
+
+		conf, err := Decode(v)
+
+		c.Assert(err, qt.IsNil)
+		c.Assert(conf.Goldmark.Renderer.Unsafe, qt.Equals, true)
+		c.Assert(conf.BlackFriday.Fractions, qt.Equals, true)
+
+	})
+
+	c.Run("legacy", func(c *qt.C) {
+		c.Parallel()
+		v := viper.New()
+
+		v.Set("blackfriday", map[string]interface{}{
+			"angledQuotes": true,
+		})
+
+		v.Set("footnoteAnchorPrefix", "myprefix")
+		v.Set("footnoteReturnLinkContents", "myreturn")
+		v.Set("pygmentsStyle", "hugo")
+
+		conf, err := Decode(v)
+
+		c.Assert(err, qt.IsNil)
+		c.Assert(conf.BlackFriday.AngledQuotes, qt.Equals, true)
+		c.Assert(conf.BlackFriday.FootnoteAnchorPrefix, qt.Equals, "myprefix")
+		c.Assert(conf.BlackFriday.FootnoteReturnLinkContents, qt.Equals, "myreturn")
+		c.Assert(conf.Highlight.Style, qt.Equals, "hugo")
+		c.Assert(conf.Highlight.CodeFences, qt.Equals, true)
+	})
+
+}

markup/markup_test.go

@@ -29,13 +29,23 @@ func TestConverterRegistry(t *testing.T) {
 	r, err := NewConverterProvider(converter.ProviderConfig{Cfg: viper.New()})
 
 	c.Assert(err, qt.IsNil)
+	c.Assert("goldmark", qt.Equals, r.GetMarkupConfig().DefaultMarkdownHandler)
+
+	checkName := func(name string) {
+		p := r.Get(name)
+		c.Assert(p, qt.Not(qt.IsNil))
+		c.Assert(p.Name(), qt.Equals, name)
+	}
 
 	c.Assert(r.Get("foo"), qt.IsNil)
-	c.Assert(r.Get("markdown"), qt.Not(qt.IsNil))
+	c.Assert(r.Get("markdown").Name(), qt.Equals, "goldmark")
-	c.Assert(r.Get("mmark"), qt.Not(qt.IsNil))
+
-	c.Assert(r.Get("asciidoc"), qt.Not(qt.IsNil))
+	checkName("goldmark")
-	c.Assert(r.Get("rst"), qt.Not(qt.IsNil))
+	checkName("mmark")
-	c.Assert(r.Get("pandoc"), qt.Not(qt.IsNil))
+	checkName("asciidoc")
-	c.Assert(r.Get("org"), qt.Not(qt.IsNil))
+	checkName("rst")
+	checkName("pandoc")
+	checkName("org")
+	checkName("blackfriday")
 
 }

markup/mmark/convert.go

@@ -15,33 +15,28 @@
 package mmark
 
 import (
-	"github.com/gohugoio/hugo/markup/internal"
+	"github.com/gohugoio/hugo/markup/blackfriday/blackfriday_config"
-
 	"github.com/gohugoio/hugo/markup/converter"
 	"github.com/miekg/mmark"
 )
 
 // Provider is the package entry point.
-var Provider converter.NewProvider = provider{}
+var Provider converter.ProviderProvider = provider{}
 
 type provider struct {
 }
 
 func (p provider) New(cfg converter.ProviderConfig) (converter.Provider, error) {
-	defaultBlackFriday, err := internal.NewBlackfriday(cfg)
+	defaultBlackFriday := cfg.MarkupConfig.BlackFriday
-	if err != nil {
-		return nil, err
-	}
-
 	defaultExtensions := getMmarkExtensions(defaultBlackFriday)
 
-	var n converter.NewConverter = func(ctx converter.DocumentContext) (converter.Converter, error) {
+	return converter.NewProvider("mmark", func(ctx converter.DocumentContext) (converter.Converter, error) {
 		b := defaultBlackFriday
 		extensions := defaultExtensions
 
 		if ctx.ConfigOverrides != nil {
 			var err error
-			b, err = internal.UpdateBlackFriday(b, ctx.ConfigOverrides)
+			b, err = blackfriday_config.UpdateConfig(b, ctx.ConfigOverrides)
 			if err != nil {
 				return nil, err
 			}
@@ -54,16 +49,14 @@ func (p provider) New(cfg converter.ProviderConfig) (converter.Provider, error)
 			extensions: extensions,
 			cfg:        cfg,
 		}, nil
-	}
+	}), nil
-
-	return n, nil
 
 }
 
 type mmarkConverter struct {
 	ctx        converter.DocumentContext
 	extensions int
-	b          *internal.BlackFriday
+	b          blackfriday_config.Config
 	cfg        converter.ProviderConfig
 }
 
@@ -74,7 +67,7 @@ func (c *mmarkConverter) Convert(ctx converter.RenderContext) (converter.Result,
 
 func getHTMLRenderer(
 	ctx converter.DocumentContext,
-	cfg *internal.BlackFriday,
+	cfg blackfriday_config.Config,
 	pcfg converter.ProviderConfig) mmark.Renderer {
 
 	var (
@@ -97,15 +90,14 @@ func getHTMLRenderer(
 	htmlFlags |= mmark.HTML_FOOTNOTE_RETURN_LINKS
 
 	return &mmarkRenderer{
-		Config:    cfg,
+		BlackfridayConfig: cfg,
-		Cfg:       pcfg.Cfg,
+		Config:            pcfg,
-		highlight: pcfg.Highlight,
+		Renderer:          mmark.HtmlRendererWithParameters(htmlFlags, "", "", renderParameters),
-		Renderer:  mmark.HtmlRendererWithParameters(htmlFlags, "", "", renderParameters),
 	}
 
 }
 
-func getMmarkExtensions(cfg *internal.BlackFriday) int {
+func getMmarkExtensions(cfg blackfriday_config.Config) int {
 	flags := 0
 	flags |= mmark.EXTENSION_TABLES
 	flags |= mmark.EXTENSION_FENCED_CODE

markup/mmark/convert_test.go

@@ -20,19 +20,14 @@ import (
 
 	"github.com/gohugoio/hugo/common/loggers"
 
-	"github.com/miekg/mmark"
-
-	"github.com/gohugoio/hugo/markup/internal"
-
-	"github.com/gohugoio/hugo/markup/converter"
-
 	qt "github.com/frankban/quicktest"
+	"github.com/gohugoio/hugo/markup/blackfriday/blackfriday_config"
+	"github.com/gohugoio/hugo/markup/converter"
+	"github.com/miekg/mmark"
 )
 
 func TestGetMmarkExtensions(t *testing.T) {
-	c := qt.New(t)
+	b := blackfriday_config.Default
-	b, err := internal.NewBlackfriday(converter.ProviderConfig{Cfg: viper.New()})
-	c.Assert(err, qt.IsNil)
 
 	//TODO: This is doing the same just with different marks...
 	type data struct {

markup/mmark/renderer.go

@@ -17,26 +17,24 @@ import (
 	"bytes"
 	"strings"
 
+	"github.com/gohugoio/hugo/markup/blackfriday/blackfriday_config"
+	"github.com/gohugoio/hugo/markup/converter"
 	"github.com/miekg/mmark"
-
-	"github.com/gohugoio/hugo/config"
-	"github.com/gohugoio/hugo/markup/internal"
 )
 
 // hugoHTMLRenderer wraps a blackfriday.Renderer, typically a blackfriday.Html
 // adding some custom behaviour.
 type mmarkRenderer struct {
-	Cfg       config.Provider
+	Config            converter.ProviderConfig
-	Config    *internal.BlackFriday
+	BlackfridayConfig blackfriday_config.Config
-	highlight func(code, lang, optsStr string) (string, error)
 	mmark.Renderer
 }
 
 // BlockCode renders a given text as a block of code.
 func (r *mmarkRenderer) BlockCode(out *bytes.Buffer, text []byte, lang string, caption []byte, subfigure bool, callouts bool) {
-	if r.Cfg.GetBool("pygmentsCodeFences") && (lang != "" || r.Cfg.GetBool("pygmentsCodeFencesGuessSyntax")) {
+	if r.Config.MarkupConfig.Highlight.CodeFences {
 		str := strings.Trim(string(text), "\n\r")
-		highlighted, _ := r.highlight(str, lang, "")
+		highlighted, _ := r.Config.Highlight(str, lang, "")
 		out.WriteString(highlighted)
 	} else {
 		r.Renderer.BlockCode(out, text, lang, caption, subfigure, callouts)

markup/org/convert.go

@@ -23,19 +23,18 @@ import (
 )
 
 // Provider is the package entry point.
-var Provider converter.NewProvider = provide{}
+var Provider converter.ProviderProvider = provide{}
 
 type provide struct {
 }
 
 func (p provide) New(cfg converter.ProviderConfig) (converter.Provider, error) {
-	var n converter.NewConverter = func(ctx converter.DocumentContext) (converter.Converter, error) {
+	return converter.NewProvider("org", func(ctx converter.DocumentContext) (converter.Converter, error) {
 		return &orgConverter{
 			ctx: ctx,
 			cfg: cfg,
 		}, nil
-	}
+	}), nil
-	return n, nil
 }
 
 type orgConverter struct {

markup/pandoc/convert.go

@@ -23,19 +23,18 @@ import (
 )
 
 // Provider is the package entry point.
-var Provider converter.NewProvider = provider{}
+var Provider converter.ProviderProvider = provider{}
 
 type provider struct {
 }
 
 func (p provider) New(cfg converter.ProviderConfig) (converter.Provider, error) {
-	var n converter.NewConverter = func(ctx converter.DocumentContext) (converter.Converter, error) {
+	return converter.NewProvider("pandoc", func(ctx converter.DocumentContext) (converter.Converter, error) {
 		return &pandocConverter{
 			ctx: ctx,
 			cfg: cfg,
 		}, nil
-	}
+	}), nil
-	return n, nil
 
 }
 

markup/rst/convert.go

@@ -25,20 +25,18 @@ import (
 )
 
 // Provider is the package entry point.
-var Provider converter.NewProvider = provider{}
+var Provider converter.ProviderProvider = provider{}
 
 type provider struct {
 }
 
 func (p provider) New(cfg converter.ProviderConfig) (converter.Provider, error) {
-	var n converter.NewConverter = func(ctx converter.DocumentContext) (converter.Converter, error) {
+	return converter.NewProvider("rst", func(ctx converter.DocumentContext) (converter.Converter, error) {
 		return &rstConverter{
 			ctx: ctx,
 			cfg: cfg,
 		}, nil
-	}
+	}), nil
-	return n, nil
-
 }
 
 type rstConverter struct {

markup/tableofcontents/tableofcontents.go

@@ -0,0 +1,148 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tableofcontents
+
+import (
+	"strings"
+)
+
+type Headers []Header
+
+type Header struct {
+	ID   string
+	Text string
+
+	Headers Headers
+}
+
+func (h Header) IsZero() bool {
+	return h.ID == "" && h.Text == ""
+}
+
+type Root struct {
+	Headers Headers
+}
+
+func (toc *Root) AddAt(h Header, y, x int) {
+	for i := len(toc.Headers); i <= y; i++ {
+		toc.Headers = append(toc.Headers, Header{})
+	}
+
+	if x == 0 {
+		toc.Headers[y] = h
+		return
+	}
+
+	header := &toc.Headers[y]
+
+	for i := 1; i < x; i++ {
+		if len(header.Headers) == 0 {
+			header.Headers = append(header.Headers, Header{})
+		}
+		header = &header.Headers[len(header.Headers)-1]
+	}
+	header.Headers = append(header.Headers, h)
+}
+
+func (toc Root) ToHTML(startLevel, stopLevel int) string {
+	b := &tocBuilder{
+		s:          strings.Builder{},
+		h:          toc.Headers,
+		startLevel: startLevel,
+		stopLevel:  stopLevel,
+	}
+	b.Build()
+	return b.s.String()
+}
+
+type tocBuilder struct {
+	s strings.Builder
+	h Headers
+
+	startLevel int
+	stopLevel  int
+}
+
+func (b *tocBuilder) Build() {
+	b.buildHeaders2(b.h)
+}
+
+func (b *tocBuilder) buildHeaders2(h Headers) {
+	b.s.WriteString("<nav id=\"TableOfContents\">")
+	b.buildHeaders(1, 0, b.h)
+	b.s.WriteString("</nav>")
+}
+
+func (b *tocBuilder) buildHeaders(level, indent int, h Headers) {
+	if level < b.startLevel {
+		for _, h := range h {
+			b.buildHeaders(level+1, indent, h.Headers)
+		}
+		return
+	}
+
+	if b.stopLevel != -1 && level > b.stopLevel {
+		return
+	}
+
+	hasChildren := len(h) > 0
+
+	if hasChildren {
+		b.s.WriteString("\n")
+		b.indent(indent + 1)
+		b.s.WriteString("<ul>\n")
+	}
+
+	for _, h := range h {
+		b.buildHeader(level+1, indent+2, h)
+	}
+
+	if hasChildren {
+		b.indent(indent + 1)
+		b.s.WriteString("</ul>")
+		b.s.WriteString("\n")
+		b.indent(indent)
+	}
+
+}
+func (b *tocBuilder) buildHeader(level, indent int, h Header) {
+	b.indent(indent)
+	b.s.WriteString("<li>")
+	if !h.IsZero() {
+		b.s.WriteString("<a href=\"#" + h.ID + "\">" + h.Text + "</a>")
+	}
+	b.buildHeaders(level, indent, h.Headers)
+	b.s.WriteString("</li>\n")
+}
+
+func (b *tocBuilder) indent(n int) {
+	for i := 0; i < n; i++ {
+		b.s.WriteString("  ")
+	}
+}
+
+var DefaultConfig = Config{
+	StartLevel: 2,
+	EndLevel:   3,
+}
+
+type Config struct {
+	// Heading start level to include in the table of contents, starting
+	// at h1 (inclusive).
+	StartLevel int
+
+	// Heading end level, inclusive, to include in the table of contents.
+	// Default is 3, a value of -1 will include everything.
+	EndLevel int
+}

markup/tableofcontents/tableofcontents_test.go

@@ -0,0 +1,119 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tableofcontents
+
+import (
+	"testing"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestToc(t *testing.T) {
+	c := qt.New(t)
+
+	toc := &Root{}
+
+	toc.AddAt(Header{Text: "Header 1", ID: "h1-1"}, 0, 0)
+	toc.AddAt(Header{Text: "1-H2-1", ID: "1-h2-1"}, 0, 1)
+	toc.AddAt(Header{Text: "1-H2-2", ID: "1-h2-2"}, 0, 1)
+	toc.AddAt(Header{Text: "1-H3-1", ID: "1-h2-2"}, 0, 2)
+	toc.AddAt(Header{Text: "Header 2", ID: "h1-2"}, 1, 0)
+
+	got := toc.ToHTML(1, -1)
+	c.Assert(got, qt.Equals, `<nav id="TableOfContents">
+  <ul>
+    <li><a href="#h1-1">Header 1</a>
+      <ul>
+        <li><a href="#1-h2-1">1-H2-1</a></li>
+        <li><a href="#1-h2-2">1-H2-2</a>
+          <ul>
+            <li><a href="#1-h2-2">1-H3-1</a></li>
+          </ul>
+        </li>
+      </ul>
+    </li>
+    <li><a href="#h1-2">Header 2</a></li>
+  </ul>
+</nav>`, qt.Commentf(got))
+
+	got = toc.ToHTML(1, 1)
+	c.Assert(got, qt.Equals, `<nav id="TableOfContents">
+  <ul>
+    <li><a href="#h1-1">Header 1</a></li>
+    <li><a href="#h1-2">Header 2</a></li>
+  </ul>
+</nav>`, qt.Commentf(got))
+
+	got = toc.ToHTML(1, 2)
+	c.Assert(got, qt.Equals, `<nav id="TableOfContents">
+  <ul>
+    <li><a href="#h1-1">Header 1</a>
+      <ul>
+        <li><a href="#1-h2-1">1-H2-1</a></li>
+        <li><a href="#1-h2-2">1-H2-2</a></li>
+      </ul>
+    </li>
+    <li><a href="#h1-2">Header 2</a></li>
+  </ul>
+</nav>`, qt.Commentf(got))
+
+	got = toc.ToHTML(2, 2)
+	c.Assert(got, qt.Equals, `<nav id="TableOfContents">
+  <ul>
+    <li><a href="#1-h2-1">1-H2-1</a></li>
+    <li><a href="#1-h2-2">1-H2-2</a></li>
+  </ul>
+</nav>`, qt.Commentf(got))
+
+}
+
+func TestTocMissingParent(t *testing.T) {
+	c := qt.New(t)
+
+	toc := &Root{}
+
+	toc.AddAt(Header{Text: "H2", ID: "h2"}, 0, 1)
+	toc.AddAt(Header{Text: "H3", ID: "h3"}, 1, 2)
+	toc.AddAt(Header{Text: "H3", ID: "h3"}, 1, 2)
+
+	got := toc.ToHTML(1, -1)
+	c.Assert(got, qt.Equals, `<nav id="TableOfContents">
+  <ul>
+    <li>
+      <ul>
+        <li><a href="#h2">H2</a></li>
+      </ul>
+    </li>
+    <li>
+      <ul>
+        <li>
+          <ul>
+            <li><a href="#h3">H3</a></li>
+            <li><a href="#h3">H3</a></li>
+          </ul>
+        </li>
+      </ul>
+    </li>
+  </ul>
+</nav>`, qt.Commentf(got))
+
+	got = toc.ToHTML(3, 3)
+	c.Assert(got, qt.Equals, `<nav id="TableOfContents">
+  <ul>
+    <li><a href="#h3">H3</a></li>
+    <li><a href="#h3">H3</a></li>
+  </ul>
+</nav>`, qt.Commentf(got))
+
+}

parser/lowercase_camel_json.go

@@ -0,0 +1,51 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package parser
+
+import (
+	"encoding/json"
+	"regexp"
+	"unicode"
+	"unicode/utf8"
+)
+
+// Regexp definitions
+var keyMatchRegex = regexp.MustCompile(`\"(\w+)\":`)
+var wordBarrierRegex = regexp.MustCompile(`(\w)([A-Z])`)
+
+// Code adapted from https://gist.github.com/piersy/b9934790a8892db1a603820c0c23e4a7
+type LowerCaseCamelJSONMarshaller struct {
+	Value interface{}
+}
+
+func (c LowerCaseCamelJSONMarshaller) MarshalJSON() ([]byte, error) {
+	marshalled, err := json.Marshal(c.Value)
+
+	converted := keyMatchRegex.ReplaceAllFunc(
+		marshalled,
+		func(match []byte) []byte {
+			// Empty keys are valid JSON, only lowercase if we do not have an
+			// empty key.
+			if len(match) > 2 {
+				// Decode first rune after the double quotes
+				r, width := utf8.DecodeRune(match[1:])
+				r = unicode.ToLower(r)
+				utf8.EncodeRune(match[1:width+1], r)
+			}
+			return match
+		},
+	)
+
+	return converted, err
+}

tpl/transform/remarshal.go

@@ -18,32 +18,41 @@ import (
 // change without notice if it serves a purpose in the docs.
 // Format is one of json, yaml or toml.
 func (ns *Namespace) Remarshal(format string, data interface{}) (string, error) {
-	from, err := cast.ToStringE(data)
+	var meta map[string]interface{}
-	if err != nil {
-		return "", err
-	}
 
-	from = strings.TrimSpace(from)
 	format = strings.TrimSpace(strings.ToLower(format))
 
-	if from == "" {
-		return "", nil
-	}
-
 	mark, err := toFormatMark(format)
 	if err != nil {
 		return "", err
 	}
 
-	fromFormat := metadecoders.Default.FormatFromContentString(from)
+	if m, ok := data.(map[string]interface{}); ok {
-	if fromFormat == "" {
+		meta = m
-		return "", errors.New("failed to detect format from content")
+	} else {
+		from, err := cast.ToStringE(data)
+		if err != nil {
+			return "", err
+		}
+
+		from = strings.TrimSpace(from)
+		if from == "" {
+			return "", nil
+		}
+
+		fromFormat := metadecoders.Default.FormatFromContentString(from)
+		if fromFormat == "" {
+			return "", errors.New("failed to detect format from content")
+		}
+
+		meta, err = metadecoders.Default.UnmarshalToMap([]byte(from), fromFormat)
+		if err != nil {
+			return "", err
+		}
 	}
 
-	meta, err := metadecoders.Default.UnmarshalToMap([]byte(from), fromFormat)
+	// Make it so 1.0 float64 prints as 1 etc.
-	if err != nil {
+	applyMarshalTypes(meta)
-		return "", err
-	}
 
 	var result bytes.Buffer
 	if err := parser.InterfaceToConfig(meta, mark, &result); err != nil {
@@ -53,6 +62,23 @@ func (ns *Namespace) Remarshal(format string, data interface{}) (string, error)
 	return result.String(), nil
 }
 
+// The unmarshal/marshal dance is extremely type lossy, and we need
+// to make sure that integer types prints as "43" and not "43.0" in
+// all formats, hence this hack.
+func applyMarshalTypes(m map[string]interface{}) {
+	for k, v := range m {
+		switch t := v.(type) {
+		case map[string]interface{}:
+			applyMarshalTypes(t)
+		case float64:
+			i := int64(t)
+			if t == float64(i) {
+				m[k] = i
+			}
+		}
+	}
+}
+
 func toFormatMark(format string) (metadecoders.Format, error) {
 	if f := metadecoders.FormatFromString(format); f != "" {
 		return f, nil

tpl/transform/remarshal_test.go

@@ -156,11 +156,11 @@ Hugo = "Rules"
 
 func TestTestRemarshalError(t *testing.T) {
 	t.Parallel()
+	c := qt.New(t)
 
 	v := viper.New()
 	v.Set("contentDir", "content")
 	ns := New(newDeps(v))
-	c := qt.New(t)
 
 	_, err := ns.Remarshal("asdf", "asdf")
 	c.Assert(err, qt.Not(qt.IsNil))
@@ -169,3 +169,19 @@ func TestTestRemarshalError(t *testing.T) {
 	c.Assert(err, qt.Not(qt.IsNil))
 
 }
+
+func TestTestRemarshalMapInput(t *testing.T) {
+	t.Parallel()
+	c := qt.New(t)
+	v := viper.New()
+	v.Set("contentDir", "content")
+	ns := New(newDeps(v))
+
+	input := map[string]interface{}{
+		"hello": "world",
+	}
+
+	output, err := ns.Remarshal("toml", input)
+	c.Assert(err, qt.IsNil)
+	c.Assert(output, qt.Equals, "hello = \"world\"\n")
+}

tpl/transform/transform.go

@@ -65,7 +65,7 @@ func (ns *Namespace) Highlight(s interface{}, lang, opts string) (template.HTML,
 		return "", err
 	}
 
-	highlighted, _ := ns.deps.ContentSpec.Highlight(ss, lang, opts)
+	highlighted, _ := ns.deps.ContentSpec.Converters.Highlight(ss, lang, opts)
 	return template.HTML(highlighted), nil
 }
 

tpl/transform/transform_test.go

@@ -204,7 +204,7 @@ And then some.
 	result, err := ns.Markdownify(text)
 	c.Assert(err, qt.IsNil)
 	c.Assert(result, qt.Equals, template.HTML(
-		"<p>#First</p>\n\n<p>This is some <em>bold</em> text.</p>\n\n<h2 id=\"second\">Second</h2>\n\n<p>This is some more text.</p>\n\n<p>And then some.</p>\n"))
+		"<p>#First</p>\n<p>This is some <em>bold</em> text.</p>\n<h2 id=\"second\">Second</h2>\n<p>This is some more text.</p>\n<p>And then some.</p>\n"))
 
 }