github/hugo
1
0
Fork 0
mirror of https://github.com/gohugoio/hugo.git synced 2024-11-14 20:37:55 -05:00
Code Issues Projects Releases Packages Wiki Activity

Add Goldmark as the new default markdown handler

Browse source 
This commit adds the fast and CommonMark compliant Goldmark as the new default markdown handler in Hugo.

If you want to continue using BlackFriday as the default for md/markdown extensions, you can use this configuration:

```toml
[markup]
defaultMarkdownHandler="blackfriday"
```

Fixes #5963
Fixes #1778
Fixes #6355
This commit is contained in:
Bjørn Erik Pedersen 2019-11-06 20:10:47 +01:00
parent a3fe5e5e35
commit bfb9613a14
No known key found for this signature in database
GPG key ID: 330E6E2BD4859D8F
69 changed files with 3424 additions and 1668 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
commands/genchromastyles.go
View file

@ -68,7 +68,7 @@ func (g *genChromaStyles) generate() error {
if err != nil {
return err
}
formatter := html.New(html.WithClasses())
formatter := html.New(html.WithClasses(true))
formatter.WriteCSS(os.Stdout, style)
return nil
}

12
docs/config.toml
View file

@ -15,18 +15,6 @@ pluralizeListTitles = false
# We do redirects via Netlify's _redirects file, generated by Hugo (see "outputs" below).
disableAliases = true
# Highlighting config (Pygments)
# It is (currently) not in use, but you can do ```go in a content file if you want to.
pygmentsCodeFences = true
pygmentsOptions = ""
# Use the Chroma stylesheet
pygmentsUseClasses = true
pygmentsUseClassic = false
# See https://help.farbox.com/pygments.html
pygmentsStyle = "trac"
[module]
[module.hugoVersion]
min = "0.56.0"

16
docs/config/_default/config.toml
View file

@ -14,17 +14,11 @@ pluralizeListTitles = false
# We do redirects via Netlify's _redirects file, generated by Hugo (see "outputs" below).
disableAliases = true
# Highlighting config (Pygments)
# It is (currently) not in use, but you can do ```go in a content file if you want to.
pygmentsCodeFences = true
pygmentsOptions = ""
# Use the Chroma stylesheet
pygmentsUseClasses = true
pygmentsUseClassic = false
# See https://help.farbox.com/pygments.html
pygmentsStyle = "trac"
[markup]
[markup.highlight]
style = "trac"
lineNumbersInTable = true
noClasses = false
[outputs]
home = [ "HTML", "RSS", "REDIR", "HEADERS" ]

2
docs/content/en/about/features.md
View file

@ -52,7 +52,7 @@ toc: true
* Integrated [Google Analytics][] support
* Automatic [RSS][] creation
* Support for [Go][], [Amber], and [Ace][] HTML templates
* [Syntax highlighting][] powered by [Chroma][] (partly compatible with Pygments)
* [Syntax highlighting][] powered by [Chroma][]
[Ace]: /templates/alternatives/

198
docs/content/en/content-management/formats.md
View file

@ -1,6 +1,6 @@
---
title: Supported Content Formats
linktitle: Supported Content Formats
title: Content Formats
linktitle: Content Formats
description: Both HTML and Markdown are supported content formats.
date: 2017-01-10
publishdate: 2017-01-10
@ -13,191 +13,37 @@ menu:
weight: 20
weight: 20 #rem
draft: false
aliases: [/content/markdown-extras/,/content/supported-formats/,/doc/supported-formats/,/tutorials/mathjax/]
aliases: [/content/markdown-extras/,/content/supported-formats/,/doc/supported-formats/]
toc: true
---
**Markdown is the main content format** and comes in two flavours: The excellent [Blackfriday project][blackfriday] (name your files `*.md` or set `markup = "markdown"` in front matter) or its fork [Mmark][mmark] (name your files `*.mmark` or set `markup = "mmark"` in front matter), both very fast markdown engines written in Go.
You can put any file type into your `/content` directories, but Hugo uses the `markup` front matter value if set or the file extension (see `Markup identifiers` in the table below) to determine if the markup needs to be processed, e.g.:
For Emacs users, [go-org](https://github.com/niklasfasching/go-org) provides built-in native support for Org-mode (name your files `*.org` or set `markup = "org"` in front matter)
* Markdown converted to HTML
* [Shortcodes](/content-management/shortcodes/) processed
* Layout applied
But in many situations, plain HTML is what you want. Just name your files with `.html` or `.htm` extension inside your content folder. Note that if you want your HTML files to have a layout, they need front matter. It can be empty, but it has to be there:
## List of content formats
```html
---
title: "This is a content file in HTML"
---
The current list of content formats in Hugo:
<div>
<h1>Hello, Hugo!</h1>
</div>
```
| Name | Markup identifiers | Comment |
| ------------- | ------------- |-------------|
| Goldmark | md, markdown, goldmark |Note that you can set the default handler of `md` and `markdown` to something else, see [Configure Markup](/getting-started/configuration-markup/).{{< new-in "0.60.0" >}} |
| Blackfriday | blackfriday |Blackfriday will eventually be deprecated.|
|MMark|mmark|Mmark is deprecated and will be removed in a future release.|
|Emacs Org-Mode|org|See [go-org](https://github.com/niklasfasching/go-org).|
|Asciidoc|asciidoc, adoc, ad|Needs Asciidoc or [Asciidoctor][ascii] installed.|
|RST|rst|Needs [RST](http://docutils.sourceforge.net/rst.html) installed.|
|Pandoc|pandoc, pdc|Needs [Pandoc](https://www.pandoc.org/) installed.|
|HTML|html, htm|To be treated as a content file, with layout, shortcodes etc., it must have front matter. If not, it will be copied as-is.|
{{% note "Deeply Nested Lists" %}}
Before you begin writing your content in markdown, Blackfriday has a known issue [(#329)](https://github.com/russross/blackfriday/issues/329) with handling deeply nested lists. Luckily, there is an easy workaround. Use 4-spaces (i.e., <kbd>tab</kbd>) rather than 2-space indentations.
{{% /note %}}
The `markup identifier` is fetched from either the `markup` variable in front matter or from the file extension. For markup-related configuration, see [Configure Markup](/getting-started/configuration-markup/).
## Configure BlackFriday Markdown Rendering
You can configure multiple aspects of Blackfriday as shown in the following list. See the docs on [Configuration][config] for the full list of explicit directions you can give to Hugo when rendering your site.
## External Helpers
{{< readfile file="/content/en/readfiles/bfconfig.md" markdown="true" >}}
## Extend Markdown
Hugo provides some convenient methods for extending markdown.
### Task Lists
Hugo supports [GitHub-styled task lists (i.e., TODO lists)][gfmtasks] for the Blackfriday markdown renderer. If you do not want to use this feature, you can disable it in your configuration.
#### Example Task List Input
{{< code file="content/my-to-do-list.md" >}}
- [ ] a task list item
- [ ] list syntax required
- [ ] incomplete
- [x] completed
{{< /code >}}
#### Example Task List Output
The preceding markdown produces the following HTML in your rendered website:
```
<ul class="task-list">
<li><input type="checkbox" disabled="" class="task-list-item"> a task list item</li>
<li><input type="checkbox" disabled="" class="task-list-item"> list syntax required</li>
<li><input type="checkbox" disabled="" class="task-list-item"> incomplete</li>
<li><input type="checkbox" checked="" disabled="" class="task-list-item"> completed</li>
</ul>
```
#### Example Task List Display
The following shows how the example task list will look to the end users of your website. Note that visual styling of lists is up to you. This list has been styled according to [the Hugo Docs stylesheet][hugocss].
- [ ] a task list item
- [ ] list syntax required
- [ ] incomplete
- [x] completed
### Emojis
To add emojis directly to content, set `enableEmoji` to `true` in your [site configuration][config]. To use emojis in templates or shortcodes, see [`emojify` function][].
For a full list of emojis, see the [Emoji cheat sheet][emojis].
### Shortcodes
If you write in Markdown and find yourself frequently embedding your content with raw HTML, Hugo provides built-in shortcodes functionality. This is one of the most powerful features in Hugo and allows you to create your own Markdown extensions very quickly.
See [Shortcodes][sc] for usage, particularly for the built-in shortcodes that ship with Hugo, and [Shortcode Templating][sct] to learn how to build your own.
### Code Blocks
Hugo supports GitHub-flavored markdown's use of triple back ticks, as well as provides a special [`highlight` shortcode][hlsc], and syntax highlights those code blocks natively using *Chroma*. Users also have an option to use *Pygments* instead. See the [Syntax Highlighting][hl] section for details.
## Mmark
Mmark is a [fork of BlackFriday][mmark] and markdown superset that is well suited for writing [IETF documentation][ietf]. You can see examples of the syntax in the [Mmark GitHub repository][mmark] or the full syntax on [Miek Gieben's website][].
### Use Mmark
As Hugo ships with Mmark, using the syntax is as easy as changing the extension of your content files from `.md` to `.mmark`.
In the event that you want to only use Mmark in specific files, you can also define the Mmark syntax in your content's front matter:
```
---
title: My Post
date: 2017-04-01
markup: mmark
---
```
{{% warning %}}
Thare are some features not available in Mmark; one example being that shortcodes are not translated when used in an included `.mmark` file ([#3131](https://github.com/gohugoio/hugo/issues/3137)), and `EXTENSION_ABBREVIATION` ([#1970](https://github.com/gohugoio/hugo/issues/1970)) and the aforementioned GFM todo lists ([#2270](https://github.com/gohugoio/hugo/issues/2270)) are not fully supported. Contributions are welcome.
{{% /warning %}}
## MathJax with Hugo
[MathJax](https://www.mathjax.org/) is a JavaScript library that allows the display of mathematical expressions described via a LaTeX-style syntax in the HTML (or Markdown) source of a web page. As it is a pure a JavaScript library, getting it to work within Hugo is fairly straightforward, but does have some oddities that will be discussed here.
This is not an introduction into actually using MathJax to render typeset mathematics on your website. Instead, this page is a collection of tips and hints for one way to get MathJax working on a website built with Hugo.
### Enable MathJax
The first step is to enable MathJax on pages that you would like to have typeset math. There are multiple ways to do this (adventurous readers can consult the [Loading and Configuring](https://docs.mathjax.org/en/latest/web/configuration.html) section of the MathJax documentation for additional methods of including MathJax), but the easiest way is to use the secure MathJax CDN by include a `<script>` tag for the officially recommended secure CDN ([cdn.js.com](https://cdnjs.com)):
{{< code file="add-mathjax-to-page.html" >}}
<script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
</script>
{{< /code >}}
One way to ensure that this code is included in all pages is to put it in one of the templates that live in the `layouts/partials/` directory. For example, I have included this in the bottom of my template `footer.html` because I know that the footer will be included in every page of my website.
### Options and Features
MathJax is a stable open-source library with many features. I encourage the interested reader to view the [MathJax Documentation](https://docs.mathjax.org/en/latest/index.html), specifically the sections on [Basic Usage](http://docs.mathjax.org/en/latest/index.html#basic-usage) and [MathJax Configuration Options](http://docs.mathjax.org/en/latest/index.html#mathjax-configuration-options).
### Issues with Markdown
{{% note %}}
The following issues with Markdown assume you are using `.md` for content and BlackFriday for parsing. Using [Mmark](#mmark) as your content format will obviate the need for the following workarounds.
When using Mmark with MathJax, use `displayMath: [['$$','$$'], ['\\[','\\]']]`. See the [Mmark `README.md`](https://github.com/miekg/mmark/wiki/Syntax#math-blocks) for more information. In addition to MathJax, Mmark has been shown to work well with [KaTeX](https://github.com/Khan/KaTeX). See this [related blog post from a Hugo user](http://nosubstance.me/post/a-great-toolset-for-static-blogging/).
{{% /note %}}
After enabling MathJax, any math entered between proper markers (see the [MathJax documentation][mathjaxdocs]) will be processed and typeset in the web page. One issue that comes up, however, with Markdown is that the underscore character (`_`) is interpreted by Markdown as a way to wrap text in `emph` blocks while LaTeX (MathJax) interprets the underscore as a way to create a subscript. This "double speak" of the underscore can result in some unexpected and unwanted behavior.
### Solution
There are multiple ways to remedy this problem. One solution is to simply escape each underscore in your math code by entering `\_` instead of `_`. This can become quite tedious if the equations you are entering are full of subscripts.
Another option is to tell Markdown to treat the MathJax code as verbatim code and not process it. One way to do this is to wrap the math expression inside a `<div>` `</div>` block. Markdown would ignore these sections and they would get passed directly on to MathJax and processed correctly. This works great for display style mathematics, but for inline math expressions the line break induced by the `<div>` is not acceptable. The syntax for instructing Markdown to treat inline text as verbatim is by wrapping it in backticks (`` ` ``). You might have noticed, however, that the text included in between backticks is rendered differently than standard text (on this site these are items highlighted in red). To get around this problem, we could create a new CSS entry that would apply standard styling to all inline verbatim text that includes MathJax code. Below I will show the HTML and CSS source that would accomplish this (note this solution was adapted from [this blog post](http://doswa.com/2011/07/20/mathjax-in-markdown.html)---all credit goes to the original author).
{{< code file="mathjax-markdown-solution.html" >}}
<script type="text/x-mathjax-config">
MathJax.Hub.Config({
tex2jax: {
inlineMath: [['$','$'], ['\\(','\\)']],
displayMath: [['$$','$$'], ['\[','\]']],
processEscapes: true,
processEnvironments: true,
skipTags: ['script', 'noscript', 'style', 'textarea', 'pre'],
TeX: { equationNumbers: { autoNumber: "AMS" },
extensions: ["AMSmath.js", "AMSsymbols.js"] }
}
});
</script>
{{< /code >}}
As before, this content should be included in the HTML source of each page that will be using MathJax. The next code snippet contains the CSS that is used to have verbatim MathJax blocks render with the same font style as the body of the page.
{{< code file="mathjax-style.css" >}}
code.has-jax {
font: inherit;
font-size: 100%;
background: inherit;
border: inherit;
color: #515151;
}
{{< /code >}}
In the CSS snippet, notice the line `color: #515151;`. `#515151` is the value assigned to the `color` attribute of the `body` class in my CSS. In order for the equations to fit in with the body of a web page, this value should be the same as the color of the body.
### Usage
With this setup, everything is in place for a natural usage of MathJax on pages generated using Hugo. In order to include inline mathematics, just put LaTeX code in between `` `$ TeX Code $` `` or `` `\( TeX Code \)` ``. To include display style mathematics, just put LaTeX code in between `<div>$$TeX Code$$</div>`. All the math will be properly typeset and displayed within your Hugo generated web page!
## Additional Formats Through External Helpers
Hugo has a new concept called _external helpers_. It means that you can write your content using [Asciidoc][ascii], [reStructuredText][rest], or [pandoc]. If you have files with associated extensions, Hugo will call external commands to generate the content. ([See the Hugo source code for external helpers][helperssource].)
For example, for Asciidoc files, Hugo will try to call the `asciidoctor` or `asciidoc` command. This means that you will have to install the associated tool on your machine to be able to use these formats. ([See the Asciidoctor docs for installation instructions](https://asciidoctor.org/docs/install-toolchain/)).
To use these formats, just use the standard extension and the front matter exactly as you would do with natively supported `.md` files.
Some of the formats in the table above needs external helpers installed on your PC. For example, for Asciidoc files, Hugo will try to call the `asciidoctor` or `asciidoc` command. This means that you will have to install the associated tool on your machine to be able to use these formats. ([See the Asciidoctor docs for installation instructions](https://asciidoctor.org/docs/install-toolchain/)).
Hugo passes reasonable default arguments to these external helpers by default:

187
docs/content/en/content-management/syntax-highlighting.md
View file

@ -3,7 +3,7 @@ title: Syntax Highlighting
description: Hugo comes with really fast syntax highlighting from Chroma.
date: 2017-02-01
publishdate: 2017-02-01
keywords: [highlighting,pygments,chroma,code blocks,syntax]
keywords: [highlighting,chroma,code blocks,syntax]
categories: [content management]
menu:
docs:
@ -16,17 +16,39 @@ aliases: [/extras/highlighting/,/extras/highlight/,/tools/syntax-highlighting/]
toc: true
---
From Hugo 0.28, the default syntax highlighter in Hugo is [Chroma](https://github.com/alecthomas/chroma); it is built in Go and is really, really fast -- and for the most important parts compatible with Pygments.
If you want to continue to use Pygments (see below), set `pygmentsUseClassic=true` in your site config.
Hugo uses [Chroma](https://github.com/alecthomas/chroma) as its code highlighter; it is built in Go and is really, really fast -- and for the most important parts compatible with Pygments we used before.
The example below shows a simple code snippet from the Hugo source highlighted with the `highlight` shortcode. Note that the gohugo.io site is generated with `pygmentsUseClasses=true` (see [Generate Syntax Highlighter CSS](#generate-syntax-highlighter-css)).
## Configure Syntax Highlighter
* `linenos=inline` or `linenos=table` (`table` will give copy-and-paste friendly code blocks) turns on line numbers.
* `hl_lines` lists a set of line numbers or line number ranges to be highlighted. Note that the hyphen range syntax is only supported for Chroma.
See [Configure Highlight](/getting-started/configuration-markup#highlight).
## Generate Syntax Highlighter CSS
If you run with `pygmentsUseClasses=true` in your site config, you need a style sheet.
You can generate one with Hugo:
```bash
hugo gen chromastyles --style=monokai > syntax.css
```
Run `hugo gen chromastyles -h` for more options. See https://xyproto.github.io/splash/docs/ for a gallery of available styles.
## Highlight Shortcode
Highlighting is carried out via the [built-in shortcode](/content-management/shortcodes/) `highlight`. `highlight` takes exactly one required parameter for the programming language to be highlighted and requires a closing shortcode. Note that `highlight` is *not* used for client-side javascript highlighting.
Options:
* `linenos`: Valid values are `true`, `false`, `table`, `inline`. `table` will give copy-and-paste friendly code blocks) turns on line numbers.
* Setting `linenos` to `false` will turn off linenumbers if it's configured to be on in site config.{{< new-in "0.60.0" >}}
* `hl_lines` lists a set of line numbers or line number ranges to be highlighted.
* `linenostart=199` starts the line number count from 199.
With that, this:
### Example: Highlight Shortcode
```
{{</* highlight go "linenos=table,hl_lines=8 15-17,linenostart=199" */>}}
@ -62,134 +84,63 @@ func GetTitleFunc(style string) func(s string) string {
{{< / highlight >}}
## Configure Syntax Highlighter
To make the transition from Pygments to Chroma seamless, they share a common set of configuration options:
pygmentsOptions
: A comma separated list of options. See below for a full list.
pygmentsCodeFences
: Set to true to enable syntax highlighting in code fences with a language tag in markdown (see below for an example).
pygmentsStyle
: The style of code highlighting. Note that this option is not
relevant when `pygmentsUseClasses` is set.
Syntax highlighting galleries:
**Chroma** ([short snippets](https://xyproto.github.io/splash/docs/all.html),
[long snippets](https://xyproto.github.io/splash/docs/longer/all.html)),
[Pygments](https://help.farbox.com/pygments.html)
pygmentsUseClasses
: Set to `true` to use CSS classes to format your highlighted code. See [Generate Syntax Highlighter CSS](#generate-syntax-highlighter-css).
pygmentsCodeFencesGuessSyntax
: Set to `true` to try to do syntax highlighting on code fenced blocks in markdown without a language tag.
pygmentsUseClassic
: Set to true to use Pygments instead of the much faster Chroma.
### Options
`pygmentsOptions` can be set either in site config or overridden per code block in the Highlight shortcode or template func.
noclasses
: Use inline style.
linenos
: For Chroma, any value in this setting will print line numbers. Pygments has some more fine grained control.
linenostart
: Start the line numbers from this value (default is 1).
hl_lines
: Highlight a space separated list of line numbers. For Chroma, you can provide a list of ranges, i.e. "3-8 10-20".
The full set of supported options for Pygments is: `encoding`, `outencoding`, `nowrap`, `full`, `title`, `style`, `noclasses`, `classprefix`, `cssclass`, `cssstyles`, `prestyles`, `linenos`, `hl_lines`, `linenostart`, `linenostep`, `linenospecial`, `nobackground`, `lineseparator`, `lineanchors`, `linespans`, `anchorlinenos`, `startinline`. See the [Pygments HTML Formatter Documentation](http://pygments.org/docs/formatters/#HtmlFormatter) for details.
## Generate Syntax Highlighter CSS
If you run with `pygmentsUseClasses=true` in your site config, you need a style sheet.
You can generate one with Hugo:
```bash
hugo gen chromastyles --style=monokai > syntax.css
```
Run `hugo gen chromastyles -h` for more options. See https://xyproto.github.io/splash/docs/ for a gallery of available styles.
## Highlight Shortcode
Highlighting is carried out via the [built-in shortcode](/content-management/shortcodes/) `highlight`. `highlight` takes exactly one required parameter for the programming language to be highlighted and requires a closing shortcode. Note that `highlight` is *not* used for client-side javascript highlighting.
### Example `highlight` Shortcode
{{< code file="example-highlight-shortcode-input.md" >}}
{{</* highlight html */>}}
<section id="main">
<div>
<h1 id="title">{{ .Title }}</h1>
{{ range .Pages }}
{{ .Render "summary"}}
{{ end }}
</div>
</section>
{{</* /highlight */>}}
{{< /code >}}
## Highlight Template Func
See [Highlight](/functions/highlight/).
## Highlight in Code Fences
## Highlighting in Code Fences
It is also possible to add syntax highlighting with GitHub flavored code fences. To enable this, set the `pygmentsCodeFences` to `true` in Hugo's [configuration file](/getting-started/configuration/);
Highlighting in code fences is enabled by default.{{< new-in "0.60.0" >}}
````
```go-html-template
<section id="main">
<div>
<h1 id="title">{{ .Title }}</h1>
{{ range .Pages }}
{{ .Render "summary"}}
{{ end }}
</div>
</section>
```go-html-template{hl_lines=[3,"5-6"],linenos=true}
```
````
````
```go {linenos=table,hl_lines=[8,"15-17"],linenostart=199}
// ... code
````
Gives this:
```go {linenos=table,hl_lines=[8,"15-17"],linenostart=199}
// GetTitleFunc returns a func that can be used to transform a string to
// title case.
//
// The supported styles are
//
// - "Go" (strings.Title)
// - "AP" (see https://www.apstylebook.com/)
// - "Chicago" (see https://www.chicagomanualofstyle.org/home.html)
//
// If an unknown or empty style is provided, AP style is what you get.
func GetTitleFunc(style string) func(s string) string {
switch strings.ToLower(style) {
case "go":
return strings.Title
case "chicago":
tc := transform.NewTitleConverter(transform.ChicagoStyle)
return tc.Title
default:
tc := transform.NewTitleConverter(transform.APStyle)
return tc.Title
}
}
```
{{< new-in "0.60.0" >}}Note that only Goldmark supports passing attributes such as `hl_lines`, and it's important that it does not contain any spaces. See [goldmark-highlighting](https://github.com/yuin/goldmark-highlighting) for more information.
The options are the same as in the [highlighting shortcode](/content-management/syntax-highlighting/#highlight-shortcode),including `linenos=false`, but note the slightly different Markdown attribute syntax.
## List of Chroma Highlighting Languages
The full list of Chroma lexers and their aliases (which is the identifier used in the `highlight` template func or when doing highlighting in code fences):
{{< chroma-lexers >}}
## Highlight with Pygments Classic
If you for some reason don't want to use the built-in Chroma highlighter, you can set `pygmentsUseClassic=true` in your config and add Pygments to your path.
{{% note "Disclaimers on Pygments" %}}
* Pygments is relatively slow and _causes a performance hit when building your site_, but Hugo has been designed to cache the results to disk.
* The caching can be turned off by setting the `--ignoreCache` flag to `true`.
* The languages available for highlighting depend on your Pygments installation.
{{% /note %}}
If you have never worked with Pygments before, here is a brief primer:
+ Install Python from [python.org](https://www.python.org/downloads/). Version 2.7.x is already sufficient.
+ Run `pip install Pygments` in order to install Pygments. Once installed, Pygments gives you a command `pygmentize`. Make sure it sits in your PATH; otherwise, Hugo will not be able to find and use it.
On Debian and Ubuntu systems, you may also install Pygments by running `sudo apt-get install python3-pygments`.
[Prism]: https://prismjs.com
[prismdownload]: https://prismjs.com/download.html
[Highlight.js]: https://highlightjs.org/

1
docs/content/en/contribute/documentation.md
View file

@ -104,6 +104,7 @@ Your options for languages are `xml`/`html`, `go`/`golang`, `md`/`markdown`/`mkd
```
````
### Code Block Shortcode
The Hugo documentation comes with a very robust shortcode for adding interactive code blocks.

6
docs/content/en/functions/highlight.md
View file

@ -1,7 +1,7 @@
---
title: highlight
linktitle: highlight
description: Takes a string of code and language declaration and uses Pygments to return syntax-highlighted HTML with inline-styles.
description: Takes a string of code and language declaration and uses Chroma to return syntax-highlighted HTML.
godocref:
date: 2017-02-01
publishdate: 2017-02-01
@ -10,7 +10,7 @@ categories: [functions]
menu:
docs:
parent: "functions"
keywords: [highlighting,pygments,code blocks,syntax]
keywords: [highlighting,code blocks,syntax]
signature: ["highlight INPUT LANG OPTIONS"]
workson: []
hugoversion:
@ -20,8 +20,6 @@ deprecated: false
[`highlight` is used in Hugo's built-in `highlight` shortcode][highlight].
See [Installing Hugo][installpygments] for more information on Pygments or [Syntax Highlighting][syntax] for more options on how to add syntax highlighting to your code blocks with Hugo.
[highlight]: /content-management/shortcodes/#highlight
[installpygments]: /getting-started/installing/#installing-pygments-optional

73
docs/content/en/getting-started/configuration-markup.md Normal file
View file

@ -0,0 +1,73 @@
---
title: Configure Markup
description: How to handle Markdown and other markup related configuration.
date: 2019-11-15
categories: [getting started,fundamentals]
keywords: [configuration,highlighting]
weight: 65
sections_weight: 65
slug: configuration-markup
toc: true
---
## Configure Markup
{{< new-in "0.60.0" >}}
See [Goldmark](#goldmark) for settings related to the default Markdown handler in Hugo.
Below are all markup related configuration in Hugo with their default settings:
{{< code-toggle config="markup" />}}
**See each section below for details.**
### Goldmark
[Goldmark](https://github.com/yuin/goldmark/) is from Hugo 0.60 the default library used for Markdown. It's fast, it's [CommonMark](https://spec.commonmark.org/0.29/) compliant and it's very flexible. Note that the feature set of Goldmark vs Blackfriday isn't the same; you gain a lot but also lose some, but we will work to bridge any gap in the upcoming Hugo versions.
This is the default configuration:
{{< code-toggle config="markup.goldmark" />}}
Some settings explained:
unsafe
: By default, Goldmark does not render raw HTMLs and potentially dangerous links. If you have lots of inline HTML and/or JavaScript, you may need to turn this on.
typographer
: This extension substitutes punctuations with typographic entities like [smartypants](https://daringfireball.net/projects/smartypants/).
### Blackfriday
[Blackfriday](https://github.com/russross/blackfriday) was Hugo's default Markdown rendering engine, now replaced with Goldmark. But you can still use it: Just set `defaultMarkdownHandler` to `blackfriday` in your top level `markup` config.
This is the default config:
{{< code-toggle config="markup.blackFriday" />}}
### Highlight
This is the default `highlight` configuration. Note that some of these settings can be set per code block, see [Syntax Highlighting](/content-management/syntax-highlighting/).
{{< code-toggle config="markup.highlight" />}}
For `style`, see these galleries:
* [Short snippets](https://xyproto.github.io/splash/docs/all.html)
* [Long snippets](https://xyproto.github.io/splash/docs/longer/all.html)
For CSS, see [Generate Syntax Highlighter CSS](/content-management/syntax-highlighting/#generate-syntax-highlighter-css).
### Table Of Contents
{{< code-toggle config="markup.tableOfContents" />}}
These settings only works for the Goldmark renderer:
startLevel
: The heading level, values starting at 1 (`h1`), to start render the table of contents.
endLevel
: The heading level, inclusive, to stop render the table of contents.

48
docs/content/en/getting-started/configuration.md
View file

@ -184,11 +184,14 @@ log (false)
logFile ("")
: Log File path (if set, logging enabled automatically).
markup
: See [Configure Markup](/getting-started/configuration-markup).{{< new-in "0.60.0" >}}
menu
: See [Add Non-content Entries to a Menu](/content-management/menus/#add-non-content-entries-to-a-menu).
module
: Module config see [Module Config](/hugo-modules/configuration/).
: Module config see [Module Config](/hugo-modules/configuration/).{{< new-in "0.56.0" >}}
newContentEditor ("")
: The editor to use when creating new content.
@ -214,26 +217,8 @@ pluralizeListTitles (true)
publishDir ("public")
: The directory to where Hugo will write the final static site (the HTML files etc.).
pygmentsOptions ("")
: A comma separated list of options for syntax highlighting. See the [Syntax Highlighting Options](/content-management/syntax-highlighting/#options) for the full list of available options.
pygmentsCodeFences (false)
: Enables syntax highlighting in [code fences with a language tag](/content-management/syntax-highlighting/#highlight-in-code-fences) in markdown.
pygmentsCodeFencesGuessSyntax (false)
: Enable syntax guessing for code fences without specified language.
pygmentsStyle ("monokai")
: Color-theme or style for syntax highlighting. See [Pygments Color Themes](https://help.farbox.com/pygments.html).
pygmentsUseClasses (false)
: Enable using external CSS for syntax highlighting.
pygmentsUseClassic (false)
: Enable using Pygments instead of the much faster Chroma for syntax highlighting.
related
: See [Related Content](/content-management/related/#configure-related-content).
: See [Related Content](/content-management/related/#configure-related-content).{{< new-in "0.27" >}}
relativeURLs (false)
: Enable this to make all relative URLs relative to content root. Note that this does not affect absolute URLs.
@ -436,29 +421,6 @@ The above will try first to extract the value for `.Date` from the filename, the
`:git`
: This is the Git author date for the last revision of this content file. This will only be set if `--enableGitInfo` is set or `enableGitInfo = true` is set in site config.
## Configure Blackfriday
[Blackfriday](https://github.com/russross/blackfriday) is Hugo's built-in Markdown rendering engine.
Hugo typically configures Blackfriday with sane default values that should fit most use cases reasonably well.
However, if you have specific needs with respect to Markdown, Hugo exposes some of its Blackfriday behavior options for you to alter. The following table lists these Hugo options, paired with the corresponding flags from Blackfriday's source code ( [html.go](https://github.com/russross/blackfriday/blob/master/html.go) and [markdown.go](https://github.com/russross/blackfriday/blob/master/markdown.go)).
{{< readfile file="/content/en/readfiles/bfconfig.md" markdown="true" >}}
{{% note %}}
1. Blackfriday flags are *case sensitive* as of Hugo v0.15.
2. Blackfriday flags must be grouped under the `blackfriday` key and can be set on both the site level *and* the page level. Any setting on a page will override its respective site setting.
{{% /note %}}
{{< code-toggle file="config" >}}
[blackfriday]
angledQuotes = true
fractions = false
plainIDAnchors = true
extensions = ["hardLineBreak"]
{{< /code-toggle >}}
## Configure Additional Output Formats
Hugo v0.20 introduced the ability to render your content to multiple output formats (e.g., to JSON, AMP html, or CSV). See [Output Formats][] for information on how to add these values to your Hugo project's configuration file.

6
docs/content/en/getting-started/installing.md
View file

@ -502,12 +502,6 @@ OpenBSD provides a package for Hugo via `pkg_add`:
Upgrading Hugo is as easy as downloading and replacing the executable you’ve placed in your `PATH` or run `brew upgrade hugo` if using Homebrew.
## Install Pygments (Optional)
The Hugo executable has one *optional* external dependency for source code highlighting ([Pygments][pygments]).
If you want to have source code highlighting using the [highlight shortcode][], you need to install the Python-based Pygments program. The procedure is outlined on the [Pygments homepage][pygments].
## Next Steps
Now that you've installed Hugo, read the [Quick Start guide][quickstart] and explore the rest of the documentation. If you have questions, ask the Hugo community directly by visiting the [Hugo Discussion Forum][forum].

197
docs/content/en/readfiles/bfconfig.md
View file

@ -1,197 +0,0 @@
## Blackfriday Options
`taskLists`
: default: **`true`**<br>
Blackfriday flag: <br>
Purpose: `false` turns off GitHub-style automatic task/TODO list generation.
`smartypants`
: default: **`true`** <br>
Blackfriday flag: **`HTML_USE_SMARTYPANTS`** <br>
Purpose: `false` disables smart punctuation substitutions, including smart quotes, smart dashes, smart fractions, etc. If `true`, it may be fine-tuned with the `angledQuotes`, `fractions`, `smartDashes`, and `latexDashes` flags (see below).
`smartypantsQuotesNBSP`
: default: **`false`** <br>
Blackfriday flag: **`HTML_SMARTYPANTS_QUOTES_NBSP`** <br>
Purpose: `true` enables French style Guillemets with non-breaking space inside the quotes.
`angledQuotes`
: default: **`false`**<br>
Blackfriday flag: **`HTML_SMARTYPANTS_ANGLED_QUOTES`**<br>
Purpose: `true` enables smart, angled double quotes. Example: "Hugo" renders to «Hugo» instead of “Hugo”.
`fractions`
: default: **`true`**<br>
Blackfriday flag: **`HTML_SMARTYPANTS_FRACTIONS`** <br>
Purpose: <code>false</code> disables smart fractions.<br>
Example: `5/12` renders to <sup>5</sup>&frasl;<sub>12</sub>(<code>&lt;sup&gt;5&lt;/sup&gt;&amp;frasl;&lt;sub&gt;12&lt;/sub&gt;</code>).<br> <small><strong>Caveat:</strong> Even with <code>fractions = false</code>, Blackfriday still converts `1/2`, `1/4`, and `3/4` respectively to ½ (<code>&amp;frac12;</code>), ¼ (<code>&amp;frac14;</code>) and ¾ (<code>&amp;frac34;</code>), but only these three.</small>
`smartDashes`
: default: **`true`** <br>
Blackfriday flag: **`HTML_SMARTY_DASHES`** <br>
Purpose: `false` disables smart dashes; i.e., the conversion of multiple hyphens into an en-dash or em-dash. If `true`, its behavior can be modified with the `latexDashes` flag below.
`latexDashes`
: default: **`true`** <br>
Blackfriday flag: **`HTML_SMARTYPANTS_LATEX_DASHES`** <br>
Purpose: `false` disables LaTeX-style smart dashes and selects conventional smart dashes. Assuming `smartDashes`: <br>
If `true`, `--` is translated into &ndash; (`&ndash;`), whereas `---` is translated into &mdash; (`&mdash;`). <br>
However, *spaced* single hyphen between two words is translated into an en&nbsp;dash&mdash; e.g., "`12 June - 3 July`" becomes `12 June &ndash; 3 July` upon rendering.
`hrefTargetBlank`
: default: **`false`** <br>
Blackfriday flag: **`HTML_HREF_TARGET_BLANK`** <br>
Purpose: `true` opens <s>external links</s> **absolute** links in a new window or tab. While the `target="_blank"` attribute is typically used for external links, Blackfriday does that for _all_ absolute links ([ref](https://discourse.gohugo.io/t/internal-links-in-same-tab-external-links-in-new-tab/11048/8)). One needs to make note of this if they use absolute links throughout, for internal links too (for example, by setting `canonifyURLs` to `true` or via `absURL`).
`nofollowLinks`
: default: **`false`** <br>
Blackfriday flag: **`HTML_NOFOLLOW_LINKS`** <br>
Purpose: `true` creates <s>external links</s> **absolute** links with `nofollow` being added to their `rel` attribute. Thereby crawlers are advised to not follow the link. While the `rel="nofollow"` attribute is typically used for external links, Blackfriday does that for _all_ absolute links. One needs to make note of this if they use absolute links throughout, for internal links too (for example, by setting `canonifyURLs` to `true` or via `absURL`).
`noreferrerLinks`
: default: **`false`** <br>
Blackfriday flag: **`HTML_NOREFERRER_LINKS`** <br>
Purpose: `true` creates <s>external links</s> **absolute** links with `noreferrer` being added to their `rel` attribute. Thus when following the link no referrer information will be leaked. While the `rel="noreferrer"` attribute is typically used for external links, Blackfriday does that for _all_ absolute links. One needs to make note of this if they use absolute links throughout, for internal links too (for example, by setting `canonifyURLs` to `true` or via `absURL`).
`plainIDAnchors`
: default **`true`** <br>
Blackfriday flag: **`FootnoteAnchorPrefix` and `HeaderIDSuffix`** <br>
Purpose: `true` renders any heading and footnote IDs without the document ID. <br>
Example: renders `#my-heading` instead of `#my-heading:bec3ed8ba720b970`
`extensions`
: default: **`[]`** <br>
Purpose: Enable one or more Blackfriday's Markdown extensions (**`EXTENSION_*`**). <br>
Example: Include `hardLineBreak` in the list to enable Blackfriday's `EXTENSION_HARD_LINE_BREAK`. <br>
*See [Blackfriday extensions](#blackfriday-extensions) section for information on all extensions.*
`extensionsmask`
: default: **`[]`** <br>
Purpose: Disable one or more of Blackfriday's Markdown extensions (**`EXTENSION_*`**). <br>
Example: Include `autoHeaderIds` as `false` in the list to disable Blackfriday's `EXTENSION_AUTO_HEADER_IDS`. <br>
*See [Blackfriday extensions](#blackfriday-extensions) section for information on all extensions.*
`skipHTML`
: default: **`false`** <br>
Blackfriday flag: **`HTML_SKIP_HTML`** <br>
Purpose: `true` causes any HTML in the markdown files to be skipped.
## Blackfriday extensions
`noIntraEmphasis`
: default: *enabled* <br>
Purpose: The "\_" character is commonly used inside words when discussing
code, so having Markdown interpret it as an emphasis command is usually the
wrong thing. When enabled, Blackfriday lets you treat all emphasis markers
as normal characters when they occur inside a word.
`tables`
: default: *enabled* <br>
Purpose: When enabled, tables can be created by drawing them in the input
using the below syntax:
Example:
Name | Age
--------|------
Bob | 27
Alice | 23
`fencedCode`
: default: *enabled* <br>
Purpose: When enabled, in addition to the normal 4-space indentation to mark
code blocks, you can explicitly mark them and supply a language (to make
syntax highlighting simple).
You can use 3 or more backticks to mark the beginning of the block, and the
same number to mark the end of the block.
Example:
```md
# Heading Level 1
Some test
## Heading Level 2
Some more test
```
`autolink`
: default: *enabled* <br>
Purpose: When enabled, URLs that have not been explicitly marked as links
will be converted into links.
`strikethrough`
: default: *enabled* <br>
Purpose: When enabled, text wrapped with two tildes will be crossed out. <br>
Example: `~~crossed-out~~`
`laxHtmlBlocks`
: default: *disabled* <br>
Purpose: When enabled, loosen up HTML block parsing rules.
`spaceHeaders`
: default: *enabled* <br>
Purpose: When enabled, be strict about prefix header rules.
`hardLineBreak`
: default: *disabled* <br>
Purpose: When enabled, newlines in the input translate into line breaks in
the output.
`tabSizeEight`
: default: *disabled* <br>
Purpose: When enabled, expand tabs to eight spaces instead of four.
`footnotes`
: default: *enabled* <br>
Purpose: When enabled, Pandoc-style footnotes will be supported. The
footnote marker in the text that will become a superscript text; the
footnote definition will be placed in a list of footnotes at the end of the
document. <br>
Example:
This is a footnote.[^1]
[^1]: the footnote text.
`noEmptyLineBeforeBlock`
: default: *disabled* <br>
Purpose: When enabled, no need to insert an empty line to start a (code,
quote, ordered list, unordered list) block.
`headerIds`
: default: *enabled* <br>
Purpose: When enabled, allow specifying header IDs with `{#id}`.
`titleblock`
: default: *disabled* <br>
Purpose: When enabled, support [Pandoc-style title blocks][1].
`autoHeaderIds`
: default: *enabled* <br>
Purpose: When enabled, auto-create the header ID's from the headline text.
`backslashLineBreak`
: default: *enabled* <br>
Purpose: When enabled, translate trailing backslashes into line breaks.
`definitionLists`
: default: *enabled* <br>
Purpose: When enabled, a simple definition list is made of a single-line
term followed by a colon and the definition for that term. <br>
Example:
Cat
: Fluffy animal everyone likes
Internet
: Vector of transmission for pictures of cats
Terms must be separated from the previous definition by a blank line.
`joinLines`
: default: *enabled* <br>
Purpose: When enabled, delete newlines and join the lines.
[1]: http://pandoc.org/MANUAL.html#extension-pandoc_title_block

4
docs/content/en/templates/shortcode-templates.md
View file

@ -299,10 +299,6 @@ The rendered output of the HTML example code block will be as follows:
</pre></div>
{{< /code >}}
{{% note %}}
The preceding shortcode makes use of a Hugo-specific template function called `highlight`, which uses [Pygments](http://pygments.org) to add syntax highlighting to the example HTML code block. See the [developer tools page on syntax highlighting](/tools/syntax-highlighting/) for more information.
{{% /note %}}
### Nested Shortcode: Image Gallery
Hugo's [`.Parent` shortcode variable][parent] returns a boolean value depending on whether the shortcode in question is called within the context of a *parent* shortcode. This provides an inheritance model for common shortcode parameters.

167
docs/data/docs.json
View file

@ -1,6 +1,13 @@
{
"chroma": {
"lexers": [
{
"Name": "ABAP",
"Aliases": [
"ABAP",
"abap"
]
},
{
"Name": "ABNF",
"Aliases": [
@ -134,6 +141,13 @@
"winbatch"
]
},
{
"Name": "BibTeX",
"Aliases": [
"bib",
"bibtex"
]
},
{
"Name": "BlitzBasic",
"Aliases": [
@ -405,7 +419,8 @@
"Aliases": [
"forth",
"frt",
"fs"
"fs",
"fth"
]
},
{
@ -579,12 +594,27 @@
"idris"
]
},
{
"Name": "Igor",
"Aliases": [
"igor",
"igorpro",
"ipf"
]
},
{
"Name": "Io",
"Aliases": [
"io"
]
},
{
"Name": "J",
"Aliases": [
"ijs",
"j"
]
},
{
"Name": "JSON",
"Aliases": [
@ -1342,6 +1372,63 @@
}
]
},
"config": {
"markup": {
"defaultMarkdownHandler": "goldmark",
"highlight": {
"style": "monokai",
"codeFences": true,
"noClasses": true,
"lineNos": false,
"lineNumbersInTable": true,
"lineNoStart": 1,
"hl_Lines": "",
"tabWidth": 4
},
"tableOfContents": {
"startLevel": 2,
"endLevel": 3
},
"goldmark": {
"renderer": {
"hardWraps": false,
"xHTML": false,
"unsafe": false
},
"parser": {
"autoHeadingID": true,
"attribute": true
},
"extensions": {
"typographer": true,
"footnote": true,
"definitionList": true,
"table": true,
"strikethrough": true,
"linkify": true,
"taskList": true
}
},
"blackFriday": {
"smartypants": true,
"smartypantsQuotesNBSP": false,
"angledQuotes": false,
"fractions": true,
"hrefTargetBlank": false,
"nofollowLinks": false,
"noreferrerLinks": false,
"smartDashes": true,
"latexDashes": true,
"taskLists": true,
"plainIDAnchors": true,
"extensions": null,
"extensionsMask": null,
"skipHTML": false,
"footnoteAnchorPrefix": "",
"footnoteReturnLinkContents": ""
}
}
},
"media": {
"types": [
{
@ -1513,6 +1600,68 @@
"suffixes": [
"scss"
]
},
{
"type": "video/3gpp",
"string": "video/3gpp",
"mainType": "video",
"subType": "3gpp",
"delimiter": ".",
"suffixes": [
"3gpp",
"3gp"
]
},
{
"type": "video/mp4",
"string": "video/mp4",
"mainType": "video",
"subType": "mp4",
"delimiter": ".",
"suffixes": [
"mp4"
]
},
{
"type": "video/mpeg",
"string": "video/mpeg",
"mainType": "video",
"subType": "mpeg",
"delimiter": ".",
"suffixes": [
"mpg",
"mpeg"
]
},
{
"type": "video/ogg",
"string": "video/ogg",
"mainType": "video",
"subType": "ogg",
"delimiter": ".",
"suffixes": [
"ogv"
]
},
{
"type": "video/webm",
"string": "video/webm",
"mainType": "video",
"subType": "webm",
"delimiter": ".",
"suffixes": [
"webm"
]
},
{
"type": "video/x-msvideo",
"string": "video/x-msvideo",
"mainType": "video",
"subType": "x-msvideo",
"delimiter": ".",
"suffixes": [
"avi"
]
}
]
},
@ -2290,10 +2439,10 @@
]
},
"Eq": {
"Description": "Eq returns the boolean truth of arg1 == arg2.",
"Description": "Eq returns the boolean truth of arg1 == arg2 || arg1 == arg3 || arg1 == arg4.",
"Args": [
"x",
"y"
"first",
"others"
],
"Aliases": [
"eq"
@ -2454,7 +2603,7 @@
]
},
"Dictionary": {
"Description": "Dictionary creates a map[string]interface{} from the given parameters by\nwalking the parameters and treating them as key-value pairs. The number\nof parameters must be even.",
"Description": "Dictionary creates a map[string]interface{} from the given parameters by\nwalking the parameters and treating them as key-value pairs. The number\nof parameters must be even.\nThe keys can be string slices, which will create the needed nested structure.",
"Args": [
"values"
],
@ -2521,7 +2670,7 @@
"Description": "Index returns the result of indexing its first argument by the following\narguments. Thus \"index x 1 2 3\" is, in Go syntax, x[1][2][3]. Each\nindexed item must be a map, slice, or array.\n\nCopied from Go stdlib src/text/template/funcs.go.\n\nWe deviate from the stdlib due to https://github.com/golang/go/issues/14751.\n\nTODO(moorereason): merge upstream changes.",
"Args": [
"item",
"indices"
"args"
],
"Aliases": [
"index"
@ -2630,6 +2779,12 @@
]
]
},
"Reverse": {
"Description": "",
"Args": null,
"Aliases": null,
"Examples": null
},
"Seq": {
"Description": "Seq creates a sequence of integers. It's named and used as GNU's seq.\n\nExamples:\n 3 =\u003e 1, 2, 3\n 1 2 4 =\u003e 1, 3\n -3 =\u003e -1, -2, -3\n 1 4 =\u003e 1, 2, 3, 4\n 1 -2 =\u003e 1, 0, -1, -2",
"Args": [

34
docs/layouts/shortcodes/code-toggle.html Normal file
View file

@ -0,0 +1,34 @@
{{ $file := .Get "file" }}
{{ $code := "" }}
{{ with .Get "config" }}
{{ $file = $file | default "config" }}
{{ $sections := (split . ".") }}
{{ $configSection := index $.Site.Data.docs.config $sections }}
{{ $code = dict $sections $configSection }}
{{ else }}
{{ $code = $.Inner }}
{{ end }}
{{ $langs := (slice "yaml" "toml" "json") }}
<div class="code relative" {{ with $file }}id="{{ . | urlize}}"{{ end }}>
<div class="code-nav flex flex-nowrap items-stretch">
{{- with $file -}}
<div class="san-serif f6 dib lh-solid pl2 pv2 mr2">{{ . }}.</div>
{{- end -}}
{{ range $langs }}
<button data-toggle-tab="{{ . }}" class="tab-button {{ cond (eq . "yaml") "active" ""}} ba san-serif f6 dib lh-solid ph2 pv2">{{ . }}</button>&nbsp;
{{ end }}
</div>
<div class="tab-content">
{{ range $langs }}
<div data-pane="{{ . }}" class="code-copy-content nt3 tab-pane {{ cond (eq . "yaml") "active" ""}}">
{{ highlight ($code | transform.Remarshal . | safeHTML) . ""}}
</div>
{{ if ne ($.Get "copy") "false" }}
<button class="needs-js copy copy-toggle bg-accent-color-dark f6 absolute top-0 right-0 lh-solid hover-bg-primary-color-dark bn white ph3 pv2" title="Copy this code to your clipboard." data-clipboard-action="copy" aria-label="copy button">
</button>
{{/* Functionality located within filesaver.js The copy here is located in the css with .copy class so it can be replaced with JS on success */}}
{{end}}
{{ end }}
</div>
</div>

8
docs/layouts/shortcodes/new-in.html Normal file
View file

@ -0,0 +1,8 @@
{{ $version := .Get 0 }}
{{ if not $version }}
{{ errorf "Missing version in new-in shortcode "}}
{{ end }}
{{ $version = $version | strings.TrimPrefix "v" }}
<button class="bg-white hover:bg-gray-100 text-gray-800 font-semibold py-2 mr2 ml2 px-4 border border-gray-400 rounded shadow">
<a href="{{ printf "https://gohugo.io/news/%s-relnotes/" $version }}" target="_blank">New in v{{$version}}</a>
</button>

1
docs/requirements.txt
View file

@ -1 +0,0 @@
Pygments==2.1.3

3
go.mod
View file

@ -5,7 +5,7 @@ require (
github.com/BurntSushi/toml v0.3.1
github.com/PuerkitoBio/purell v1.1.0
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
github.com/alecthomas/chroma v0.6.9
github.com/alecthomas/chroma v0.6.10-0.20191121231300-5921c52787e3
github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1 // indirect
github.com/armon/go-radix v1.0.0
github.com/aws/aws-sdk-go v1.19.40
@ -54,6 +54,7 @@ require (
github.com/spf13/viper v1.4.0
github.com/tdewolff/minify/v2 v2.5.2
github.com/yosssi/ace v0.0.5
github.com/yuin/goldmark v1.1.4
go.opencensus.io v0.22.0 // indirect
gocloud.dev v0.15.0
golang.org/x/image v0.0.0-20190523035834-f03afa92d3ff

8
go.sum
View file

@ -40,8 +40,8 @@ github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMx
github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c=
github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38 h1:smF2tmSOzy2Mm+0dGI2AIUHY+w0BUc+4tn40djz7+6U=
github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38/go.mod h1:r7bzyVFMNntcxPZXK3/+KdruV1H5KSlyVY0gc+NgInI=
github.com/alecthomas/chroma v0.6.9 h1:afiCdwnNPo6fcyvoqqsXs78t7NbR9TuW4wDB7NJkcag=
github.com/alecthomas/chroma v0.6.9/go.mod h1:1U/PfCsTALWWYHDnsIQkxEBM0+6LLe0v8+RSVMOwxeY=
github.com/alecthomas/chroma v0.6.10-0.20191121231300-5921c52787e3 h1:7QpewsuR2wMuqAR9SADf+Cb043SlnmlA7LtDJGDx7UQ=
github.com/alecthomas/chroma v0.6.10-0.20191121231300-5921c52787e3/go.mod h1:1U/PfCsTALWWYHDnsIQkxEBM0+6LLe0v8+RSVMOwxeY=
github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721 h1:JHZL0hZKJ1VENNfmXvHbgYlbUOvpzYzvy2aZU5gXVeo=
github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721/go.mod h1:QO9JBoKquHd+jz9nshCh40fOfO+JzsoXy8qTHF68zU0=
github.com/alecthomas/kong v0.1.17-0.20190424132513-439c674f7ae0/go.mod h1:+inYUSluD+p4L8KdviBSgzcqEjUQOfC5fQDRFuc36lI=
@ -116,8 +116,6 @@ github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8
github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
github.com/frankban/quicktest v1.4.1 h1:Wv2VwvNn73pAdFIVUQRXYDFp31lXKbqblIXo/Q5GPSg=
github.com/frankban/quicktest v1.4.1/go.mod h1:36zfPVQyHxymz4cH7wlDmVwDrJuljRB60qkgn7rorfQ=
github.com/frankban/quicktest v1.5.0 h1:Tb4jWdSpdjKzTUicPnY61PZxKbDoGa7ABbrReT3gQVY=
github.com/frankban/quicktest v1.5.0/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o=
github.com/frankban/quicktest v1.6.0 h1:Cd62nl66vQsx8Uv1t8M0eICyxIwZG7MxiAOrdnnUSW0=
github.com/frankban/quicktest v1.6.0/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
@ -349,6 +347,8 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
github.com/yosssi/ace v0.0.5 h1:tUkIP/BLdKqrlrPwcmH0shwEEhTRHoGnc1wFIWmaBUA=
github.com/yosssi/ace v0.0.5/go.mod h1:ALfIzm2vT7t5ZE7uoIZqF3TQ7SAOyupFZnkrF5id+K0=
github.com/yuin/goldmark v1.1.4 h1:Fj9vOhXMWRBITkIfa8OG/5j6PTKPkyPHxZbT1bvmjV8=
github.com/yuin/goldmark v1.1.4/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.mongodb.org/mongo-driver v1.0.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
go.opencensus.io v0.15.0/go.mod h1:UffZAU+4sDEINUGP/B7UfBBkq4fqLu9zXAX7ke6CHW0=

45
helpers/content.go
View file

@ -32,7 +32,6 @@ import (
bp "github.com/gohugoio/hugo/bufferpool"
"github.com/gohugoio/hugo/config"
"github.com/spf13/afero"
jww "github.com/spf13/jwalterweatherman"
"strings"
)
@ -58,9 +57,6 @@ type ContentSpec struct {
BuildExpired bool
BuildDrafts bool
Highlight func(code, lang, optsStr string) (string, error)
defatultPygmentsOpts map[string]string
Cfg config.Provider
}
@ -77,36 +73,10 @@ func NewContentSpec(cfg config.Provider, logger *loggers.Logger, contentFs afero
Cfg: cfg,
}
// Highlighting setup
options, err := parseDefaultPygmentsOpts(cfg)
if err != nil {
return nil, err
}
spec.defatultPygmentsOpts = options
// Use the Pygmentize on path if present
useClassic := false
h := newHiglighters(spec)
if cfg.GetBool("pygmentsUseClassic") {
if !hasPygments() {
jww.WARN.Println("Highlighting with pygmentsUseClassic set requires Pygments to be installed and in the path")
} else {
useClassic = true
}
}
if useClassic {
spec.Highlight = h.pygmentsHighlight
} else {
spec.Highlight = h.chromaHighlight
}
converterProvider, err := markup.NewConverterProvider(converter.ProviderConfig{
Cfg: cfg,
ContentFs: contentFs,
Logger: logger,
Highlight: spec.Highlight,
})
if err != nil {
return nil, err
@ -220,6 +190,21 @@ func (c *ContentSpec) RenderMarkdown(src []byte) ([]byte, error) {
return b.Bytes(), nil
}
func (c *ContentSpec) ResolveMarkup(in string) string {
in = strings.ToLower(in)
switch in {
case "md", "markdown", "mdown":
return "markdown"
case "html", "htm":
return "html"
default:
if conv := c.Converters.Get(in); conv != nil {
return conv.Name()
}
}
return ""
}
// TotalWords counts instance of one or more consecutive white space
// characters, as defined by unicode.IsSpace, in s.
// This is a cheaper way of word counting than the obvious len(strings.Fields(s)).

27
helpers/general.go
View file

@ -44,11 +44,6 @@ import (
// FilePathSeparator as defined by os.Separator.
const FilePathSeparator = string(filepath.Separator)
// Strips carriage returns from third-party / external processes (useful for Windows)
func normalizeExternalHelperLineFeeds(content []byte) []byte {
return bytes.Replace(content, []byte("\r"), []byte(""), -1)
}
// FindAvailablePort returns an available and valid TCP port.
func FindAvailablePort() (*net.TCPAddr, error) {
l, err := net.Listen("tcp", ":0")
@ -74,28 +69,6 @@ func InStringArray(arr []string, el string) bool {
return false
}
// GuessType attempts to guess the type of file from a given string.
func GuessType(in string) string {
switch strings.ToLower(in) {
case "md", "markdown", "mdown":
return "markdown"
case "asciidoc", "adoc", "ad":
return "asciidoc"
case "mmark":
return "mmark"
case "rst":
return "rst"
case "pandoc", "pdc":
return "pandoc"
case "html", "htm":
return "html"
case "org":
return "org"
}
return ""
}
// FirstUpper returns a string with the first character as upper case.
func FirstUpper(s string) string {
if s == "" {

13
helpers/general_test.go
View file

@ -19,11 +19,20 @@ import (
"strings"
"testing"
"github.com/spf13/viper"
"github.com/gohugoio/hugo/common/loggers"
qt "github.com/frankban/quicktest"
"github.com/spf13/afero"
)
func TestGuessType(t *testing.T) {
func TestResolveMarkup(t *testing.T) {
c := qt.New(t)
cfg := viper.New()
spec, err := NewContentSpec(cfg, loggers.NewErrorLogger(), afero.NewMemMapFs())
c.Assert(err, qt.IsNil)
for i, this := range []struct {
in string
expect string
@ -43,7 +52,7 @@ func TestGuessType(t *testing.T) {
{"org", "org"},
{"excel", ""},
} {
result := GuessType(this.in)
result := spec.ResolveMarkup(this.in)
if result != this.expect {
t.Errorf("[%d] got %s but expected %s", i, result, this.expect)
}

402
helpers/pygments.go
View file

@ -1,402 +0,0 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package helpers
import (
"bytes"
"crypto/sha1"
"fmt"
"io"
"io/ioutil"
"os/exec"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"github.com/alecthomas/chroma"
"github.com/alecthomas/chroma/formatters"
"github.com/alecthomas/chroma/formatters/html"
"github.com/alecthomas/chroma/lexers"
"github.com/alecthomas/chroma/styles"
bp "github.com/gohugoio/hugo/bufferpool"
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/hugofs"
jww "github.com/spf13/jwalterweatherman"
)
const pygmentsBin = "pygmentize"
// hasPygments checks to see if Pygments is installed and available
// on the system.
func hasPygments() bool {
if _, err := exec.LookPath(pygmentsBin); err != nil {
return false
}
return true
}
type highlighters struct {
cs *ContentSpec
ignoreCache bool
cacheDir string
}
func newHiglighters(cs *ContentSpec) highlighters {
return highlighters{cs: cs, ignoreCache: cs.Cfg.GetBool("ignoreCache"), cacheDir: cs.Cfg.GetString("cacheDir")}
}
func (h highlighters) chromaHighlight(code, lang, optsStr string) (string, error) {
opts, err := h.cs.parsePygmentsOpts(optsStr)
if err != nil {
jww.ERROR.Print(err.Error())
return code, err
}
style, found := opts["style"]
if !found || style == "" {
style = "friendly"
}
f, err := h.cs.chromaFormatterFromOptions(opts)
if err != nil {
jww.ERROR.Print(err.Error())
return code, err
}
b := bp.GetBuffer()
defer bp.PutBuffer(b)
err = chromaHighlight(b, code, lang, style, f)
if err != nil {
jww.ERROR.Printf("Highlight failed: %s\nLang: %q\nCode: \n%s", err, lang, code)
return code, err
}
return h.injectCodeTag(`<div class="highlight">`+b.String()+"</div>", lang), nil
}
func (h highlighters) pygmentsHighlight(code, lang, optsStr string) (string, error) {
options, err := h.cs.createPygmentsOptionsString(optsStr)
if err != nil {
jww.ERROR.Print(err.Error())
return code, nil
}
// Try to read from cache first
hash := sha1.New()
io.WriteString(hash, code)
io.WriteString(hash, lang)
io.WriteString(hash, options)
fs := hugofs.Os
var cachefile string
if !h.ignoreCache && h.cacheDir != "" {
cachefile = filepath.Join(h.cacheDir, fmt.Sprintf("pygments-%x", hash.Sum(nil)))
exists, err := Exists(cachefile, fs)
if err != nil {
jww.ERROR.Print(err.Error())
return code, nil
}
if exists {
f, err := fs.Open(cachefile)
if err != nil {
jww.ERROR.Print(err.Error())
return code, nil
}
s, err := ioutil.ReadAll(f)
if err != nil {
jww.ERROR.Print(err.Error())
return code, nil
}
return string(s), nil
}
}
// No cache file, render and cache it
var out bytes.Buffer
var stderr bytes.Buffer
var langOpt string
if lang == "" {
langOpt = "-g" // Try guessing the language
} else {
langOpt = "-l" + lang
}
cmd := exec.Command(pygmentsBin, langOpt, "-fhtml", "-O", options)
cmd.Stdin = strings.NewReader(code)
cmd.Stdout = &out
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
jww.ERROR.Print(stderr.String())
return code, err
}
str := string(normalizeExternalHelperLineFeeds(out.Bytes()))
str = h.injectCodeTag(str, lang)
if !h.ignoreCache && cachefile != "" {
// Write cache file
if err := WriteToDisk(cachefile, strings.NewReader(str), fs); err != nil {
jww.ERROR.Print(stderr.String())
}
}
return str, nil
}
var preRe = regexp.MustCompile(`(?s)(.*?<pre.*?>)(.*?)(</pre>)`)
func (h highlighters) injectCodeTag(code, lang string) string {
if lang == "" {
return code
}
codeTag := fmt.Sprintf(`<code class="language-%s" data-lang="%s">`, lang, lang)
return preRe.ReplaceAllString(code, fmt.Sprintf("$1%s$2</code>$3", codeTag))
}
func chromaHighlight(w io.Writer, source, lexer, style string, f chroma.Formatter) error {
l := lexers.Get(lexer)
if l == nil {
l = lexers.Analyse(source)
}
if l == nil {
l = lexers.Fallback
}
l = chroma.Coalesce(l)
if f == nil {
f = formatters.Fallback
}
s := styles.Get(style)
if s == nil {
s = styles.Fallback
}
it, err := l.Tokenise(nil, source)
if err != nil {
return err
}
return f.Format(w, s, it)
}
var pygmentsKeywords = make(map[string]bool)
func init() {
pygmentsKeywords["encoding"] = true
pygmentsKeywords["outencoding"] = true
pygmentsKeywords["nowrap"] = true
pygmentsKeywords["full"] = true
pygmentsKeywords["title"] = true
pygmentsKeywords["style"] = true
pygmentsKeywords["noclasses"] = true
pygmentsKeywords["classprefix"] = true
pygmentsKeywords["cssclass"] = true
pygmentsKeywords["cssstyles"] = true
pygmentsKeywords["prestyles"] = true
pygmentsKeywords["linenos"] = true
pygmentsKeywords["hl_lines"] = true
pygmentsKeywords["linenostart"] = true
pygmentsKeywords["linenostep"] = true
pygmentsKeywords["linenospecial"] = true
pygmentsKeywords["nobackground"] = true
pygmentsKeywords["lineseparator"] = true
pygmentsKeywords["lineanchors"] = true
pygmentsKeywords["linespans"] = true
pygmentsKeywords["anchorlinenos"] = true
pygmentsKeywords["startinline"] = true
}
func parseOptions(defaults map[string]string, in string) (map[string]string, error) {
in = strings.Trim(in, " ")
opts := make(map[string]string)
for k, v := range defaults {
opts[k] = v
}
if in == "" {
return opts, nil
}
for _, v := range strings.Split(in, ",") {
keyVal := strings.Split(v, "=")
key := strings.ToLower(strings.Trim(keyVal[0], " "))
if len(keyVal) != 2 || !pygmentsKeywords[key] {
return opts, fmt.Errorf("invalid Pygments option: %s", key)
}
opts[key] = keyVal[1]
}
return opts, nil
}
func createOptionsString(options map[string]string) string {
var keys []string
for k := range options {
keys = append(keys, k)
}
sort.Strings(keys)
var optionsStr string
for i, k := range keys {
optionsStr += fmt.Sprintf("%s=%s", k, options[k])
if i < len(options)-1 {
optionsStr += ","
}
}
return optionsStr
}
func parseDefaultPygmentsOpts(cfg config.Provider) (map[string]string, error) {
options, err := parseOptions(nil, cfg.GetString("pygmentsOptions"))
if err != nil {
return nil, err
}
if cfg.IsSet("pygmentsStyle") {
options["style"] = cfg.GetString("pygmentsStyle")
}
if cfg.IsSet("pygmentsUseClasses") {
if cfg.GetBool("pygmentsUseClasses") {
options["noclasses"] = "false"
} else {
options["noclasses"] = "true"
}
}
if _, ok := options["encoding"]; !ok {
options["encoding"] = "utf8"
}
return options, nil
}
func (cs *ContentSpec) chromaFormatterFromOptions(pygmentsOpts map[string]string) (chroma.Formatter, error) {
var options = []html.Option{html.TabWidth(4)}
if pygmentsOpts["noclasses"] == "false" {
options = append(options, html.WithClasses())
}
lineNumbers := pygmentsOpts["linenos"]
if lineNumbers != "" {
options = append(options, html.WithLineNumbers())
if lineNumbers != "inline" {
options = append(options, html.LineNumbersInTable())
}
}
startLineStr := pygmentsOpts["linenostart"]
var startLine = 1
if startLineStr != "" {
line, err := strconv.Atoi(strings.TrimSpace(startLineStr))
if err == nil {
startLine = line
options = append(options, html.BaseLineNumber(startLine))
}
}
hlLines := pygmentsOpts["hl_lines"]
if hlLines != "" {
ranges, err := hlLinesToRanges(startLine, hlLines)
if err == nil {
options = append(options, html.HighlightLines(ranges))
}
}
return html.New(options...), nil
}
func (cs *ContentSpec) parsePygmentsOpts(in string) (map[string]string, error) {
opts, err := parseOptions(cs.defatultPygmentsOpts, in)
if err != nil {
return nil, err
}
return opts, nil
}
func (cs *ContentSpec) createPygmentsOptionsString(in string) (string, error) {
opts, err := cs.parsePygmentsOpts(in)
if err != nil {
return "", err
}
return createOptionsString(opts), nil
}
// startLine compansates for https://github.com/alecthomas/chroma/issues/30
func hlLinesToRanges(startLine int, s string) ([][2]int, error) {
var ranges [][2]int
s = strings.TrimSpace(s)
if s == "" {
return ranges, nil
}
// Variants:
// 1 2 3 4
// 1-2 3-4
// 1-2 3
// 1 3-4
// 1 3-4
fields := strings.Split(s, " ")
for _, field := range fields {
field = strings.TrimSpace(field)
if field == "" {
continue
}
numbers := strings.Split(field, "-")
var r [2]int
first, err := strconv.Atoi(numbers[0])
if err != nil {
return ranges, err
}
first = first + startLine - 1
r[0] = first
if len(numbers) > 1 {
second, err := strconv.Atoi(numbers[1])
if err != nil {
return ranges, err
}
second = second + startLine - 1
r[1] = second
} else {
r[1] = first
}
ranges = append(ranges, r)
}
return ranges, nil
}

300
helpers/pygments_test.go
View file

@ -1,300 +0,0 @@
// Copyright 2015 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package helpers
import (
"fmt"
"reflect"
"testing"
"github.com/alecthomas/chroma/formatters/html"
qt "github.com/frankban/quicktest"
"github.com/spf13/viper"
)
func TestParsePygmentsArgs(t *testing.T) {
c := qt.New(t)
for i, this := range []struct {
in string
pygmentsStyle string
pygmentsUseClasses bool
expect1 interface{}
}{
{"", "foo", true, "encoding=utf8,noclasses=false,style=foo"},
{"style=boo,noclasses=true", "foo", true, "encoding=utf8,noclasses=true,style=boo"},
{"Style=boo, noClasses=true", "foo", true, "encoding=utf8,noclasses=true,style=boo"},
{"noclasses=true", "foo", true, "encoding=utf8,noclasses=true,style=foo"},
{"style=boo", "foo", true, "encoding=utf8,noclasses=false,style=boo"},
{"boo=invalid", "foo", false, false},
{"style", "foo", false, false},
} {
v := viper.New()
v.Set("pygmentsStyle", this.pygmentsStyle)
v.Set("pygmentsUseClasses", this.pygmentsUseClasses)
spec, err := NewContentSpec(v, nil, nil)
c.Assert(err, qt.IsNil)
result1, err := spec.createPygmentsOptionsString(this.in)
if b, ok := this.expect1.(bool); ok && !b {
if err == nil {
t.Errorf("[%d] parsePygmentArgs didn't return an expected error", i)
}
} else {
if err != nil {
t.Errorf("[%d] parsePygmentArgs failed: %s", i, err)
continue
}
if result1 != this.expect1 {
t.Errorf("[%d] parsePygmentArgs got %v but expected %v", i, result1, this.expect1)
}
}
}
}
func TestParseDefaultPygmentsArgs(t *testing.T) {
c := qt.New(t)
expect := "encoding=utf8,noclasses=false,style=foo"
for i, this := range []struct {
in string
pygmentsStyle interface{}
pygmentsUseClasses interface{}
pygmentsOptions string
}{
{"", "foo", true, "style=override,noclasses=override"},
{"", nil, nil, "style=foo,noclasses=false"},
{"style=foo,noclasses=false", nil, nil, "style=override,noclasses=override"},
{"style=foo,noclasses=false", "override", false, "style=override,noclasses=override"},
} {
v := viper.New()
v.Set("pygmentsOptions", this.pygmentsOptions)
if s, ok := this.pygmentsStyle.(string); ok {
v.Set("pygmentsStyle", s)
}
if b, ok := this.pygmentsUseClasses.(bool); ok {
v.Set("pygmentsUseClasses", b)
}
spec, err := NewContentSpec(v, nil, nil)
c.Assert(err, qt.IsNil)
result, err := spec.createPygmentsOptionsString(this.in)
if err != nil {
t.Errorf("[%d] parsePygmentArgs failed: %s", i, err)
continue
}
if result != expect {
t.Errorf("[%d] parsePygmentArgs got %v but expected %v", i, result, expect)
}
}
}
type chromaInfo struct {
classes bool
lineNumbers bool
lineNumbersInTable bool
highlightRangesLen int
highlightRangesStr string
baseLineNumber int
}
func formatterChromaInfo(f *html.Formatter) chromaInfo {
v := reflect.ValueOf(f).Elem()
c := chromaInfo{}
// Hack:
c.classes = f.Classes
c.lineNumbers = v.FieldByName("lineNumbers").Bool()
c.lineNumbersInTable = v.FieldByName("lineNumbersInTable").Bool()
c.baseLineNumber = int(v.FieldByName("baseLineNumber").Int())
vv := v.FieldByName("highlightRanges")
c.highlightRangesLen = vv.Len()
c.highlightRangesStr = fmt.Sprint(vv)
return c
}
func TestChromaHTMLHighlight(t *testing.T) {
c := qt.New(t)
v := viper.New()
v.Set("pygmentsUseClasses", true)
spec, err := NewContentSpec(v, nil, nil)
c.Assert(err, qt.IsNil)
result, err := spec.Highlight(`echo "Hello"`, "bash", "")
c.Assert(err, qt.IsNil)
c.Assert(result, qt.Contains, `<div class="highlight"><pre class="chroma"><code class="language-bash" data-lang="bash"><span class="nb">echo</span> <span class="s2">&#34;Hello&#34;</span></code></pre></div>`)
}
func TestChromaHTMLFormatterFromOptions(t *testing.T) {
c := qt.New(t)
for i, this := range []struct {
in string
pygmentsStyle interface{}
pygmentsUseClasses interface{}
pygmentsOptions string
assert func(c chromaInfo)
}{
{"", "monokai", true, "style=manni,noclasses=true", func(ci chromaInfo) {
c.Assert(ci.classes, qt.Equals, true)
c.Assert(ci.lineNumbers, qt.Equals, false)
c.Assert(ci.highlightRangesLen, qt.Equals, 0)
}},
{"", nil, nil, "style=monokai,noclasses=false", func(ci chromaInfo) {
c.Assert(ci.classes, qt.Equals, true)
}},
{"linenos=sure,hl_lines=1 2 3", nil, nil, "style=monokai,noclasses=false", func(ci chromaInfo) {
c.Assert(ci.classes, qt.Equals, true)
c.Assert(ci.lineNumbers, qt.Equals, true)
c.Assert(ci.highlightRangesLen, qt.Equals, 3)
c.Assert(ci.highlightRangesStr, qt.Equals, "[[1 1] [2 2] [3 3]]")
c.Assert(ci.baseLineNumber, qt.Equals, 1)
}},
{"linenos=inline,hl_lines=1,linenostart=4", nil, nil, "style=monokai,noclasses=false", func(ci chromaInfo) {
c.Assert(ci.classes, qt.Equals, true)
c.Assert(ci.lineNumbers, qt.Equals, true)
c.Assert(ci.lineNumbersInTable, qt.Equals, false)
c.Assert(ci.highlightRangesLen, qt.Equals, 1)
// This compansates for https://github.com/alecthomas/chroma/issues/30
c.Assert(ci.highlightRangesStr, qt.Equals, "[[4 4]]")
c.Assert(ci.baseLineNumber, qt.Equals, 4)
}},
{"linenos=table", nil, nil, "style=monokai", func(ci chromaInfo) {
c.Assert(ci.lineNumbers, qt.Equals, true)
c.Assert(ci.lineNumbersInTable, qt.Equals, true)
}},
{"style=monokai,noclasses=false", nil, nil, "style=manni,noclasses=true", func(ci chromaInfo) {
c.Assert(ci.classes, qt.Equals, true)
}},
{"style=monokai,noclasses=true", "friendly", false, "style=manni,noclasses=false", func(ci chromaInfo) {
c.Assert(ci.classes, qt.Equals, false)
}},
} {
v := viper.New()
v.Set("pygmentsOptions", this.pygmentsOptions)
if s, ok := this.pygmentsStyle.(string); ok {
v.Set("pygmentsStyle", s)
}
if b, ok := this.pygmentsUseClasses.(bool); ok {
v.Set("pygmentsUseClasses", b)
}
spec, err := NewContentSpec(v, nil, nil)
c.Assert(err, qt.IsNil)
opts, err := spec.parsePygmentsOpts(this.in)
if err != nil {
t.Fatalf("[%d] parsePygmentsOpts failed: %s", i, err)
}
chromaFormatter, err := spec.chromaFormatterFromOptions(opts)
if err != nil {
t.Fatalf("[%d] chromaFormatterFromOptions failed: %s", i, err)
}
this.assert(formatterChromaInfo(chromaFormatter.(*html.Formatter)))
}
}
func TestHlLinesToRanges(t *testing.T) {
var zero [][2]int
for _, this := range []struct {
in string
startLine int
expected interface{}
}{
{"", 1, zero},
{"1 4", 1, [][2]int{{1, 1}, {4, 4}}},
{"1 4", 2, [][2]int{{2, 2}, {5, 5}}},
{"1-4 5-8", 1, [][2]int{{1, 4}, {5, 8}}},
{" 1 4 ", 1, [][2]int{{1, 1}, {4, 4}}},
{"1-4 5-8 ", 1, [][2]int{{1, 4}, {5, 8}}},
{"1-4 5", 1, [][2]int{{1, 4}, {5, 5}}},
{"4 5-9", 1, [][2]int{{4, 4}, {5, 9}}},
{" 1 -4 5 - 8 ", 1, true},
{"a b", 1, true},
} {
got, err := hlLinesToRanges(this.startLine, this.in)
if expectErr, ok := this.expected.(bool); ok && expectErr {
if err == nil {
t.Fatal("No error")
}
} else if err != nil {
t.Fatalf("Got error: %s", err)
} else if !reflect.DeepEqual(this.expected, got) {
t.Fatalf("Expected\n%v but got\n%v", this.expected, got)
}
}
}
func BenchmarkChromaHighlight(b *testing.B) {
c := qt.New(b)
v := viper.New()
v.Set("pygmentsstyle", "trac")
v.Set("pygmentsuseclasses", false)
v.Set("pygmentsuseclassic", false)
code := `// GetTitleFunc returns a func that can be used to transform a string to
// title case.
//
// The supported styles are
//
// - "Go" (strings.Title)
// - "AP" (see https://www.apstylebook.com/)
// - "Chicago" (see http://www.chicagomanualofstyle.org/home.html)
//
// If an unknown or empty style is provided, AP style is what you get.
func GetTitleFunc(style string) func(s string) string {
switch strings.ToLower(style) {
case "go":
return strings.Title
case "chicago":
tc := transform.NewTitleConverter(transform.ChicagoStyle)
return tc.Title
default:
tc := transform.NewTitleConverter(transform.APStyle)
return tc.Title
}
}
`
spec, err := NewContentSpec(v, nil, nil)
c.Assert(err, qt.IsNil)
for i := 0; i < b.N; i++ {
_, err := spec.Highlight(code, "go", "linenos=inline,hl_lines=8 15-17")
if err != nil {
b.Fatal(err)
}
}
}

1
hugolib/case_insensitive_test.go
View file

@ -208,7 +208,6 @@ Page2: {{ $page2.Params.ColoR }}
"Partial Site Global: green|yellow",
"Page Title: Side 1",
"Site Title: Nynorsk title",
"&laquo;Hi&raquo;", // angled quotes
"Page2: black ",
)

5
hugolib/config.go
View file

@ -597,11 +597,6 @@ func loadDefaultSettingsFor(v *viper.Viper) error {
v.SetDefault("taxonomies", map[string]string{"tag": "tags", "category": "categories"})
v.SetDefault("permalinks", make(map[string]string))
v.SetDefault("sitemap", config.Sitemap{Priority: -1, Filename: "sitemap.xml"})
v.SetDefault("pygmentsStyle", "monokai")
v.SetDefault("pygmentsUseClasses", false)
v.SetDefault("pygmentsCodeFences", false)
v.SetDefault("pygmentsUseClassic", false)
v.SetDefault("pygmentsOptions", "")
v.SetDefault("disableLiveReload", false)
v.SetDefault("pluralizeListTitles", true)
v.SetDefault("forceSyncStatic", false)

29
hugolib/hugo_sites_build_test.go
View file

@ -671,7 +671,7 @@ END
b.CreateSites().Build(BuildCfg{})
contentMatchers := []string{"<h2 id=\"another-header\">Another header</h2>", "<h2 id=\"another-header-99\">Another header</h2>", "<p>The End.</p>"}
contentMatchers := []string{"<h2 id=\"another-header\">Another header</h2>", "<h2 id=\"another-header99\">Another header</h2>", "<p>The End.</p>"}
for i := 1; i <= numPages; i++ {
if i%3 != 0 {
@ -691,13 +691,13 @@ END
checkContent(b, fmt.Sprintf("public/%s/page%d/index.json", section, i), contentMatchers...)
}
checkContent(b, "public/s1/index.html", "P: s1/_index.md\nList: 10|List Content: 8335\n\n\nL1: 500 L2: 5\n\nRender 0: View: 8335\n\nRender 1: View: 8335\n\nRender 2: View: 8335\n\nRender 3: View: 8335\n\nRender 4: View: 8335\n\nEND\n")
checkContent(b, "public/s2/index.html", "P: s2/_index.md\nList: 10|List Content: 8335", "Render 4: View: 8335\n\nEND")
checkContent(b, "public/index.html", "P: _index.md\nList: 10|List Content: 8335", "4: View: 8335\n\nEND")
checkContent(b, "public/s1/index.html", "P: s1/_index.md\nList: 10|List Content: 8033\n\n\nL1: 500 L2: 5\n\nRender 0: View: 8033\n\nRender 1: View: 8033\n\nRender 2: View: 8033\n\nRender 3: View: 8033\n\nRender 4: View: 8033\n\nEND\n")
checkContent(b, "public/s2/index.html", "P: s2/_index.md\nList: 10|List Content: 8033", "Render 4: View: 8033\n\nEND")
checkContent(b, "public/index.html", "P: _index.md\nList: 10|List Content: 8033", "4: View: 8033\n\nEND")
// Check paginated pages
for i := 2; i <= 9; i++ {
checkContent(b, fmt.Sprintf("public/page/%d/index.html", i), fmt.Sprintf("Page: %d", i), "Content: 8335\n\n\nL1: 500 L2: 5\n\nRender 0: View: 8335", "Render 4: View: 8335\n\nEND")
checkContent(b, fmt.Sprintf("public/page/%d/index.html", i), fmt.Sprintf("Page: %d", i), "Content: 8033\n\n\nL1: 500 L2: 5\n\nRender 0: View: 8033", "Render 4: View: 8033\n\nEND")
}
}
@ -977,7 +977,10 @@ enableRobotsTXT = true
[permalinks]
other = "/somewhere/else/:filename"
[blackfriday]
# TODO(bep)
[markup]
defaultMarkdownHandler = "blackfriday"
[markup.blackfriday]
angledQuotes = true
[Taxonomies]
@ -1035,7 +1038,10 @@ enableRobotsTXT: true
permalinks:
other: "/somewhere/else/:filename"
blackfriday:
# TODO(bep)
markup:
defaultMarkdownHandler: blackfriday
blackFriday:
angledQuotes: true
Taxonomies:
@ -1093,9 +1099,12 @@ var multiSiteJSONConfigTemplate = `
"permalinks": {
"other": "/somewhere/else/:filename"
},
"blackfriday": {
"angledQuotes": true
},
"markup": {
"defaultMarkdownHandler": "blackfriday",
"blackfriday": {
"angledQuotes": true
}
},
"Taxonomies": {
"tag": "tags"
},

14
hugolib/page__meta.go
View file

@ -565,7 +565,7 @@ func (pm *pageMeta) setMetadata(bucket *pagesMapBucket, p *pageState, frontmatte
pm.sitemap = p.s.siteCfg.sitemap
}
pm.markup = helpers.GuessType(pm.markup)
pm.markup = p.s.ContentSpec.ResolveMarkup(pm.markup)
if draft != nil && published != nil {
pm.draft = *draft
@ -596,7 +596,7 @@ func (p *pageMeta) applyDefaultValues() error {
if p.markup == "" {
if !p.File().IsZero() {
// Fall back to file extension
p.markup = helpers.GuessType(p.File().Ext())
p.markup = p.s.ContentSpec.ResolveMarkup(p.File().Ext())
}
if p.markup == "" {
p.markup = "markdown"
@ -638,14 +638,20 @@ func (p *pageMeta) applyDefaultValues() error {
}
}
if !p.f.IsZero() && p.markup != "html" {
if !p.f.IsZero() {
var renderingConfigOverrides map[string]interface{}
bfParam := getParamToLower(p, "blackfriday")
if bfParam != nil {
renderingConfigOverrides = maps.ToStringMap(bfParam)
}
cp := p.s.ContentSpec.Converters.Get(p.markup)
markup := p.markup
if markup == "html" {
// Only used for shortcode inner content.
markup = "markdown"
}
cp := p.s.ContentSpec.Converters.Get(markup)
if cp == nil {
return errors.Errorf("no content renderer found for markup %q", p.markup)
}

18
hugolib/page__per_output.go
View file

@ -77,7 +77,7 @@ func newPageContentOutput(p *pageState) func(f output.Format) (*pageContentOutpu
// See https://github.com/gohugoio/hugo/issues/6210
if r := recover(); r != nil {
err = fmt.Errorf("%s", r)
p.s.Log.ERROR.Println("[BUG] Got panic:\n", string(debug.Stack()))
p.s.Log.ERROR.Printf("[BUG] Got panic:\n%s\n%s", r, string(debug.Stack()))
}
}()
@ -103,11 +103,14 @@ func newPageContentOutput(p *pageState) func(f output.Format) (*pageContentOutpu
if err != nil {
return err
}
cp.convertedResult = r
cp.workContent = r.Bytes()
tmpContent, tmpTableOfContents := helpers.ExtractTOC(cp.workContent)
cp.tableOfContents = helpers.BytesToHTML(tmpTableOfContents)
cp.workContent = tmpContent
if _, ok := r.(converter.TableOfContentsProvider); !ok {
tmpContent, tmpTableOfContents := helpers.ExtractTOC(cp.workContent)
cp.tableOfContents = helpers.BytesToHTML(tmpTableOfContents)
cp.workContent = tmpContent
}
}
if cp.placeholdersEnabled {
@ -223,7 +226,8 @@ type pageContentOutput struct {
// Content state
workContent []byte
workContent []byte
convertedResult converter.Result
// Temporary storage of placeholders mapped to their content.
// These are shortcodes etc. Some of these will need to be replaced
@ -284,6 +288,10 @@ func (p *pageContentOutput) Summary() template.HTML {
func (p *pageContentOutput) TableOfContents() template.HTML {
p.p.s.initInit(p.initMain, p.p)
if tocProvider, ok := p.convertedResult.(converter.TableOfContentsProvider); ok {
cfg := p.p.s.ContentSpec.Converters.GetMarkupConfig()
return template.HTML(tocProvider.TableOfContents().ToHTML(cfg.TableOfContents.StartLevel, cfg.TableOfContents.EndLevel))
}
return p.tableOfContents
}

168
hugolib/page_test.go
View file

@ -326,7 +326,7 @@ func normalizeContent(c string) string {
func checkPageTOC(t *testing.T, page page.Page, toc string) {
if page.TableOfContents() != template.HTML(toc) {
t.Fatalf("Page TableOfContents is: %q.\nExpected %q", page.TableOfContents(), toc)
t.Fatalf("Page TableOfContents is:\n%q.\nExpected %q", page.TableOfContents(), toc)
}
}
@ -442,6 +442,7 @@ func testAllMarkdownEnginesForPages(t *testing.T,
func TestPageWithDelimiterForMarkdownThatCrossesBorder(t *testing.T) {
t.Parallel()
cfg, fs := newTestCfg()
c := qt.New(t)
writeSource(t, fs, filepath.Join("content", "simple.md"), simplePageWithSummaryDelimiterAndMarkdownThatCrossesBorder)
@ -453,12 +454,12 @@ func TestPageWithDelimiterForMarkdownThatCrossesBorder(t *testing.T) {
p := s.RegularPages()[0]
if p.Summary() != template.HTML(
"<p>The <a href=\"http://gohugo.io/\">best static site generator</a>.<sup class=\"footnote-ref\" id=\"fnref:1\"><a href=\"#fn:1\">1</a></sup></p>") {
"<p>The <a href=\"http://gohugo.io/\">best static site generator</a>.<sup id=\"fnref:1\"><a href=\"#fn:1\" class=\"footnote-ref\" role=\"doc-noteref\">1</a></sup></p>") {
t.Fatalf("Got summary:\n%q", p.Summary())
}
cnt := content(p)
if cnt != "<p>The <a href=\"http://gohugo.io/\">best static site generator</a>.<sup class=\"footnote-ref\" id=\"fnref:1\"><a href=\"#fn:1\">1</a></sup></p>\n\n<div class=\"footnotes\">\n\n<hr />\n\n<ol>\n<li id=\"fn:1\">Many people say so.\n <a class=\"footnote-return\" href=\"#fnref:1\"><sup>[return]</sup></a></li>\n</ol>\n</div>" {
if cnt != "<p>The <a href=\"http://gohugo.io/\">best static site generator</a>.<sup id=\"fnref:1\"><a href=\"#fn:1\" class=\"footnote-ref\" role=\"doc-noteref\">1</a></sup></p>\n<section class=\"footnotes\" role=\"doc-endnotes\">\n<hr>\n<ol>\n<li id=\"fn:1\" role=\"doc-endnote\">\n<p>Many people say so.</p>\n</li>\n</ol>\n</section>" {
t.Fatalf("Got content:\n%q", cnt)
}
}
@ -673,23 +674,13 @@ func TestPageWithShortCodeInSummary(t *testing.T) {
testAllMarkdownEnginesForPages(t, assertFunc, nil, simplePageWithShortcodeInSummary)
}
func TestPageWithEmbeddedScriptTag(t *testing.T) {
t.Parallel()
assertFunc := func(t *testing.T, ext string, pages page.Pages) {
p := pages[0]
if ext == "ad" || ext == "rst" {
// TOD(bep)
return
}
checkPageContent(t, p, "<script type='text/javascript'>alert('the script tags are still there, right?');</script>\n", ext)
}
testAllMarkdownEnginesForPages(t, assertFunc, nil, simplePageWithEmbeddedScript)
}
func TestPageWithAdditionalExtension(t *testing.T) {
t.Parallel()
cfg, fs := newTestCfg()
cfg.Set("markup", map[string]interface{}{
"defaultMarkdownHandler": "blackfriday", // TODO(bep)
})
c := qt.New(t)
writeSource(t, fs, filepath.Join("content", "simple.md"), simplePageWithAdditionalExtension)
@ -716,8 +707,8 @@ func TestTableOfContents(t *testing.T) {
p := s.RegularPages()[0]
checkPageContent(t, p, "\n\n<p>For some moments the old man did not reply. He stood with bowed head, buried in deep thought. But at last he spoke.</p>\n\n<h2 id=\"aa\">AA</h2>\n\n<p>I have no idea, of course, how long it took me to reach the limit of the plain,\nbut at last I entered the foothills, following a pretty little canyon upward\ntoward the mountains. Beside me frolicked a laughing brooklet, hurrying upon\nits noisy way down to the silent sea. In its quieter pools I discovered many\nsmall fish, of four-or five-pound weight I should imagine. In appearance,\nexcept as to size and color, they were not unlike the whale of our own seas. As\nI watched them playing about I discovered, not only that they suckled their\nyoung, but that at intervals they rose to the surface to breathe as well as to\nfeed upon certain grasses and a strange, scarlet lichen which grew upon the\nrocks just above the water line.</p>\n\n<h3 id=\"aaa\">AAA</h3>\n\n<p>I remember I felt an extraordinary persuasion that I was being played with,\nthat presently, when I was upon the very verge of safety, this mysterious\ndeath&ndash;as swift as the passage of light&ndash;would leap after me from the pit about\nthe cylinder and strike me down. ## BB</p>\n\n<h3 id=\"bbb\">BBB</h3>\n\n<p>&ldquo;You&rsquo;re a great Granser,&rdquo; he cried delightedly, &ldquo;always making believe them little marks mean something.&rdquo;</p>\n")
checkPageTOC(t, p, "<nav id=\"TableOfContents\">\n<ul>\n<li>\n<ul>\n<li><a href=\"#aa\">AA</a>\n<ul>\n<li><a href=\"#aaa\">AAA</a></li>\n<li><a href=\"#bbb\">BBB</a></li>\n</ul></li>\n</ul></li>\n</ul>\n</nav>")
checkPageContent(t, p, "<p>For some moments the old man did not reply. He stood with bowed head, buried in deep thought. But at last he spoke.</p><h2 id=\"aa\">AA</h2> <p>I have no idea, of course, how long it took me to reach the limit of the plain, but at last I entered the foothills, following a pretty little canyon upward toward the mountains. Beside me frolicked a laughing brooklet, hurrying upon its noisy way down to the silent sea. In its quieter pools I discovered many small fish, of four-or five-pound weight I should imagine. In appearance, except as to size and color, they were not unlike the whale of our own seas. As I watched them playing about I discovered, not only that they suckled their young, but that at intervals they rose to the surface to breathe as well as to feed upon certain grasses and a strange, scarlet lichen which grew upon the rocks just above the water line.</p><h3 id=\"aaa\">AAA</h3> <p>I remember I felt an extraordinary persuasion that I was being played with, that presently, when I was upon the very verge of safety, this mysterious death&ndash;as swift as the passage of light&ndash;would leap after me from the pit about the cylinder and strike me down. ## BB</p><h3 id=\"bbb\">BBB</h3> <p>&ldquo;You're a great Granser,&rdquo; he cried delightedly, &ldquo;always making believe them little marks mean something.&rdquo;</p>")
checkPageTOC(t, p, "<nav id=\"TableOfContents\">\n <ul>\n <li><a href=\"#aa\">AA</a>\n <ul>\n <li><a href=\"#aaa\">AAA</a></li>\n <li><a href=\"#bbb\">BBB</a></li>\n </ul>\n </li>\n </ul>\n</nav>")
}
func TestPageWithMoreTag(t *testing.T) {
@ -1518,12 +1509,12 @@ Summary: In Chinese, 好 means good.
b.AssertFileContent("public/p1/index.html", "WordCount: 510\nFuzzyWordCount: 600\nReadingTime: 3\nLen Plain: 2550\nLen PlainWords: 510\nTruncated: false\nLen Summary: 2549\nLen Content: 2557")
b.AssertFileContent("public/p2/index.html", "WordCount: 314\nFuzzyWordCount: 400\nReadingTime: 2\nLen Plain: 1569\nLen PlainWords: 314\nTruncated: true\nLen Summary: 25\nLen Content: 1583")
b.AssertFileContent("public/p2/index.html", "WordCount: 314\nFuzzyWordCount: 400\nReadingTime: 2\nLen Plain: 1569\nLen PlainWords: 314\nTruncated: true\nLen Summary: 25\nLen Content: 1582")
b.AssertFileContent("public/p3/index.html", "WordCount: 206\nFuzzyWordCount: 300\nReadingTime: 1\nLen Plain: 638\nLen PlainWords: 7\nTruncated: true\nLen Summary: 43\nLen Content: 652")
b.AssertFileContent("public/p4/index.html", "WordCount: 7\nFuzzyWordCount: 100\nReadingTime: 1\nLen Plain: 638\nLen PlainWords: 7\nTruncated: true\nLen Summary: 43\nLen Content: 652")
b.AssertFileContent("public/p5/index.html", "WordCount: 206\nFuzzyWordCount: 300\nReadingTime: 1\nLen Plain: 638\nLen PlainWords: 7\nTruncated: true\nLen Summary: 229\nLen Content: 653")
b.AssertFileContent("public/p6/index.html", "WordCount: 7\nFuzzyWordCount: 100\nReadingTime: 1\nLen Plain: 638\nLen PlainWords: 7\nTruncated: false\nLen Summary: 637\nLen Content: 653")
b.AssertFileContent("public/p3/index.html", "WordCount: 206\nFuzzyWordCount: 300\nReadingTime: 1\nLen Plain: 638\nLen PlainWords: 7\nTruncated: true\nLen Summary: 43\nLen Content: 651")
b.AssertFileContent("public/p4/index.html", "WordCount: 7\nFuzzyWordCount: 100\nReadingTime: 1\nLen Plain: 638\nLen PlainWords: 7\nTruncated: true\nLen Summary: 43\nLen Content: 651")
b.AssertFileContent("public/p5/index.html", "WordCount: 206\nFuzzyWordCount: 300\nReadingTime: 1\nLen Plain: 638\nLen PlainWords: 7\nTruncated: true\nLen Summary: 229\nLen Content: 652")
b.AssertFileContent("public/p6/index.html", "WordCount: 7\nFuzzyWordCount: 100\nReadingTime: 1\nLen Plain: 638\nLen PlainWords: 7\nTruncated: false\nLen Summary: 637\nLen Content: 652")
}
@ -1611,3 +1602,132 @@ author = "Jo Nesbø"
"Author site config: Kurt Vonnegut")
}
func TestGoldmark(t *testing.T) {
t.Parallel()
b := newTestSitesBuilder(t).WithConfigFile("toml", `
baseURL = "https://example.org"
[markup]
defaultMarkdownHandler="goldmark"
[markup.goldmark]
[markup.goldmark.renderer]
unsafe = false
[markup.highlight]
noClasses=false
`)
b.WithTemplatesAdded("_default/single.html", `
Title: {{ .Title }}
ToC: {{ .TableOfContents }}
Content: {{ .Content }}
`, "shortcodes/t.html", `T-SHORT`, "shortcodes/s.html", `## Code
{{ .Inner }}
`)
content := `
+++
title = "A Page!"
+++
## Shortcode {{% t %}} in header
## Code Fense in Shortcode
{{% s %}}
$$$bash {hl_lines=[1]}
SHORT
$$$
{{% /s %}}
## Code Fence
$$$bash {hl_lines=[1]}
MARKDOWN
$$$
`
content = strings.ReplaceAll(content, "$$$", "```")
b.WithContent("page.md", content)
b.Build(BuildCfg{})
b.AssertFileContent("public/page/index.html",
`<nav id="TableOfContents">`,
`<li><a href="#shortcode-tshort-in-header">Shortcode T-SHORT in header</a></li>`,
`<code class="language-bash" data-lang="bash"><span class="hl">SHORT`,
`<code class="language-bash" data-lang="bash"><span class="hl">MARKDOWN`)
}
func TestBlackfridayDefault(t *testing.T) {
t.Parallel()
b := newTestSitesBuilder(t).WithConfigFile("toml", `
baseURL = "https://example.org"
[markup]
defaultMarkdownHandler="blackfriday"
[markup.highlight]
noClasses=false
[markup.goldmark]
[markup.goldmark.renderer]
unsafe=true
`)
// Use the new attribute syntax to make sure it's not Goldmark.
b.WithTemplatesAdded("_default/single.html", `
Title: {{ .Title }}
Content: {{ .Content }}
`, "shortcodes/s.html", `## Code
{{ .Inner }}
`)
content := `
+++
title = "A Page!"
+++
## Code Fense in Shortcode
{{% s %}}
S:
{{% s %}}
$$$bash {hl_lines=[1]}
SHORT
$$$
{{% /s %}}
{{% /s %}}
## Code Fence
$$$bash {hl_lines=[1]}
MARKDOWN
$$$
`
content = strings.ReplaceAll(content, "$$$", "```")
for i, ext := range []string{"md", "html"} {
b.WithContent(fmt.Sprintf("page%d.%s", i+1, ext), content)
}
b.Build(BuildCfg{})
// Blackfriday does not support this extended attribute syntax.
b.AssertFileContent("public/page1/index.html",
`<pre><code class="language-bash {hl_lines=[1]}" data-lang="bash {hl_lines=[1]}">SHORT</code></pre>`,
`<pre><code class="language-bash {hl_lines=[1]}" data-lang="bash {hl_lines=[1]}">MARKDOWN`,
)
b.AssertFileContent("public/page2/index.html",
`<pre><code class="language-bash {hl_lines=[1]}" data-lang="bash {hl_lines=[1]}">SHORT`,
)
}

12
hugolib/shortcode_test.go
View file

@ -44,6 +44,11 @@ func CheckShortCodeMatch(t *testing.T, input, expected string, withTemplate func
func CheckShortCodeMatchAndError(t *testing.T, input, expected string, withTemplate func(templ tpl.TemplateHandler) error, expectError bool) {
t.Helper()
cfg, fs := newTestCfg()
cfg.Set("markup", map[string]interface{}{
"defaultMarkdownHandler": "blackfriday", // TODO(bep)
})
c := qt.New(t)
// Need some front matter, see https://github.com/gohugoio/hugo/issues/2337
@ -584,6 +589,9 @@ title: "Foo"
cfg.Set("pygmentsUseClasses", true)
cfg.Set("pygmentsCodefences", true)
cfg.Set("markup", map[string]interface{}{
"defaultMarkdownHandler": "blackfriday", // TODO(bep)
})
writeSourcesToSource(t, "content", fs, sources...)
@ -597,6 +605,7 @@ title: "Foo"
th := newTestHelper(s.Cfg, s.Fs, t)
expected := cast.ToStringSlice(test.expected)
th.assertFileContent(filepath.FromSlash(test.outFile), expected...)
})
@ -1245,6 +1254,9 @@ func TestShortcodeRef(t *testing.T) {
v.Set("blackfriday", map[string]interface{}{
"plainIDAnchors": plainIDAnchors,
})
v.Set("markup", map[string]interface{}{
"defaultMarkdownHandler": "blackfriday", // TODO(bep)
})
builder := newTestSitesBuilder(t).WithViper(v)

2
hugolib/site_benchmark_new_test.go
View file

@ -181,7 +181,7 @@ pygmentsCodeFences = true
return sb
},
func(s *sitesBuilder) {
s.AssertFileContent("public/page8/index.html", `<div class="highlight"><pre style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-bash" data-lang="bash">echo <span style="color:#e6db74">&#34;Hugo Rocks!&#34;</span></code></pre></div>`)
s.AssertFileContent("public/page8/index.html", `<div class="highlight"><pre style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-bash" data-lang="bash">echo <span style="color:#e6db74">&#34;Hugo Rocks!&#34;</span>`)
},
},
{"Deep content tree", func(b testing.TB) *sitesBuilder {

22
hugolib/site_test.go
View file

@ -231,12 +231,12 @@ THE END.`, refShortcode),
// Issue #1753: Should not add a trailing newline after shortcode.
{
filepath.FromSlash("sect/doc3.md"),
fmt.Sprintf(`**Ref 1:**{{< %s "sect/doc3.md" >}}.`, refShortcode),
fmt.Sprintf(`**Ref 1:** {{< %s "sect/doc3.md" >}}.`, refShortcode),
},
// Issue #3703
{
filepath.FromSlash("sect/doc4.md"),
fmt.Sprintf(`**Ref 1:**{{< %s "%s" >}}.`, refShortcode, doc3Slashed),
fmt.Sprintf(`**Ref 1:** {{< %s "%s" >}}.`, refShortcode, doc3Slashed),
},
}
@ -267,9 +267,9 @@ THE END.`, refShortcode),
expected string
}{
{filepath.FromSlash(fmt.Sprintf("public/sect/doc1%s", expectedPathSuffix)), fmt.Sprintf("<p>Ref 2: %s/sect/doc2%s</p>\n", expectedBase, expectedURLSuffix)},
{filepath.FromSlash(fmt.Sprintf("public/sect/doc2%s", expectedPathSuffix)), fmt.Sprintf("<p><strong>Ref 1:</strong></p>\n\n%s/sect/doc1%s\n\n<p>THE END.</p>\n", expectedBase, expectedURLSuffix)},
{filepath.FromSlash(fmt.Sprintf("public/sect/doc3%s", expectedPathSuffix)), fmt.Sprintf("<p><strong>Ref 1:</strong>%s/sect/doc3%s.</p>\n", expectedBase, expectedURLSuffix)},
{filepath.FromSlash(fmt.Sprintf("public/sect/doc4%s", expectedPathSuffix)), fmt.Sprintf("<p><strong>Ref 1:</strong>%s/sect/doc3%s.</p>\n", expectedBase, expectedURLSuffix)},
{filepath.FromSlash(fmt.Sprintf("public/sect/doc2%s", expectedPathSuffix)), fmt.Sprintf("<p><strong>Ref 1:</strong></p>\n%s/sect/doc1%s\n<p>THE END.</p>\n", expectedBase, expectedURLSuffix)},
{filepath.FromSlash(fmt.Sprintf("public/sect/doc3%s", expectedPathSuffix)), fmt.Sprintf("<p><strong>Ref 1:</strong> %s/sect/doc3%s.</p>\n", expectedBase, expectedURLSuffix)},
{filepath.FromSlash(fmt.Sprintf("public/sect/doc4%s", expectedPathSuffix)), fmt.Sprintf("<p><strong>Ref 1:</strong> %s/sect/doc3%s.</p>\n", expectedBase, expectedURLSuffix)},
}
for _, test := range tests {
@ -330,12 +330,12 @@ func doTestShouldAlwaysHaveUglyURLs(t *testing.T, uglyURLs bool) {
expected string
}{
{filepath.FromSlash("public/index.html"), "Home Sweet Home."},
{filepath.FromSlash(expectedPagePath), "\n\n<h1 id=\"title\">title</h1>\n\n<p>some <em>content</em></p>\n"},
{filepath.FromSlash(expectedPagePath), "<h1 id=\"title\">title</h1>\n<p>some <em>content</em></p>\n"},
{filepath.FromSlash("public/404.html"), "Page Not Found."},
{filepath.FromSlash("public/index.xml"), "<root>RSS</root>"},
{filepath.FromSlash("public/sitemap.xml"), "<root>SITEMAP</root>"},
// Issue #1923
{filepath.FromSlash("public/ugly.html"), "\n\n<h1 id=\"title\">title</h1>\n\n<p>doc2 <em>content</em></p>\n"},
{filepath.FromSlash("public/ugly.html"), "<h1 id=\"title\">title</h1>\n<p>doc2 <em>content</em></p>\n"},
}
for _, p := range s.RegularPages() {
@ -540,14 +540,14 @@ func TestSkipRender(t *testing.T) {
doc string
expected string
}{
{filepath.FromSlash("public/sect/doc1.html"), "\n\n<h1 id=\"title\">title</h1>\n\n<p>some <em>content</em></p>\n"},
{filepath.FromSlash("public/sect/doc1.html"), "<h1 id=\"title\">title</h1>\n<p>some <em>content</em></p>\n"},
{filepath.FromSlash("public/sect/doc2.html"), "<!doctype html><html><body>more content</body></html>"},
{filepath.FromSlash("public/sect/doc3.html"), "\n\n<h1 id=\"doc3\">doc3</h1>\n\n<p><em>some</em> content</p>\n"},
{filepath.FromSlash("public/sect/doc4.html"), "\n\n<h1 id=\"doc4\">doc4</h1>\n\n<p><em>some content</em></p>\n"},
{filepath.FromSlash("public/sect/doc3.html"), "<h1 id=\"doc3\">doc3</h1>\n<p><em>some</em> content</p>\n"},
{filepath.FromSlash("public/sect/doc4.html"), "<h1 id=\"doc4\">doc4</h1>\n<p><em>some content</em></p>\n"},
{filepath.FromSlash("public/sect/doc5.html"), "<!doctype html><html><head><script src=\"script.js\"></script></head><body>body5</body></html>"},
{filepath.FromSlash("public/sect/doc6.html"), "<!doctype html><html><head><script src=\"http://auth/bub/script.js\"></script></head><body>body5</body></html>"},
{filepath.FromSlash("public/doc7.html"), "<html><body>doc7 content</body></html>"},
{filepath.FromSlash("public/sect/doc8.html"), "\n\n<h1 id=\"title\">title</h1>\n\n<p>some <em>content</em></p>\n"},
{filepath.FromSlash("public/sect/doc8.html"), "<h1 id=\"title\">title</h1>\n<p>some <em>content</em></p>\n"},
{filepath.FromSlash("public/doc9.html"), "<html><body>doc9: SHORT</body></html>"},
}

4
hugolib/testhelpers_test.go
View file

@ -743,7 +743,7 @@ func (th testHelper) assertFileContent(filename string, matches ...string) {
content := readDestination(th, th.Fs, filename)
for _, match := range matches {
match = th.replaceDefaultContentLanguageValue(match)
th.Assert(strings.Contains(content, match), qt.Equals, true, qt.Commentf(content))
th.Assert(strings.Contains(content, match), qt.Equals, true, qt.Commentf(match+" not in: \n"+content))
}
}
@ -755,7 +755,7 @@ func (th testHelper) assertFileContentRegexp(filename string, matches ...string)
r := regexp.MustCompile(match)
matches := r.MatchString(content)
if !matches {
fmt.Println(content)
fmt.Println(match+":\n", content)
}
th.Assert(matches, qt.Equals, true)
}

7
markup/asciidoc/convert.go
View file

@ -24,19 +24,18 @@ import (
)
// Provider is the package entry point.
var Provider converter.NewProvider = provider{}
var Provider converter.ProviderProvider = provider{}
type provider struct {
}
func (p provider) New(cfg converter.ProviderConfig) (converter.Provider, error) {
var n converter.NewConverter = func(ctx converter.DocumentContext) (converter.Converter, error) {
return converter.NewProvider("asciidoc", func(ctx converter.DocumentContext) (converter.Converter, error) {
return &asciidocConverter{
ctx: ctx,
cfg: cfg,
}, nil
}
return n, nil
}), nil
}
type asciidocConverter struct {

70
markup/blackfriday/blackfriday_config/config.go Normal file
View file

@ -0,0 +1,70 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package helpers implements general utility functions that work with
// and on content. The helper functions defined here lay down the
// foundation of how Hugo works with files and filepaths, and perform
// string operations on content.
package blackfriday_config
import (
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
)
// Default holds the default BlackFriday config.
// Do not change!
var Default = Config{
Smartypants: true,
AngledQuotes: false,
SmartypantsQuotesNBSP: false,
Fractions: true,
HrefTargetBlank: false,
NofollowLinks: false,
NoreferrerLinks: false,
SmartDashes: true,
LatexDashes: true,
PlainIDAnchors: true,
TaskLists: true,
SkipHTML: false,
}
// Config holds configuration values for BlackFriday rendering.
// It is kept here because it's used in several packages.
type Config struct {
Smartypants bool
SmartypantsQuotesNBSP bool
AngledQuotes bool
Fractions bool
HrefTargetBlank bool
NofollowLinks bool
NoreferrerLinks bool
SmartDashes bool
LatexDashes bool
TaskLists bool
PlainIDAnchors bool
Extensions []string
ExtensionsMask []string
SkipHTML bool
FootnoteAnchorPrefix string
FootnoteReturnLinkContents string
}
func UpdateConfig(b Config, m map[string]interface{}) (Config, error) {
if err := mapstructure.Decode(m, &b); err != nil {
return b, errors.WithMessage(err, "failed to decode rendering config")
}
return b, nil
}

43
markup/blackfriday/convert.go
View file

@ -15,36 +15,27 @@
package blackfriday
import (
"github.com/gohugoio/hugo/markup/blackfriday/blackfriday_config"
"github.com/gohugoio/hugo/markup/converter"
"github.com/gohugoio/hugo/markup/internal"
"github.com/russross/blackfriday"
)
// Provider is the package entry point.
var Provider converter.NewProvider = provider{}
var Provider converter.ProviderProvider = provider{}
type provider struct {
}
func (p provider) New(cfg converter.ProviderConfig) (converter.Provider, error) {
defaultBlackFriday, err := internal.NewBlackfriday(cfg)
if err != nil {
return nil, err
}
defaultExtensions := getMarkdownExtensions(cfg.MarkupConfig.BlackFriday)
defaultExtensions := getMarkdownExtensions(defaultBlackFriday)
pygmentsCodeFences := cfg.Cfg.GetBool("pygmentsCodeFences")
pygmentsCodeFencesGuessSyntax := cfg.Cfg.GetBool("pygmentsCodeFencesGuessSyntax")
pygmentsOptions := cfg.Cfg.GetString("pygmentsOptions")
var n converter.NewConverter = func(ctx converter.DocumentContext) (converter.Converter, error) {
b := defaultBlackFriday
return converter.NewProvider("blackfriday", func(ctx converter.DocumentContext) (converter.Converter, error) {
b := cfg.MarkupConfig.BlackFriday
extensions := defaultExtensions
if ctx.ConfigOverrides != nil {
var err error
b, err = internal.UpdateBlackFriday(b, ctx.ConfigOverrides)
b, err = blackfriday_config.UpdateConfig(b, ctx.ConfigOverrides)
if err != nil {
return nil, err
}
@ -56,27 +47,16 @@ func (p provider) New(cfg converter.ProviderConfig) (converter.Provider, error)
bf: b,
extensions: extensions,
cfg: cfg,
pygmentsCodeFences: pygmentsCodeFences,
pygmentsCodeFencesGuessSyntax: pygmentsCodeFencesGuessSyntax,
pygmentsOptions: pygmentsOptions,
}, nil
}
return n, nil
}), nil
}
type blackfridayConverter struct {
ctx converter.DocumentContext
bf *internal.BlackFriday
bf blackfriday_config.Config
extensions int
pygmentsCodeFences bool
pygmentsCodeFencesGuessSyntax bool
pygmentsOptions string
cfg converter.ProviderConfig
cfg converter.ProviderConfig
}
func (c *blackfridayConverter) AnchorSuffix() string {
@ -90,7 +70,6 @@ func (c *blackfridayConverter) Convert(ctx converter.RenderContext) (converter.R
r := c.getHTMLRenderer(ctx.RenderTOC)
return converter.Bytes(blackfriday.Markdown(ctx.Src, r, c.extensions)), nil
}
func (c *blackfridayConverter) getHTMLRenderer(renderTOC bool) blackfriday.Renderer {
@ -114,7 +93,7 @@ func (c *blackfridayConverter) getHTMLRenderer(renderTOC bool) blackfriday.Rende
}
}
func getFlags(renderTOC bool, cfg *internal.BlackFriday) int {
func getFlags(renderTOC bool, cfg blackfriday_config.Config) int {
var flags int
@ -168,7 +147,7 @@ func getFlags(renderTOC bool, cfg *internal.BlackFriday) int {
return flags
}
func getMarkdownExtensions(cfg *internal.BlackFriday) int {
func getMarkdownExtensions(cfg blackfriday_config.Config) int {
// Default Blackfriday common extensions
commonExtensions := 0 |
blackfriday.EXTENSION_NO_INTRA_EMPHASIS |

31
markup/blackfriday/convert_test.go
View file

@ -18,19 +18,15 @@ import (
"github.com/spf13/viper"
"github.com/gohugoio/hugo/markup/internal"
"github.com/gohugoio/hugo/markup/converter"
qt "github.com/frankban/quicktest"
"github.com/gohugoio/hugo/markup/blackfriday/blackfriday_config"
"github.com/russross/blackfriday"
)
func TestGetMarkdownExtensionsMasksAreRemovedFromExtensions(t *testing.T) {
c := qt.New(t)
b, err := internal.NewBlackfriday(converter.ProviderConfig{Cfg: viper.New()})
c.Assert(err, qt.IsNil)
b := blackfriday_config.Default
b.Extensions = []string{"headerId"}
b.ExtensionsMask = []string{"noIntraEmphasis"}
@ -45,9 +41,7 @@ func TestGetMarkdownExtensionsByDefaultAllExtensionsAreEnabled(t *testing.T) {
testFlag int
}
c := qt.New(t)
b, err := internal.NewBlackfriday(converter.ProviderConfig{Cfg: viper.New()})
c.Assert(err, qt.IsNil)
b := blackfriday_config.Default
b.Extensions = []string{""}
b.ExtensionsMask = []string{""}
@ -79,9 +73,7 @@ func TestGetMarkdownExtensionsByDefaultAllExtensionsAreEnabled(t *testing.T) {
}
func TestGetMarkdownExtensionsAddingFlagsThroughRenderingContext(t *testing.T) {
c := qt.New(t)
b, err := internal.NewBlackfriday(converter.ProviderConfig{Cfg: viper.New()})
c.Assert(err, qt.IsNil)
b := blackfriday_config.Default
b.Extensions = []string{"definitionLists"}
b.ExtensionsMask = []string{""}
@ -93,10 +85,7 @@ func TestGetMarkdownExtensionsAddingFlagsThroughRenderingContext(t *testing.T) {
}
func TestGetFlags(t *testing.T) {
c := qt.New(t)
cfg := converter.ProviderConfig{Cfg: viper.New()}
b, err := internal.NewBlackfriday(cfg)
c.Assert(err, qt.IsNil)
b := blackfriday_config.Default
flags := getFlags(false, b)
if flags&blackfriday.HTML_USE_XHTML != blackfriday.HTML_USE_XHTML {
t.Errorf("Test flag: %d was not found amongs set flags:%d; Result: %d", blackfriday.HTML_USE_XHTML, flags, flags&blackfriday.HTML_USE_XHTML)
@ -105,9 +94,8 @@ func TestGetFlags(t *testing.T) {
func TestGetAllFlags(t *testing.T) {
c := qt.New(t)
cfg := converter.ProviderConfig{Cfg: viper.New()}
b, err := internal.NewBlackfriday(cfg)
c.Assert(err, qt.IsNil)
b := blackfriday_config.Default
type data struct {
testFlag int
@ -145,9 +133,8 @@ func TestGetAllFlags(t *testing.T) {
for _, d := range allFlags {
expectedFlags |= d.testFlag
}
if expectedFlags != actualFlags {
t.Errorf("Expected flags (%d) did not equal actual (%d) flags.", expectedFlags, actualFlags)
}
c.Assert(actualFlags, qt.Equals, expectedFlags)
}
func TestConvert(t *testing.T) {

5
markup/blackfriday/renderer.go
View file

@ -30,10 +30,9 @@ type hugoHTMLRenderer struct {
// BlockCode renders a given text as a block of code.
// Pygments is used if it is setup to handle code fences.
func (r *hugoHTMLRenderer) BlockCode(out *bytes.Buffer, text []byte, lang string) {
if r.c.pygmentsCodeFences && (lang != "" || r.c.pygmentsCodeFencesGuessSyntax) {
opts := r.c.pygmentsOptions
if r.c.cfg.MarkupConfig.Highlight.CodeFences {
str := strings.Trim(string(text), "\n\r")
highlighted, _ := r.c.cfg.Highlight(str, lang, opts)
highlighted, _ := r.c.cfg.Highlight(str, lang, "")
out.WriteString(highlighted)
} else {
r.Renderer.BlockCode(out, text, lang)

37
markup/converter/converter.go
View file

@ -16,33 +16,51 @@ package converter
import (
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/markup/markup_config"
"github.com/gohugoio/hugo/markup/tableofcontents"
"github.com/spf13/afero"
)
// ProviderConfig configures a new Provider.
type ProviderConfig struct {
MarkupConfig markup_config.Config
Cfg config.Provider // Site config
ContentFs afero.Fs
Logger *loggers.Logger
Highlight func(code, lang, optsStr string) (string, error)
}
// NewProvider creates converter providers.
type NewProvider interface {
// ProviderProvider creates converter providers.
type ProviderProvider interface {
New(cfg ProviderConfig) (Provider, error)
}
// Provider creates converters.
type Provider interface {
New(ctx DocumentContext) (Converter, error)
Name() string
}
// NewConverter is an adapter that can be used as a ConverterProvider.
type NewConverter func(ctx DocumentContext) (Converter, error)
// NewProvider creates a new Provider with the given name.
func NewProvider(name string, create func(ctx DocumentContext) (Converter, error)) Provider {
return newConverter{
name: name,
create: create,
}
}
// New creates a new Converter for the given ctx.
func (n NewConverter) New(ctx DocumentContext) (Converter, error) {
return n(ctx)
type newConverter struct {
name string
create func(ctx DocumentContext) (Converter, error)
}
func (n newConverter) New(ctx DocumentContext) (Converter, error) {
return n.create(ctx)
}
func (n newConverter) Name() string {
return n.name
}
// Converter wraps the Convert method that converts some markup into
@ -61,6 +79,11 @@ type DocumentInfo interface {
AnchorSuffix() string
}
// TableOfContentsProvider provides the content as a ToC structure.
type TableOfContentsProvider interface {
TableOfContents() tableofcontents.Root
}
// Bytes holds a byte slice and implements the Result interface.
type Bytes []byte

233
markup/goldmark/convert.go Normal file
View file

@ -0,0 +1,233 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package goldmark converts Markdown to HTML using Goldmark.
package goldmark
import (
"bytes"
"fmt"
"path/filepath"
"github.com/pkg/errors"
"github.com/spf13/afero"
"github.com/gohugoio/hugo/hugofs"
"github.com/alecthomas/chroma/styles"
"github.com/gohugoio/hugo/markup/converter"
"github.com/gohugoio/hugo/markup/highlight"
hl "github.com/gohugoio/hugo/markup/highlight/temphighlighting"
"github.com/gohugoio/hugo/markup/markup_config"
"github.com/gohugoio/hugo/markup/tableofcontents"
"github.com/yuin/goldmark"
"github.com/yuin/goldmark/extension"
"github.com/yuin/goldmark/parser"
"github.com/yuin/goldmark/renderer"
"github.com/yuin/goldmark/renderer/html"
"github.com/yuin/goldmark/text"
"github.com/yuin/goldmark/util"
)
// Provider is the package entry point.
var Provider converter.ProviderProvider = provide{}
type provide struct {
}
func (p provide) New(cfg converter.ProviderConfig) (converter.Provider, error) {
md := newMarkdown(cfg.MarkupConfig)
return converter.NewProvider("goldmark", func(ctx converter.DocumentContext) (converter.Converter, error) {
return &goldmarkConverter{
ctx: ctx,
cfg: cfg,
md: md,
}, nil
}), nil
}
type goldmarkConverter struct {
md goldmark.Markdown
ctx converter.DocumentContext
cfg converter.ProviderConfig
}
func newMarkdown(mcfg markup_config.Config) goldmark.Markdown {
cfg := mcfg.Goldmark
var (
extensions = []goldmark.Extender{
newTocExtension(),
}
rendererOptions []renderer.Option
parserOptions []parser.Option
)
if cfg.Renderer.HardWraps {
rendererOptions = append(rendererOptions, html.WithHardWraps())
}
if cfg.Renderer.XHTML {
rendererOptions = append(rendererOptions, html.WithXHTML())
}
if cfg.Renderer.Unsafe {
rendererOptions = append(rendererOptions, html.WithUnsafe())
}
if mcfg.Highlight.CodeFences {
extensions = append(extensions, newHighlighting(mcfg.Highlight))
}
if cfg.Extensions.Table {
extensions = append(extensions, extension.Table)
}
if cfg.Extensions.Strikethrough {
extensions = append(extensions, extension.Strikethrough)
}
if cfg.Extensions.Linkify {
extensions = append(extensions, extension.Linkify)
}
if cfg.Extensions.TaskList {
extensions = append(extensions, extension.TaskList)
}
if cfg.Extensions.Typographer {
extensions = append(extensions, extension.Typographer)
}
if cfg.Extensions.DefinitionList {
extensions = append(extensions, extension.DefinitionList)
}
if cfg.Extensions.Footnote {
extensions = append(extensions, extension.Footnote)
}
if cfg.Parser.AutoHeadingID {
parserOptions = append(parserOptions, parser.WithAutoHeadingID())
}
if cfg.Parser.Attribute {
parserOptions = append(parserOptions, parser.WithAttribute())
}
md := goldmark.New(
goldmark.WithExtensions(
extensions...,
),
goldmark.WithParserOptions(
parserOptions...,
),
goldmark.WithRendererOptions(
rendererOptions...,
),
)
return md
}
type converterResult struct {
converter.Result
toc tableofcontents.Root
}
func (c converterResult) TableOfContents() tableofcontents.Root {
return c.toc
}
func (c *goldmarkConverter) Convert(ctx converter.RenderContext) (result converter.Result, err error) {
defer func() {
if r := recover(); r != nil {
dir := afero.GetTempDir(hugofs.Os, "hugo_bugs")
name := fmt.Sprintf("goldmark_%s.txt", c.ctx.DocumentID)
filename := filepath.Join(dir, name)
afero.WriteFile(hugofs.Os, filename, ctx.Src, 07555)
err = errors.Errorf("[BUG] goldmark: create an issue on GitHub attaching the file in: %s", filename)
}
}()
buf := &bytes.Buffer{}
result = buf
pctx := parser.NewContext()
pctx.Set(tocEnableKey, ctx.RenderTOC)
reader := text.NewReader(ctx.Src)
doc := c.md.Parser().Parse(
reader,
parser.WithContext(pctx),
)
if err := c.md.Renderer().Render(buf, ctx.Src, doc); err != nil {
return nil, err
}
if toc, ok := pctx.Get(tocResultKey).(tableofcontents.Root); ok {
return converterResult{
Result: buf,
toc: toc,
}, nil
}
return buf, nil
}
func newHighlighting(cfg highlight.Config) goldmark.Extender {
style := styles.Get(cfg.Style)
if style == nil {
style = styles.Fallback
}
e := hl.NewHighlighting(
hl.WithStyle(cfg.Style),
hl.WithCodeBlockOptions(highlight.GetCodeBlockOptions()),
hl.WithFormatOptions(
cfg.ToHTMLOptions()...,
),
hl.WithWrapperRenderer(func(w util.BufWriter, ctx hl.CodeBlockContext, entering bool) {
l, hasLang := ctx.Language()
var language string
if hasLang {
language = string(l)
}
if entering {
if !ctx.Highlighted() {
w.WriteString(`<pre>`)
highlight.WriteCodeTag(w, language)
return
}
w.WriteString(`<div class="highlight">`)
return
}
if !ctx.Highlighted() {
w.WriteString(`</code></pre>`)
return
}
w.WriteString("</div>")
}),
)
return e
}

219
markup/goldmark/convert_test.go Normal file
View file

@ -0,0 +1,219 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package goldmark
import (
"strings"
"testing"
"github.com/gohugoio/hugo/markup/highlight"
"github.com/gohugoio/hugo/markup/markup_config"
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/markup/converter"
qt "github.com/frankban/quicktest"
)
func TestConvert(t *testing.T) {
c := qt.New(t)
// Smoke test of the default configuration.
content := `
## Code Fences
§§§bash
LINE1
§§§
## Code Fences No Lexer
§§§moo
LINE1
§§§
## Custom ID {#custom}
## Auto ID
* Autolink: https://gohugo.io/
* Strikethrough:~~Hi~~ Hello, world!
## Table
| foo | bar |
| --- | --- |
| baz | bim |
## Task Lists (default on)
- [x] Finish my changes[^1]
- [ ] Push my commits to GitHub
- [ ] Open a pull request
## Smartypants (default on)
* Straight double "quotes" and single 'quotes' into “curly” quote HTML entities
* Dashes (“--” and “---”) into en- and em-dash entities
* Three consecutive dots (“...”) into an ellipsis entity
## Footnotes
That's some text with a footnote.[^1]
## Definition Lists
date
: the datetime assigned to this page.
description
: the description for the content.
[^1]: And that's the footnote.
`
// Code fences
content = strings.Replace(content, "§§§", "```", -1)
mconf := markup_config.Default
mconf.Highlight.NoClasses = false
p, err := Provider.New(
converter.ProviderConfig{
MarkupConfig: mconf,
Logger: loggers.NewErrorLogger(),
},
)
c.Assert(err, qt.IsNil)
conv, err := p.New(converter.DocumentContext{})
c.Assert(err, qt.IsNil)
b, err := conv.Convert(converter.RenderContext{Src: []byte(content)})
c.Assert(err, qt.IsNil)
got := string(b.Bytes())
// Header IDs
c.Assert(got, qt.Contains, `<h2 id="custom">Custom ID</h2>`, qt.Commentf(got))
c.Assert(got, qt.Contains, `<h2 id="auto-id">Auto ID</h2>`, qt.Commentf(got))
// Code fences
c.Assert(got, qt.Contains, "<div class=\"highlight\"><pre class=\"chroma\"><code class=\"language-bash\" data-lang=\"bash\">LINE1\n</code></pre></div>")
c.Assert(got, qt.Contains, "Code Fences No Lexer</h2>\n<pre><code class=\"language-moo\" data-lang=\"moo\">LINE1\n</code></pre>")
// Extensions
c.Assert(got, qt.Contains, `Autolink: <a href="https://gohugo.io/">https://gohugo.io/</a>`)
c.Assert(got, qt.Contains, `Strikethrough:<del>Hi</del> Hello, world`)
c.Assert(got, qt.Contains, `<th>foo</th>`)
c.Assert(got, qt.Contains, `<li><input disabled="" type="checkbox">Push my commits to GitHub</li>`)
c.Assert(got, qt.Contains, `Straight double &ldquo;quotes&rdquo; and single &lsquo;quotes&rsquo;`)
c.Assert(got, qt.Contains, `Dashes (“&ndash;” and “&mdash;”) `)
c.Assert(got, qt.Contains, `Three consecutive dots (“&hellip;”)`)
c.Assert(got, qt.Contains, `footnote.<sup id="fnref:1"><a href="#fn:1" class="footnote-ref" role="doc-noteref">1</a></sup>`)
c.Assert(got, qt.Contains, `<section class="footnotes" role="doc-endnotes">`)
c.Assert(got, qt.Contains, `<dt>date</dt>`)
}
func TestCodeFence(t *testing.T) {
c := qt.New(t)
lines := `LINE1
LINE2
LINE3
LINE4
LINE5
`
convertForConfig := func(c *qt.C, conf highlight.Config, code, language string) string {
mconf := markup_config.Default
mconf.Highlight = conf
p, err := Provider.New(
converter.ProviderConfig{
MarkupConfig: mconf,
Logger: loggers.NewErrorLogger(),
},
)
content := "```" + language + "\n" + code + "\n```"
c.Assert(err, qt.IsNil)
conv, err := p.New(converter.DocumentContext{})
c.Assert(err, qt.IsNil)
b, err := conv.Convert(converter.RenderContext{Src: []byte(content)})
c.Assert(err, qt.IsNil)
return string(b.Bytes())
}
c.Run("Basic", func(c *qt.C) {
cfg := highlight.DefaultConfig
cfg.NoClasses = false
result := convertForConfig(c, cfg, `echo "Hugo Rocks!"`, "bash")
// TODO(bep) there is a whitespace mismatch (\n) between this and the highlight template func.
c.Assert(result, qt.Equals, `<div class="highlight"><pre class="chroma"><code class="language-bash" data-lang="bash"><span class="nb">echo</span> <span class="s2">&#34;Hugo Rocks!&#34;</span>
</code></pre></div>`)
result = convertForConfig(c, cfg, `echo "Hugo Rocks!"`, "unknown")
c.Assert(result, qt.Equals, "<pre><code class=\"language-unknown\" data-lang=\"unknown\">echo &quot;Hugo Rocks!&quot;\n</code></pre>")
})
c.Run("Highlight lines, default config", func(c *qt.C) {
cfg := highlight.DefaultConfig
cfg.NoClasses = false
result := convertForConfig(c, cfg, lines, `bash {linenos=table,hl_lines=[2 "4-5"],linenostart=3}`)
c.Assert(result, qt.Contains, "<div class=\"highlight\"><div class=\"chroma\">\n<table class=\"lntable\"><tr><td class=\"lntd\">\n<pre class=\"chroma\"><code><span class")
c.Assert(result, qt.Contains, "<span class=\"hl\"><span class=\"lnt\">4")
result = convertForConfig(c, cfg, lines, "bash {linenos=inline,hl_lines=[2]}")
c.Assert(result, qt.Contains, "<span class=\"ln\">2</span>LINE2\n</span>")
c.Assert(result, qt.Not(qt.Contains), "<table")
result = convertForConfig(c, cfg, lines, "bash {linenos=true,hl_lines=[2]}")
c.Assert(result, qt.Contains, "<table")
c.Assert(result, qt.Contains, "<span class=\"hl\"><span class=\"lnt\">2\n</span>")
})
c.Run("Highlight lines, linenumbers default on", func(c *qt.C) {
cfg := highlight.DefaultConfig
cfg.NoClasses = false
cfg.LineNos = true
result := convertForConfig(c, cfg, lines, "bash")
c.Assert(result, qt.Contains, "<span class=\"lnt\">2\n</span>")
result = convertForConfig(c, cfg, lines, "bash {linenos=false,hl_lines=[2]}")
c.Assert(result, qt.Not(qt.Contains), "class=\"lnt\"")
})
c.Run("Highlight lines, linenumbers default on, linenumbers in table default off", func(c *qt.C) {
cfg := highlight.DefaultConfig
cfg.NoClasses = false
cfg.LineNos = true
cfg.LineNumbersInTable = false
result := convertForConfig(c, cfg, lines, "bash")
c.Assert(result, qt.Contains, "<span class=\"ln\">2</span>LINE2\n<")
result = convertForConfig(c, cfg, lines, "bash {linenos=table}")
c.Assert(result, qt.Contains, "<span class=\"lnt\">1\n</span>")
})
}

74
markup/goldmark/goldmark_config/config.go Normal file
View file

@ -0,0 +1,74 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package goldmark_config holds Goldmark related configuration.
package goldmark_config
// DefaultConfig holds the default Goldmark configuration.
var Default = Config{
Extensions: Extensions{
Typographer: true,
Footnote: true,
DefinitionList: true,
Table: true,
Strikethrough: true,
Linkify: true,
TaskList: true,
},
Renderer: Renderer{
Unsafe: false,
},
Parser: Parser{
AutoHeadingID: true,
Attribute: true,
},
}
// Config configures Goldmark.
type Config struct {
Renderer Renderer
Parser Parser
Extensions Extensions
}
type Extensions struct {
Typographer bool
Footnote bool
DefinitionList bool
// GitHub flavored markdown
Table bool
Strikethrough bool
Linkify bool
TaskList bool
}
type Renderer struct {
// Whether softline breaks should be rendered as '<br>'
HardWraps bool
// XHTML instead of HTML5.
XHTML bool
// Allow raw HTML etc.
Unsafe bool
}
type Parser struct {
// Enables custom heading ids and
// auto generated heading ids.
AutoHeadingID bool
// Enables custom attributes.
Attribute bool
}

102
markup/goldmark/toc.go Normal file
View file

@ -0,0 +1,102 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package goldmark
import (
"bytes"
"github.com/gohugoio/hugo/markup/tableofcontents"
"github.com/yuin/goldmark"
"github.com/yuin/goldmark/ast"
"github.com/yuin/goldmark/parser"
"github.com/yuin/goldmark/text"
"github.com/yuin/goldmark/util"
)
var (
tocResultKey = parser.NewContextKey()
tocEnableKey = parser.NewContextKey()
)
type tocTransformer struct {
}
func (t *tocTransformer) Transform(n *ast.Document, reader text.Reader, pc parser.Context) {
if b, ok := pc.Get(tocEnableKey).(bool); !ok || !b {
return
}
var (
toc tableofcontents.Root
header tableofcontents.Header
level int
row = -1
inHeading bool
headingText bytes.Buffer
)
ast.Walk(n, func(n ast.Node, entering bool) (ast.WalkStatus, error) {
s := ast.WalkStatus(ast.WalkContinue)
if n.Kind() == ast.KindHeading {
if inHeading && !entering {
header.Text = headingText.String()
headingText.Reset()
toc.AddAt(header, row, level-1)
header = tableofcontents.Header{}
inHeading = false
return s, nil
}
inHeading = true
}
if !(inHeading && entering) {
return s, nil
}
switch n.Kind() {
case ast.KindHeading:
heading := n.(*ast.Heading)
level = heading.Level
if level == 1 || row == -1 {
row++
}
id, found := heading.AttributeString("id")
if found {
header.ID = string(id.([]byte))
}
case ast.KindText:
textNode := n.(*ast.Text)
headingText.Write(textNode.Text(reader.Source()))
}
return s, nil
})
pc.Set(tocResultKey, toc)
}
type tocExtension struct {
}
func newTocExtension() goldmark.Extender {
return &tocExtension{}
}
func (e *tocExtension) Extend(m goldmark.Markdown) {
m.Parser().AddOptions(parser.WithASTTransformers(util.Prioritized(&tocTransformer{}, 10)))
}

76
markup/goldmark/toc_test.go Normal file
View file

@ -0,0 +1,76 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package goldmark converts Markdown to HTML using Goldmark.
package goldmark
import (
"testing"
"github.com/gohugoio/hugo/markup/markup_config"
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/markup/converter"
qt "github.com/frankban/quicktest"
)
func TestToc(t *testing.T) {
c := qt.New(t)
content := `
# Header 1
## First h2
Some text.
### H3
Some more text.
## Second h2
And then some.
### Second H3
#### First H4
`
p, err := Provider.New(
converter.ProviderConfig{
MarkupConfig: markup_config.Default,
Logger: loggers.NewErrorLogger()})
c.Assert(err, qt.IsNil)
conv, err := p.New(converter.DocumentContext{})
c.Assert(err, qt.IsNil)
b, err := conv.Convert(converter.RenderContext{Src: []byte(content), RenderTOC: true})
c.Assert(err, qt.IsNil)
got := b.(converter.TableOfContentsProvider).TableOfContents().ToHTML(2, 3)
c.Assert(got, qt.Equals, `<nav id="TableOfContents">
<ul>
<li><a href="#first-h2">First h2</a>
<ul>
<li><a href="#h3">H3</a></li>
</ul>
</li>
<li><a href="#second-h2">Second h2</a>
<ul>
<li><a href="#second-h3">Second H3</a></li>
</ul>
</li>
</ul>
</nav>`, qt.Commentf(got))
}

188
markup/highlight/config.go Normal file
View file

@ -0,0 +1,188 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package highlight provides code highlighting.
package highlight
import (
"fmt"
"strconv"
"strings"
"github.com/alecthomas/chroma/formatters/html"
"github.com/gohugoio/hugo/config"
"github.com/mitchellh/mapstructure"
)
var DefaultConfig = Config{
// The highlighter style to use.
// See https://xyproto.github.io/splash/docs/all.html
Style: "monokai",
LineNoStart: 1,
CodeFences: true,
NoClasses: true,
LineNumbersInTable: true,
TabWidth: 4,
}
//
type Config struct {
Style string
CodeFences bool
// Use inline CSS styles.
NoClasses bool
// When set, line numbers will be printed.
LineNos bool
LineNumbersInTable bool
// Start the line numbers from this value (default is 1).
LineNoStart int
// A space separated list of line numbers, e.g. “3-8 10-20”.
Hl_Lines string
// TabWidth sets the number of characters for a tab. Defaults to 4.
TabWidth int
}
func (cfg Config) ToHTMLOptions() []html.Option {
var options = []html.Option{
html.TabWidth(cfg.TabWidth),
html.WithLineNumbers(cfg.LineNos),
html.BaseLineNumber(cfg.LineNoStart),
html.LineNumbersInTable(cfg.LineNumbersInTable),
html.WithClasses(!cfg.NoClasses),
}
if cfg.Hl_Lines != "" {
ranges, err := hlLinesToRanges(cfg.LineNoStart, cfg.Hl_Lines)
if err == nil {
options = append(options, html.HighlightLines(ranges))
}
}
return options
}
func applyOptionsFromString(opts string, cfg *Config) error {
optsm, err := parseOptions(opts)
if err != nil {
return err
}
return mapstructure.WeakDecode(optsm, cfg)
}
// ApplyLegacyConfig applies legacy config from back when we had
// Pygments.
func ApplyLegacyConfig(cfg config.Provider, conf *Config) error {
if conf.Style == DefaultConfig.Style {
if s := cfg.GetString("pygmentsStyle"); s != "" {
conf.Style = s
}
}
if conf.NoClasses == DefaultConfig.NoClasses && cfg.IsSet("pygmentsUseClasses") {
conf.NoClasses = !cfg.GetBool("pygmentsUseClasses")
}
if conf.CodeFences == DefaultConfig.CodeFences && cfg.IsSet("pygmentsCodeFences") {
conf.CodeFences = cfg.GetBool("pygmentsCodeFences")
}
if cfg.IsSet("pygmentsOptions") {
if err := applyOptionsFromString(cfg.GetString("pygmentsOptions"), conf); err != nil {
return err
}
}
return nil
}
func parseOptions(in string) (map[string]interface{}, error) {
in = strings.Trim(in, " ")
opts := make(map[string]interface{})
if in == "" {
return opts, nil
}
for _, v := range strings.Split(in, ",") {
keyVal := strings.Split(v, "=")
key := strings.ToLower(strings.Trim(keyVal[0], " "))
if len(keyVal) != 2 {
return opts, fmt.Errorf("invalid Highlight option: %s", key)
}
if key == "linenos" {
opts[key] = keyVal[1] != "false"
if keyVal[1] == "table" || keyVal[1] == "inline" {
opts["lineNumbersInTable"] = keyVal[1] == "table"
}
} else {
opts[key] = keyVal[1]
}
}
return opts, nil
}
// startLine compansates for https://github.com/alecthomas/chroma/issues/30
func hlLinesToRanges(startLine int, s string) ([][2]int, error) {
var ranges [][2]int
s = strings.TrimSpace(s)
if s == "" {
return ranges, nil
}
// Variants:
// 1 2 3 4
// 1-2 3-4
// 1-2 3
// 1 3-4
// 1 3-4
fields := strings.Split(s, " ")
for _, field := range fields {
field = strings.TrimSpace(field)
if field == "" {
continue
}
numbers := strings.Split(field, "-")
var r [2]int
first, err := strconv.Atoi(numbers[0])
if err != nil {
return ranges, err
}
first = first + startLine - 1
r[0] = first
if len(numbers) > 1 {
second, err := strconv.Atoi(numbers[1])
if err != nil {
return ranges, err
}
second = second + startLine - 1
r[1] = second
} else {
r[1] = first
}
ranges = append(ranges, r)
}
return ranges, nil
}

59
markup/highlight/config_test.go Normal file
View file

@ -0,0 +1,59 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package highlight provides code highlighting.
package highlight
import (
"testing"
"github.com/spf13/viper"
qt "github.com/frankban/quicktest"
)
func TestConfig(t *testing.T) {
c := qt.New(t)
c.Run("applyLegacyConfig", func(c *qt.C) {
v := viper.New()
v.Set("pygmentsStyle", "hugo")
v.Set("pygmentsUseClasses", false)
v.Set("pygmentsCodeFences", false)
v.Set("pygmentsOptions", "linenos=inline")
cfg := DefaultConfig
err := ApplyLegacyConfig(v, &cfg)
c.Assert(err, qt.IsNil)
c.Assert(cfg.Style, qt.Equals, "hugo")
c.Assert(cfg.NoClasses, qt.Equals, true)
c.Assert(cfg.CodeFences, qt.Equals, false)
c.Assert(cfg.LineNos, qt.Equals, true)
c.Assert(cfg.LineNumbersInTable, qt.Equals, false)
})
c.Run("parseOptions", func(c *qt.C) {
cfg := DefaultConfig
opts := "noclasses=true,linenos=inline,linenostart=32,hl_lines=3-8 10-20"
err := applyOptionsFromString(opts, &cfg)
c.Assert(err, qt.IsNil)
c.Assert(cfg.NoClasses, qt.Equals, true)
c.Assert(cfg.LineNos, qt.Equals, true)
c.Assert(cfg.LineNumbersInTable, qt.Equals, false)
c.Assert(cfg.LineNoStart, qt.Equals, 32)
c.Assert(cfg.Hl_Lines, qt.Equals, "3-8 10-20")
})
}

132
markup/highlight/highlight.go Normal file
View file

@ -0,0 +1,132 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package highlight
import (
"fmt"
"io"
"strings"
"github.com/alecthomas/chroma"
"github.com/alecthomas/chroma/formatters/html"
"github.com/alecthomas/chroma/lexers"
"github.com/alecthomas/chroma/styles"
hl "github.com/gohugoio/hugo/markup/highlight/temphighlighting"
)
func New(cfg Config) Highlighter {
return Highlighter{
cfg: cfg,
}
}
type Highlighter struct {
cfg Config
}
func (h Highlighter) Highlight(code, lang, optsStr string) (string, error) {
cfg := h.cfg
if optsStr != "" {
if err := applyOptionsFromString(optsStr, &cfg); err != nil {
return "", err
}
}
return highlight(code, lang, cfg)
}
func highlight(code, lang string, cfg Config) (string, error) {
w := &strings.Builder{}
var lexer chroma.Lexer
if lang != "" {
lexer = lexers.Get(lang)
}
if lexer == nil {
wrapper := getPreWrapper(lang)
fmt.Fprint(w, wrapper.Start(true, ""))
fmt.Fprint(w, code)
fmt.Fprint(w, wrapper.End(true))
return w.String(), nil
}
style := styles.Get(cfg.Style)
if style == nil {
style = styles.Fallback
}
iterator, err := lexer.Tokenise(nil, code)
if err != nil {
return "", err
}
options := cfg.ToHTMLOptions()
options = append(options, getHtmlPreWrapper(lang))
formatter := html.New(options...)
fmt.Fprintf(w, `<div class="highlight">`)
if err := formatter.Format(w, style, iterator); err != nil {
return "", err
}
fmt.Fprintf(w, `</div>`)
return w.String(), nil
}
func GetCodeBlockOptions() func(ctx hl.CodeBlockContext) []html.Option {
return func(ctx hl.CodeBlockContext) []html.Option {
var language string
if l, ok := ctx.Language(); ok {
language = string(l)
}
return []html.Option{
getHtmlPreWrapper(language),
}
}
}
func getPreWrapper(language string) preWrapper {
return preWrapper{language: language}
}
func getHtmlPreWrapper(language string) html.Option {
return html.WithPreWrapper(getPreWrapper(language))
}
type preWrapper struct {
language string
}
func (p preWrapper) Start(code bool, styleAttr string) string {
w := &strings.Builder{}
fmt.Fprintf(w, "<pre%s>", styleAttr)
var language string
if code {
language = p.language
}
WriteCodeTag(w, language)
return w.String()
}
func WriteCodeTag(w io.Writer, language string) {
fmt.Fprint(w, "<code")
if language != "" {
fmt.Fprintf(w, " class=\"language-"+language+"\"")
fmt.Fprintf(w, " data-lang=\""+language+"\"")
}
fmt.Fprint(w, ">")
}
func (p preWrapper) End(code bool) string {
return "</code></pre>"
}

87
markup/highlight/highlight_test.go Normal file
View file

@ -0,0 +1,87 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package highlight provides code highlighting.
package highlight
import (
"testing"
qt "github.com/frankban/quicktest"
)
func TestHighlight(t *testing.T) {
c := qt.New(t)
lines := `LINE1
LINE2
LINE3
LINE4
LINE5
`
c.Run("Basic", func(c *qt.C) {
cfg := DefaultConfig
cfg.NoClasses = false
h := New(cfg)
result, _ := h.Highlight(`echo "Hugo Rocks!"`, "bash", "")
c.Assert(result, qt.Equals, `<div class="highlight"><pre class="chroma"><code class="language-bash" data-lang="bash"><span class="nb">echo</span> <span class="s2">&#34;Hugo Rocks!&#34;</span></code></pre></div>`)
result, _ = h.Highlight(`echo "Hugo Rocks!"`, "unknown", "")
c.Assert(result, qt.Equals, `<pre><code class="language-unknown" data-lang="unknown">echo "Hugo Rocks!"</code></pre>`)
})
c.Run("Highlight lines, default config", func(c *qt.C) {
cfg := DefaultConfig
cfg.NoClasses = false
h := New(cfg)
result, _ := h.Highlight(lines, "bash", "linenos=table,hl_lines=2 4-5,linenostart=3")
c.Assert(result, qt.Contains, "<div class=\"highlight\"><div class=\"chroma\">\n<table class=\"lntable\"><tr><td class=\"lntd\">\n<pre class=\"chroma\"><code><span class")
c.Assert(result, qt.Contains, "<span class=\"hl\"><span class=\"lnt\">4")
result, _ = h.Highlight(lines, "bash", "linenos=inline,hl_lines=2")
c.Assert(result, qt.Contains, "<span class=\"ln\">2</span>LINE2\n</span>")
c.Assert(result, qt.Not(qt.Contains), "<table")
result, _ = h.Highlight(lines, "bash", "linenos=true,hl_lines=2")
c.Assert(result, qt.Contains, "<table")
c.Assert(result, qt.Contains, "<span class=\"hl\"><span class=\"lnt\">2\n</span>")
})
c.Run("Highlight lines, linenumbers default on", func(c *qt.C) {
cfg := DefaultConfig
cfg.NoClasses = false
cfg.LineNos = true
h := New(cfg)
result, _ := h.Highlight(lines, "bash", "")
c.Assert(result, qt.Contains, "<span class=\"lnt\">2\n</span>")
result, _ = h.Highlight(lines, "bash", "linenos=false,hl_lines=2")
c.Assert(result, qt.Not(qt.Contains), "class=\"lnt\"")
})
c.Run("Highlight lines, linenumbers default on, linenumbers in table default off", func(c *qt.C) {
cfg := DefaultConfig
cfg.NoClasses = false
cfg.LineNos = true
cfg.LineNumbersInTable = false
h := New(cfg)
result, _ := h.Highlight(lines, "bash", "")
c.Assert(result, qt.Contains, "<span class=\"ln\">2</span>LINE2\n<")
result, _ = h.Highlight(lines, "bash", "linenos=table")
c.Assert(result, qt.Contains, "<span class=\"lnt\">1\n</span>")
})
}

512
markup/highlight/temphighlighting/highlighting.go Normal file
View file

@ -0,0 +1,512 @@
// package highlighting is a extension for the goldmark(http://github.com/yuin/goldmark).
//
// This extension adds syntax-highlighting to the fenced code blocks using
// chroma(https://github.com/alecthomas/chroma).
//
// TODO(bep) this is a very temporary fork based on https://github.com/yuin/goldmark-highlighting/pull/10
// MIT Licensed, Copyright Yusuke Inuzuka
package temphighlighting
import (
"bytes"
"io"
"strconv"
"strings"
"github.com/yuin/goldmark"
"github.com/yuin/goldmark/ast"
"github.com/yuin/goldmark/parser"
"github.com/yuin/goldmark/renderer"
"github.com/yuin/goldmark/renderer/html"
"github.com/yuin/goldmark/text"
"github.com/yuin/goldmark/util"
"github.com/alecthomas/chroma"
chromahtml "github.com/alecthomas/chroma/formatters/html"
"github.com/alecthomas/chroma/lexers"
"github.com/alecthomas/chroma/styles"
)
// ImmutableAttributes is a read-only interface for ast.Attributes.
type ImmutableAttributes interface {
// Get returns (value, true) if an attribute associated with given
// name exists, otherwise (nil, false)
Get(name []byte) (interface{}, bool)
// GetString returns (value, true) if an attribute associated with given
// name exists, otherwise (nil, false)
GetString(name string) (interface{}, bool)
// All returns all attributes.
All() []ast.Attribute
}
type immutableAttributes struct {
n ast.Node
}
func (a *immutableAttributes) Get(name []byte) (interface{}, bool) {
return a.n.Attribute(name)
}
func (a *immutableAttributes) GetString(name string) (interface{}, bool) {
return a.n.AttributeString(name)
}
func (a *immutableAttributes) All() []ast.Attribute {
if a.n.Attributes() == nil {
return []ast.Attribute{}
}
return a.n.Attributes()
}
// CodeBlockContext holds contextual information of code highlighting.
type CodeBlockContext interface {
// Language returns (language, true) if specified, otherwise (nil, false).
Language() ([]byte, bool)
// Highlighted returns true if this code block can be highlighted, otherwise false.
Highlighted() bool
// Attributes return attributes of the code block.
Attributes() ImmutableAttributes
}
type codeBlockContext struct {
language []byte
highlighted bool
attributes ImmutableAttributes
}
func newCodeBlockContext(language []byte, highlighted bool, attrs ImmutableAttributes) CodeBlockContext {
return &codeBlockContext{
language: language,
highlighted: highlighted,
attributes: attrs,
}
}
func (c *codeBlockContext) Language() ([]byte, bool) {
if c.language != nil {
return c.language, true
}
return nil, false
}
func (c *codeBlockContext) Highlighted() bool {
return c.highlighted
}
func (c *codeBlockContext) Attributes() ImmutableAttributes {
return c.attributes
}
// WrapperRenderer renders wrapper elements like div, pre, etc.
type WrapperRenderer func(w util.BufWriter, context CodeBlockContext, entering bool)
// CodeBlockOptions creates Chroma options per code block.
type CodeBlockOptions func(ctx CodeBlockContext) []chromahtml.Option
// Config struct holds options for the extension.
type Config struct {
html.Config
// Style is a highlighting style.
// Supported styles are defined under https://github.com/alecthomas/chroma/tree/master/formatters.
Style string
// FormatOptions is a option related to output formats.
// See https://github.com/alecthomas/chroma#the-html-formatter for details.
FormatOptions []chromahtml.Option
// CSSWriter is an io.Writer that will be used as CSS data output buffer.
// If WithClasses() is enabled, you can get CSS data corresponds to the style.
CSSWriter io.Writer
// CodeBlockOptions allows set Chroma options per code block.
CodeBlockOptions CodeBlockOptions
// WrapperRendererCodeBlockOptions allows you to change wrapper elements.
WrapperRenderer WrapperRenderer
}
// NewConfig returns a new Config with defaults.
func NewConfig() Config {
return Config{
Config: html.NewConfig(),
Style: "github",
FormatOptions: []chromahtml.Option{},
CSSWriter: nil,
WrapperRenderer: nil,
CodeBlockOptions: nil,
}
}
// SetOption implements renderer.SetOptioner.
func (c *Config) SetOption(name renderer.OptionName, value interface{}) {
switch name {
case optStyle:
c.Style = value.(string)
case optFormatOptions:
if value != nil {
c.FormatOptions = value.([]chromahtml.Option)
}
case optCSSWriter:
c.CSSWriter = value.(io.Writer)
case optWrapperRenderer:
c.WrapperRenderer = value.(WrapperRenderer)
case optCodeBlockOptions:
c.CodeBlockOptions = value.(CodeBlockOptions)
default:
c.Config.SetOption(name, value)
}
}
// Option interface is a functional option interface for the extension.
type Option interface {
renderer.Option
// SetHighlightingOption sets given option to the extension.
SetHighlightingOption(*Config)
}
type withHTMLOptions struct {
value []html.Option
}
func (o *withHTMLOptions) SetConfig(c *renderer.Config) {
if o.value != nil {
for _, v := range o.value {
v.(renderer.Option).SetConfig(c)
}
}
}
func (o *withHTMLOptions) SetHighlightingOption(c *Config) {
if o.value != nil {
for _, v := range o.value {
v.SetHTMLOption(&c.Config)
}
}
}
// WithHTMLOptions is functional option that wraps goldmark HTMLRenderer options.
func WithHTMLOptions(opts ...html.Option) Option {
return &withHTMLOptions{opts}
}
const optStyle renderer.OptionName = "HighlightingStyle"
var highlightLinesAttrName = []byte("hl_lines")
var styleAttrName = []byte("hl_style")
var nohlAttrName = []byte("nohl")
var linenosAttrName = []byte("linenos")
var linenosTableAttrValue = []byte("table")
var linenosInlineAttrValue = []byte("inline")
var linenostartAttrName = []byte("linenostart")
type withStyle struct {
value string
}
func (o *withStyle) SetConfig(c *renderer.Config) {
c.Options[optStyle] = o.value
}
func (o *withStyle) SetHighlightingOption(c *Config) {
c.Style = o.value
}
// WithStyle is a functional option that changes highlighting style.
func WithStyle(style string) Option {
return &withStyle{style}
}
const optCSSWriter renderer.OptionName = "HighlightingCSSWriter"
type withCSSWriter struct {
value io.Writer
}
func (o *withCSSWriter) SetConfig(c *renderer.Config) {
c.Options[optCSSWriter] = o.value
}
func (o *withCSSWriter) SetHighlightingOption(c *Config) {
c.CSSWriter = o.value
}
// WithCSSWriter is a functional option that sets io.Writer for CSS data.
func WithCSSWriter(w io.Writer) Option {
return &withCSSWriter{w}
}
const optWrapperRenderer renderer.OptionName = "HighlightingWrapperRenderer"
type withWrapperRenderer struct {
value WrapperRenderer
}
func (o *withWrapperRenderer) SetConfig(c *renderer.Config) {
c.Options[optWrapperRenderer] = o.value
}
func (o *withWrapperRenderer) SetHighlightingOption(c *Config) {
c.WrapperRenderer = o.value
}
// WithWrapperRenderer is a functional option that sets WrapperRenderer that
// renders wrapper elements like div, pre, etc.
func WithWrapperRenderer(w WrapperRenderer) Option {
return &withWrapperRenderer{w}
}
const optCodeBlockOptions renderer.OptionName = "HighlightingCodeBlockOptions"
type withCodeBlockOptions struct {
value CodeBlockOptions
}
func (o *withCodeBlockOptions) SetConfig(c *renderer.Config) {
c.Options[optWrapperRenderer] = o.value
}
func (o *withCodeBlockOptions) SetHighlightingOption(c *Config) {
c.CodeBlockOptions = o.value
}
// WithCodeBlockOptions is a functional option that sets CodeBlockOptions that
// allows setting Chroma options per code block.
func WithCodeBlockOptions(c CodeBlockOptions) Option {
return &withCodeBlockOptions{value: c}
}
const optFormatOptions renderer.OptionName = "HighlightingFormatOptions"
type withFormatOptions struct {
value []chromahtml.Option
}
func (o *withFormatOptions) SetConfig(c *renderer.Config) {
if _, ok := c.Options[optFormatOptions]; !ok {
c.Options[optFormatOptions] = []chromahtml.Option{}
}
c.Options[optStyle] = append(c.Options[optFormatOptions].([]chromahtml.Option), o.value...)
}
func (o *withFormatOptions) SetHighlightingOption(c *Config) {
c.FormatOptions = append(c.FormatOptions, o.value...)
}
// WithFormatOptions is a functional option that wraps chroma HTML formatter options.
func WithFormatOptions(opts ...chromahtml.Option) Option {
return &withFormatOptions{opts}
}
// HTMLRenderer struct is a renderer.NodeRenderer implementation for the extension.
type HTMLRenderer struct {
Config
}
// NewHTMLRenderer builds a new HTMLRenderer with given options and returns it.
func NewHTMLRenderer(opts ...Option) renderer.NodeRenderer {
r := &HTMLRenderer{
Config: NewConfig(),
}
for _, opt := range opts {
opt.SetHighlightingOption(&r.Config)
}
return r
}
// RegisterFuncs implements NodeRenderer.RegisterFuncs.
func (r *HTMLRenderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) {
reg.Register(ast.KindFencedCodeBlock, r.renderFencedCodeBlock)
}
func getAttributes(node *ast.FencedCodeBlock, infostr []byte) ImmutableAttributes {
if node.Attributes() != nil {
return &immutableAttributes{node}
}
if infostr != nil {
attrStartIdx := -1
for idx, char := range infostr {
if char == '{' {
attrStartIdx = idx
break
}
}
if attrStartIdx > 0 {
n := ast.NewTextBlock() // dummy node for storing attributes
attrStr := infostr[attrStartIdx:]
if attrs, hasAttr := parser.ParseAttributes(text.NewReader(attrStr)); hasAttr {
for _, attr := range attrs {
n.SetAttribute(attr.Name, attr.Value)
}
return &immutableAttributes{n}
}
}
}
return nil
}
func (r *HTMLRenderer) renderFencedCodeBlock(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
n := node.(*ast.FencedCodeBlock)
if !entering {
return ast.WalkContinue, nil
}
language := n.Language(source)
chromaFormatterOptions := make([]chromahtml.Option, len(r.FormatOptions))
copy(chromaFormatterOptions, r.FormatOptions)
style := styles.Get(r.Style)
nohl := false
var info []byte
if n.Info != nil {
info = n.Info.Segment.Value(source)
}
attrs := getAttributes(n, info)
if attrs != nil {
baseLineNumber := 1
if linenostartAttr, ok := attrs.Get(linenostartAttrName); ok {
baseLineNumber = int(linenostartAttr.(float64))
chromaFormatterOptions = append(chromaFormatterOptions, chromahtml.BaseLineNumber(baseLineNumber))
}
if linesAttr, hasLinesAttr := attrs.Get(highlightLinesAttrName); hasLinesAttr {
if lines, ok := linesAttr.([]interface{}); ok {
var hlRanges [][2]int
for _, l := range lines {
if ln, ok := l.(float64); ok {
hlRanges = append(hlRanges, [2]int{int(ln) + baseLineNumber - 1, int(ln) + baseLineNumber - 1})
}
if rng, ok := l.([]uint8); ok {
slices := strings.Split(string([]byte(rng)), "-")
lhs, err := strconv.Atoi(slices[0])
if err != nil {
continue
}
rhs := lhs
if len(slices) > 1 {
rhs, err = strconv.Atoi(slices[1])
if err != nil {
continue
}
}
hlRanges = append(hlRanges, [2]int{lhs + baseLineNumber - 1, rhs + baseLineNumber - 1})
}
}
chromaFormatterOptions = append(chromaFormatterOptions, chromahtml.HighlightLines(hlRanges))
}
}
if styleAttr, hasStyleAttr := attrs.Get(styleAttrName); hasStyleAttr {
styleStr := string([]byte(styleAttr.([]uint8)))
style = styles.Get(styleStr)
}
if _, hasNohlAttr := attrs.Get(nohlAttrName); hasNohlAttr {
nohl = true
}
if linenosAttr, ok := attrs.Get(linenosAttrName); ok {
switch v := linenosAttr.(type) {
case bool:
chromaFormatterOptions = append(chromaFormatterOptions, chromahtml.WithLineNumbers(v))
case []uint8:
if v != nil {
chromaFormatterOptions = append(chromaFormatterOptions, chromahtml.WithLineNumbers(true))
}
if bytes.Equal(v, linenosTableAttrValue) {
chromaFormatterOptions = append(chromaFormatterOptions, chromahtml.LineNumbersInTable(true))
} else if bytes.Equal(v, linenosInlineAttrValue) {
chromaFormatterOptions = append(chromaFormatterOptions, chromahtml.LineNumbersInTable(false))
}
}
}
}
var lexer chroma.Lexer
if language != nil {
lexer = lexers.Get(string(language))
}
if !nohl && lexer != nil {
if style == nil {
style = styles.Fallback
}
var buffer bytes.Buffer
l := n.Lines().Len()
for i := 0; i < l; i++ {
line := n.Lines().At(i)
buffer.Write(line.Value(source))
}
iterator, err := lexer.Tokenise(nil, buffer.String())
if err == nil {
c := newCodeBlockContext(language, true, attrs)
if r.CodeBlockOptions != nil {
chromaFormatterOptions = append(chromaFormatterOptions, r.CodeBlockOptions(c)...)
}
formatter := chromahtml.New(chromaFormatterOptions...)
if r.WrapperRenderer != nil {
r.WrapperRenderer(w, c, true)
}
_ = formatter.Format(w, style, iterator) == nil
if r.WrapperRenderer != nil {
r.WrapperRenderer(w, c, false)
}
if r.CSSWriter != nil {
_ = formatter.WriteCSS(r.CSSWriter, style)
}
return ast.WalkContinue, nil
}
}
var c CodeBlockContext
if r.WrapperRenderer != nil {
c = newCodeBlockContext(language, false, attrs)
r.WrapperRenderer(w, c, true)
} else {
_, _ = w.WriteString("<pre><code")
language := n.Language(source)
if language != nil {
_, _ = w.WriteString(" class=\"language-")
r.Writer.Write(w, language)
_, _ = w.WriteString("\"")
}
_ = w.WriteByte('>')
}
l := n.Lines().Len()
for i := 0; i < l; i++ {
line := n.Lines().At(i)
r.Writer.RawWrite(w, line.Value(source))
}
if r.WrapperRenderer != nil {
r.WrapperRenderer(w, c, false)
} else {
_, _ = w.WriteString("</code></pre>\n")
}
return ast.WalkContinue, nil
}
type highlighting struct {
options []Option
}
// Highlighting is a goldmark.Extender implementation.
var Highlighting = &highlighting{
options: []Option{},
}
// NewHighlighting returns a new extension with given options.
func NewHighlighting(opts ...Option) goldmark.Extender {
return &highlighting{
options: opts,
}
}
// Extend implements goldmark.Extender.
func (e *highlighting) Extend(m goldmark.Markdown) {
m.Renderer().AddOptions(renderer.WithNodeRenderers(
util.Prioritized(NewHTMLRenderer(e.options...), 200),
))
}

335
markup/highlight/temphighlighting/highlighting_test.go Normal file
View file

@ -0,0 +1,335 @@
// TODO(bep) this is a very temporary fork based on https://github.com/yuin/goldmark-highlighting/pull/10
// MIT Licensed, Copyright Yusuke Inuzuka
package temphighlighting
import (
"bytes"
"fmt"
"strings"
"testing"
"github.com/yuin/goldmark/util"
chromahtml "github.com/alecthomas/chroma/formatters/html"
"github.com/yuin/goldmark"
)
type preWrapper struct {
language string
}
func (p preWrapper) Start(code bool, styleAttr string) string {
w := &strings.Builder{}
fmt.Fprintf(w, "<pre%s><code", styleAttr)
if p.language != "" {
fmt.Fprintf(w, " class=\"language-"+p.language)
}
fmt.Fprint(w, ">")
return w.String()
}
func (p preWrapper) End(code bool) string {
return "</code></pre>"
}
func TestHighlighting(t *testing.T) {
var css bytes.Buffer
markdown := goldmark.New(
goldmark.WithExtensions(
NewHighlighting(
WithStyle("monokai"),
WithCSSWriter(&css),
WithFormatOptions(
chromahtml.WithClasses(true),
chromahtml.WithLineNumbers(false),
),
WithWrapperRenderer(func(w util.BufWriter, c CodeBlockContext, entering bool) {
_, ok := c.Language()
if entering {
if !ok {
w.WriteString("<pre><code>")
return
}
w.WriteString(`<div class="highlight">`)
} else {
if !ok {
w.WriteString("</pre></code>")
return
}
w.WriteString(`</div>`)
}
}),
WithCodeBlockOptions(func(c CodeBlockContext) []chromahtml.Option {
if language, ok := c.Language(); ok {
// Turn on line numbers for Go only.
if string(language) == "go" {
return []chromahtml.Option{
chromahtml.WithLineNumbers(true),
}
}
}
return nil
}),
),
),
)
var buffer bytes.Buffer
if err := markdown.Convert([]byte(`
Title
=======
`+"``` go\n"+`func main() {
fmt.Println("ok")
}
`+"```"+`
`), &buffer); err != nil {
t.Fatal(err)
}
if strings.TrimSpace(buffer.String()) != strings.TrimSpace(`
<h1>Title</h1>
<div class="highlight"><pre class="chroma"><span class="ln">1</span><span class="kd">func</span> <span class="nf">main</span><span class="p">(</span><span class="p">)</span> <span class="p">{</span>
<span class="ln">2</span> <span class="nx">fmt</span><span class="p">.</span><span class="nf">Println</span><span class="p">(</span><span class="s">&#34;ok&#34;</span><span class="p">)</span>
<span class="ln">3</span><span class="p">}</span>
</pre></div>
`) {
t.Error("failed to render HTML")
}
if strings.TrimSpace(css.String()) != strings.TrimSpace(`/* Background */ .chroma { color: #f8f8f2; background-color: #272822 }
/* Error */ .chroma .err { color: #960050; background-color: #1e0010 }
/* LineTableTD */ .chroma .lntd { vertical-align: top; padding: 0; margin: 0; border: 0; }
/* LineTable */ .chroma .lntable { border-spacing: 0; padding: 0; margin: 0; border: 0; width: auto; overflow: auto; display: block; }
/* LineHighlight */ .chroma .hl { display: block; width: 100%;background-color: #3c3d38 }
/* LineNumbersTable */ .chroma .lnt { margin-right: 0.4em; padding: 0 0.4em 0 0.4em;color: #7f7f7f }
/* LineNumbers */ .chroma .ln { margin-right: 0.4em; padding: 0 0.4em 0 0.4em;color: #7f7f7f }
/* Keyword */ .chroma .k { color: #66d9ef }
/* KeywordConstant */ .chroma .kc { color: #66d9ef }
/* KeywordDeclaration */ .chroma .kd { color: #66d9ef }
/* KeywordNamespace */ .chroma .kn { color: #f92672 }
/* KeywordPseudo */ .chroma .kp { color: #66d9ef }
/* KeywordReserved */ .chroma .kr { color: #66d9ef }
/* KeywordType */ .chroma .kt { color: #66d9ef }
/* NameAttribute */ .chroma .na { color: #a6e22e }
/* NameClass */ .chroma .nc { color: #a6e22e }
/* NameConstant */ .chroma .no { color: #66d9ef }
/* NameDecorator */ .chroma .nd { color: #a6e22e }
/* NameException */ .chroma .ne { color: #a6e22e }
/* NameFunction */ .chroma .nf { color: #a6e22e }
/* NameOther */ .chroma .nx { color: #a6e22e }
/* NameTag */ .chroma .nt { color: #f92672 }
/* Literal */ .chroma .l { color: #ae81ff }
/* LiteralDate */ .chroma .ld { color: #e6db74 }
/* LiteralString */ .chroma .s { color: #e6db74 }
/* LiteralStringAffix */ .chroma .sa { color: #e6db74 }
/* LiteralStringBacktick */ .chroma .sb { color: #e6db74 }
/* LiteralStringChar */ .chroma .sc { color: #e6db74 }
/* LiteralStringDelimiter */ .chroma .dl { color: #e6db74 }
/* LiteralStringDoc */ .chroma .sd { color: #e6db74 }
/* LiteralStringDouble */ .chroma .s2 { color: #e6db74 }
/* LiteralStringEscape */ .chroma .se { color: #ae81ff }
/* LiteralStringHeredoc */ .chroma .sh { color: #e6db74 }
/* LiteralStringInterpol */ .chroma .si { color: #e6db74 }
/* LiteralStringOther */ .chroma .sx { color: #e6db74 }
/* LiteralStringRegex */ .chroma .sr { color: #e6db74 }
/* LiteralStringSingle */ .chroma .s1 { color: #e6db74 }
/* LiteralStringSymbol */ .chroma .ss { color: #e6db74 }
/* LiteralNumber */ .chroma .m { color: #ae81ff }
/* LiteralNumberBin */ .chroma .mb { color: #ae81ff }
/* LiteralNumberFloat */ .chroma .mf { color: #ae81ff }
/* LiteralNumberHex */ .chroma .mh { color: #ae81ff }
/* LiteralNumberInteger */ .chroma .mi { color: #ae81ff }
/* LiteralNumberIntegerLong */ .chroma .il { color: #ae81ff }
/* LiteralNumberOct */ .chroma .mo { color: #ae81ff }
/* Operator */ .chroma .o { color: #f92672 }
/* OperatorWord */ .chroma .ow { color: #f92672 }
/* Comment */ .chroma .c { color: #75715e }
/* CommentHashbang */ .chroma .ch { color: #75715e }
/* CommentMultiline */ .chroma .cm { color: #75715e }
/* CommentSingle */ .chroma .c1 { color: #75715e }
/* CommentSpecial */ .chroma .cs { color: #75715e }
/* CommentPreproc */ .chroma .cp { color: #75715e }
/* CommentPreprocFile */ .chroma .cpf { color: #75715e }
/* GenericDeleted */ .chroma .gd { color: #f92672 }
/* GenericEmph */ .chroma .ge { font-style: italic }
/* GenericInserted */ .chroma .gi { color: #a6e22e }
/* GenericStrong */ .chroma .gs { font-weight: bold }
/* GenericSubheading */ .chroma .gu { color: #75715e }`) {
t.Error("failed to render CSS")
}
}
func TestHighlighting2(t *testing.T) {
markdown := goldmark.New(
goldmark.WithExtensions(
Highlighting,
),
)
var buffer bytes.Buffer
if err := markdown.Convert([]byte(`
Title
=======
`+"```"+`
func main() {
fmt.Println("ok")
}
`+"```"+`
`), &buffer); err != nil {
t.Fatal(err)
}
if strings.TrimSpace(buffer.String()) != strings.TrimSpace(`
<h1>Title</h1>
<pre><code>func main() {
fmt.Println(&quot;ok&quot;)
}
</code></pre>
`) {
t.Error("failed to render HTML")
}
}
func TestHighlighting3(t *testing.T) {
markdown := goldmark.New(
goldmark.WithExtensions(
Highlighting,
),
)
var buffer bytes.Buffer
if err := markdown.Convert([]byte(`
Title
=======
`+"```"+`cpp {hl_lines=[1,2]}
#include <iostream>
int main() {
std::cout<< "hello" << std::endl;
}
`+"```"+`
`), &buffer); err != nil {
t.Fatal(err)
}
if strings.TrimSpace(buffer.String()) != strings.TrimSpace(`
<h1>Title</h1>
<pre style="background-color:#fff"><span style="display:block;width:100%;background-color:#e5e5e5"><span style="color:#999;font-weight:bold;font-style:italic">#</span><span style="color:#999;font-weight:bold;font-style:italic">include</span> <span style="color:#999;font-weight:bold;font-style:italic">&lt;iostream&gt;</span><span style="color:#999;font-weight:bold;font-style:italic">
</span></span><span style="display:block;width:100%;background-color:#e5e5e5"><span style="color:#999;font-weight:bold;font-style:italic"></span><span style="color:#458;font-weight:bold">int</span> <span style="color:#900;font-weight:bold">main</span>() {
</span> std<span style="color:#000;font-weight:bold">:</span><span style="color:#000;font-weight:bold">:</span>cout<span style="color:#000;font-weight:bold">&lt;</span><span style="color:#000;font-weight:bold">&lt;</span> <span style="color:#d14"></span><span style="color:#d14">&#34;</span><span style="color:#d14">hello</span><span style="color:#d14">&#34;</span> <span style="color:#000;font-weight:bold">&lt;</span><span style="color:#000;font-weight:bold">&lt;</span> std<span style="color:#000;font-weight:bold">:</span><span style="color:#000;font-weight:bold">:</span>endl;
}
</pre>
`) {
t.Error("failed to render HTML")
}
}
func TestHighlightingHlLines(t *testing.T) {
markdown := goldmark.New(
goldmark.WithExtensions(
NewHighlighting(
WithFormatOptions(
chromahtml.WithClasses(true),
),
),
),
)
for i, test := range []struct {
attributes string
expect []int
}{
{`hl_lines=["2"]`, []int{2}},
{`hl_lines=["2-3",5],linenostart=5`, []int{2, 3, 5}},
{`hl_lines=["2-3"]`, []int{2, 3}},
} {
t.Run(fmt.Sprint(i), func(t *testing.T) {
var buffer bytes.Buffer
codeBlock := fmt.Sprintf(`bash {%s}
LINE1
LINE2
LINE3
LINE4
LINE5
LINE6
LINE7
LINE8
`, test.attributes)
if err := markdown.Convert([]byte(`
`+"```"+codeBlock+"```"+`
`), &buffer); err != nil {
t.Fatal(err)
}
for _, line := range test.expect {
expectStr := fmt.Sprintf("<span class=\"hl\">LINE%d\n</span>", line)
if !strings.Contains(buffer.String(), expectStr) {
t.Fatal("got\n", buffer.String(), "\nexpected\n", expectStr)
}
}
})
}
}
func TestHighlightingLinenos(t *testing.T) {
outputLineNumbersInTable := `<div class="chroma">
<table class="lntable"><tr><td class="lntd">
<span class="lnt">1
</span></td>
<td class="lntd">
LINE1
</td></tr></table>
</div>`
for i, test := range []struct {
attributes string
lineNumbers bool
lineNumbersInTable bool
expect string
}{
{`linenos=true`, false, false, `<span class="ln">1</span>LINE1`},
{`linenos=false`, false, false, `LINE1`},
{``, true, false, `<span class="ln">1</span>LINE1`},
{``, true, true, outputLineNumbersInTable},
{`linenos=inline`, true, true, `<span class="ln">1</span>LINE1`},
{`linenos=foo`, false, false, `<span class="ln">1</span>LINE1`},
{`linenos=table`, false, false, outputLineNumbersInTable},
} {
t.Run(fmt.Sprint(i), func(t *testing.T) {
markdown := goldmark.New(
goldmark.WithExtensions(
NewHighlighting(
WithFormatOptions(
chromahtml.WithLineNumbers(test.lineNumbers),
chromahtml.LineNumbersInTable(test.lineNumbersInTable),
chromahtml.PreventSurroundingPre(true),
chromahtml.WithClasses(true),
),
),
),
)
var buffer bytes.Buffer
codeBlock := fmt.Sprintf(`bash {%s}
LINE1
`, test.attributes)
content := "```" + codeBlock + "```"
if err := markdown.Convert([]byte(content), &buffer); err != nil {
t.Fatal(err)
}
s := strings.TrimSpace(buffer.String())
if s != test.expect {
t.Fatal("got\n", s, "\nexpected\n", test.expect)
}
})
}
}

108
markup/internal/blackfriday.go
View file

@ -1,108 +0,0 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package helpers implements general utility functions that work with
// and on content. The helper functions defined here lay down the
// foundation of how Hugo works with files and filepaths, and perform
// string operations on content.
package internal
import (
"github.com/gohugoio/hugo/common/maps"
"github.com/gohugoio/hugo/markup/converter"
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
)
// BlackFriday holds configuration values for BlackFriday rendering.
// It is kept here because it's used in several packages.
type BlackFriday struct {
Smartypants bool
SmartypantsQuotesNBSP bool
AngledQuotes bool
Fractions bool
HrefTargetBlank bool
NofollowLinks bool
NoreferrerLinks bool
SmartDashes bool
LatexDashes bool
TaskLists bool
PlainIDAnchors bool
Extensions []string
ExtensionsMask []string
SkipHTML bool
FootnoteAnchorPrefix string
FootnoteReturnLinkContents string
}
func UpdateBlackFriday(old *BlackFriday, m map[string]interface{}) (*BlackFriday, error) {
// Create a copy so we can modify it.
bf := *old
if err := mapstructure.Decode(m, &bf); err != nil {
return nil, errors.WithMessage(err, "failed to decode rendering config")
}
return &bf, nil
}
// NewBlackfriday creates a new Blackfriday filled with site config or some sane defaults.
func NewBlackfriday(cfg converter.ProviderConfig) (*BlackFriday, error) {
var siteConfig map[string]interface{}
if cfg.Cfg != nil {
siteConfig = cfg.Cfg.GetStringMap("blackfriday")
}
defaultParam := map[string]interface{}{
"smartypants": true,
"angledQuotes": false,
"smartypantsQuotesNBSP": false,
"fractions": true,
"hrefTargetBlank": false,
"nofollowLinks": false,
"noreferrerLinks": false,
"smartDashes": true,
"latexDashes": true,
"plainIDAnchors": true,
"taskLists": true,
"skipHTML": false,
}
maps.ToLower(defaultParam)
config := make(map[string]interface{})
for k, v := range defaultParam {
config[k] = v
}
for k, v := range siteConfig {
config[k] = v
}
combinedConfig := &BlackFriday{}
if err := mapstructure.Decode(config, combinedConfig); err != nil {
return nil, errors.Errorf("failed to decode Blackfriday config: %s", err)
}
// TODO(bep) update/consolidate docs
if combinedConfig.FootnoteAnchorPrefix == "" {
combinedConfig.FootnoteAnchorPrefix = cfg.Cfg.GetString("footnoteAnchorPrefix")
}
if combinedConfig.FootnoteReturnLinkContents == "" {
combinedConfig.FootnoteReturnLinkContents = cfg.Cfg.GetString("footnoteReturnLinkContents")
}
return combinedConfig, nil
}

64
markup/markup.go
View file

@ -16,6 +16,12 @@ package markup
import (
"strings"
"github.com/gohugoio/hugo/markup/highlight"
"github.com/gohugoio/hugo/markup/markup_config"
"github.com/gohugoio/hugo/markup/goldmark"
"github.com/gohugoio/hugo/markup/org"
"github.com/gohugoio/hugo/markup/asciidoc"
@ -29,39 +35,71 @@ import (
func NewConverterProvider(cfg converter.ProviderConfig) (ConverterProvider, error) {
converters := make(map[string]converter.Provider)
add := func(p converter.NewProvider, aliases ...string) error {
markupConfig, err := markup_config.Decode(cfg.Cfg)
if err != nil {
return nil, err
}
if cfg.Highlight == nil {
h := highlight.New(markupConfig.Highlight)
cfg.Highlight = func(code, lang, optsStr string) (string, error) {
return h.Highlight(code, lang, optsStr)
}
}
cfg.MarkupConfig = markupConfig
add := func(p converter.ProviderProvider, aliases ...string) error {
c, err := p.New(cfg)
if err != nil {
return err
}
name := c.Name()
aliases = append(aliases, name)
if strings.EqualFold(name, cfg.MarkupConfig.DefaultMarkdownHandler) {
aliases = append(aliases, "markdown")
}
addConverter(converters, c, aliases...)
return nil
}
if err := add(blackfriday.Provider, "md", "markdown", "blackfriday"); err != nil {
if err := add(goldmark.Provider); err != nil {
return nil, err
}
if err := add(mmark.Provider, "mmark"); err != nil {
if err := add(blackfriday.Provider); err != nil {
return nil, err
}
if err := add(asciidoc.Provider, "asciidoc"); err != nil {
if err := add(mmark.Provider); err != nil {
return nil, err
}
if err := add(rst.Provider, "rst"); err != nil {
if err := add(asciidoc.Provider, "ad", "adoc"); err != nil {
return nil, err
}
if err := add(pandoc.Provider, "pandoc"); err != nil {
if err := add(rst.Provider); err != nil {
return nil, err
}
if err := add(org.Provider, "org"); err != nil {
if err := add(pandoc.Provider, "pdc"); err != nil {
return nil, err
}
if err := add(org.Provider); err != nil {
return nil, err
}
return &converterRegistry{converters: converters}, nil
return &converterRegistry{
config: cfg,
converters: converters,
}, nil
}
type ConverterProvider interface {
Get(name string) converter.Provider
//Default() converter.Provider
GetMarkupConfig() markup_config.Config
Highlight(code, lang, optsStr string) (string, error)
}
type converterRegistry struct {
@ -70,12 +108,22 @@ type converterRegistry struct {
// may be registered multiple times.
// All names are lower case.
converters map[string]converter.Provider
config converter.ProviderConfig
}
func (r *converterRegistry) Get(name string) converter.Provider {
return r.converters[strings.ToLower(name)]
}
func (r *converterRegistry) Highlight(code, lang, optsStr string) (string, error) {
return r.config.Highlight(code, lang, optsStr)
}
func (r *converterRegistry) GetMarkupConfig() markup_config.Config {
return r.config.MarkupConfig
}
func addConverter(m map[string]converter.Provider, c converter.Provider, aliases ...string) {
for _, alias := range aliases {
m[alias] = c

105
markup/markup_config/config.go Normal file
View file

@ -0,0 +1,105 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package markup_config
import (
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/docshelper"
"github.com/gohugoio/hugo/markup/blackfriday/blackfriday_config"
"github.com/gohugoio/hugo/markup/goldmark/goldmark_config"
"github.com/gohugoio/hugo/markup/highlight"
"github.com/gohugoio/hugo/markup/tableofcontents"
"github.com/gohugoio/hugo/parser"
"github.com/mitchellh/mapstructure"
)
type Config struct {
// Default markdown handler for md/markdown extensions.
// Default is "goldmark".
// Before Hugo 0.60 this was "blackfriday".
DefaultMarkdownHandler string
Highlight highlight.Config
TableOfContents tableofcontents.Config
// Content renderers
Goldmark goldmark_config.Config
BlackFriday blackfriday_config.Config
}
func Decode(cfg config.Provider) (conf Config, err error) {
conf = Default
m := cfg.GetStringMap("markup")
if m == nil {
return
}
err = mapstructure.WeakDecode(m, &conf)
if err != nil {
return
}
if err = applyLegacyConfig(cfg, &conf); err != nil {
return
}
if err = highlight.ApplyLegacyConfig(cfg, &conf.Highlight); err != nil {
return
}
return
}
func applyLegacyConfig(cfg config.Provider, conf *Config) error {
if bm := cfg.GetStringMap("blackfriday"); bm != nil {
// Legacy top level blackfriday config.
err := mapstructure.WeakDecode(bm, &conf.BlackFriday)
if err != nil {
return err
}
}
if conf.BlackFriday.FootnoteAnchorPrefix == "" {
conf.BlackFriday.FootnoteAnchorPrefix = cfg.GetString("footnoteAnchorPrefix")
}
if conf.BlackFriday.FootnoteReturnLinkContents == "" {
conf.BlackFriday.FootnoteReturnLinkContents = cfg.GetString("footnoteReturnLinkContents")
}
return nil
}
var Default = Config{
DefaultMarkdownHandler: "goldmark",
TableOfContents: tableofcontents.DefaultConfig,
Highlight: highlight.DefaultConfig,
Goldmark: goldmark_config.Default,
BlackFriday: blackfriday_config.Default,
}
func init() {
docsProvider := func() map[string]interface{} {
docs := make(map[string]interface{})
docs["markup"] = parser.LowerCaseCamelJSONMarshaller{Value: Default}
return docs
}
// TODO(bep) merge maps
docshelper.AddDocProvider("config", docsProvider)
}

69
markup/markup_config/config_test.go Normal file
View file

@ -0,0 +1,69 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package markup_config
import (
"testing"
"github.com/spf13/viper"
qt "github.com/frankban/quicktest"
)
func TestConfig(t *testing.T) {
c := qt.New(t)
c.Run("Decode", func(c *qt.C) {
c.Parallel()
v := viper.New()
v.Set("markup", map[string]interface{}{
"goldmark": map[string]interface{}{
"renderer": map[string]interface{}{
"unsafe": true,
},
},
})
conf, err := Decode(v)
c.Assert(err, qt.IsNil)
c.Assert(conf.Goldmark.Renderer.Unsafe, qt.Equals, true)
c.Assert(conf.BlackFriday.Fractions, qt.Equals, true)
})
c.Run("legacy", func(c *qt.C) {
c.Parallel()
v := viper.New()
v.Set("blackfriday", map[string]interface{}{
"angledQuotes": true,
})
v.Set("footnoteAnchorPrefix", "myprefix")
v.Set("footnoteReturnLinkContents", "myreturn")
v.Set("pygmentsStyle", "hugo")
conf, err := Decode(v)
c.Assert(err, qt.IsNil)
c.Assert(conf.BlackFriday.AngledQuotes, qt.Equals, true)
c.Assert(conf.BlackFriday.FootnoteAnchorPrefix, qt.Equals, "myprefix")
c.Assert(conf.BlackFriday.FootnoteReturnLinkContents, qt.Equals, "myreturn")
c.Assert(conf.Highlight.Style, qt.Equals, "hugo")
c.Assert(conf.Highlight.CodeFences, qt.Equals, true)
})
}

22
markup/markup_test.go
View file

@ -29,13 +29,23 @@ func TestConverterRegistry(t *testing.T) {
r, err := NewConverterProvider(converter.ProviderConfig{Cfg: viper.New()})
c.Assert(err, qt.IsNil)
c.Assert("goldmark", qt.Equals, r.GetMarkupConfig().DefaultMarkdownHandler)
checkName := func(name string) {
p := r.Get(name)
c.Assert(p, qt.Not(qt.IsNil))
c.Assert(p.Name(), qt.Equals, name)
}
c.Assert(r.Get("foo"), qt.IsNil)
c.Assert(r.Get("markdown"), qt.Not(qt.IsNil))
c.Assert(r.Get("mmark"), qt.Not(qt.IsNil))
c.Assert(r.Get("asciidoc"), qt.Not(qt.IsNil))
c.Assert(r.Get("rst"), qt.Not(qt.IsNil))
c.Assert(r.Get("pandoc"), qt.Not(qt.IsNil))
c.Assert(r.Get("org"), qt.Not(qt.IsNil))
c.Assert(r.Get("markdown").Name(), qt.Equals, "goldmark")
checkName("goldmark")
checkName("mmark")
checkName("asciidoc")
checkName("rst")
checkName("pandoc")
checkName("org")
checkName("blackfriday")
}

32
markup/mmark/convert.go
View file

@ -15,33 +15,28 @@
package mmark
import (
"github.com/gohugoio/hugo/markup/internal"
"github.com/gohugoio/hugo/markup/blackfriday/blackfriday_config"
"github.com/gohugoio/hugo/markup/converter"
"github.com/miekg/mmark"
)
// Provider is the package entry point.
var Provider converter.NewProvider = provider{}
var Provider converter.ProviderProvider = provider{}
type provider struct {
}
func (p provider) New(cfg converter.ProviderConfig) (converter.Provider, error) {
defaultBlackFriday, err := internal.NewBlackfriday(cfg)
if err != nil {
return nil, err
}
defaultBlackFriday := cfg.MarkupConfig.BlackFriday
defaultExtensions := getMmarkExtensions(defaultBlackFriday)
var n converter.NewConverter = func(ctx converter.DocumentContext) (converter.Converter, error) {
return converter.NewProvider("mmark", func(ctx converter.DocumentContext) (converter.Converter, error) {
b := defaultBlackFriday
extensions := defaultExtensions
if ctx.ConfigOverrides != nil {
var err error
b, err = internal.UpdateBlackFriday(b, ctx.ConfigOverrides)
b, err = blackfriday_config.UpdateConfig(b, ctx.ConfigOverrides)
if err != nil {
return nil, err
}
@ -54,16 +49,14 @@ func (p provider) New(cfg converter.ProviderConfig) (converter.Provider, error)
extensions: extensions,
cfg: cfg,
}, nil
}
return n, nil
}), nil
}
type mmarkConverter struct {
ctx converter.DocumentContext
extensions int
b *internal.BlackFriday
b blackfriday_config.Config
cfg converter.ProviderConfig
}
@ -74,7 +67,7 @@ func (c *mmarkConverter) Convert(ctx converter.RenderContext) (converter.Result,
func getHTMLRenderer(
ctx converter.DocumentContext,
cfg *internal.BlackFriday,
cfg blackfriday_config.Config,
pcfg converter.ProviderConfig) mmark.Renderer {
var (
@ -97,15 +90,14 @@ func getHTMLRenderer(
htmlFlags |= mmark.HTML_FOOTNOTE_RETURN_LINKS
return &mmarkRenderer{
Config: cfg,
Cfg: pcfg.Cfg,
highlight: pcfg.Highlight,
Renderer: mmark.HtmlRendererWithParameters(htmlFlags, "", "", renderParameters),
BlackfridayConfig: cfg,
Config: pcfg,
Renderer: mmark.HtmlRendererWithParameters(htmlFlags, "", "", renderParameters),
}
}
func getMmarkExtensions(cfg *internal.BlackFriday) int {
func getMmarkExtensions(cfg blackfriday_config.Config) int {
flags := 0
flags |= mmark.EXTENSION_TABLES
flags |= mmark.EXTENSION_FENCED_CODE

13
markup/mmark/convert_test.go
View file

@ -20,19 +20,14 @@ import (
"github.com/gohugoio/hugo/common/loggers"
"github.com/miekg/mmark"
"github.com/gohugoio/hugo/markup/internal"
"github.com/gohugoio/hugo/markup/converter"
qt "github.com/frankban/quicktest"
"github.com/gohugoio/hugo/markup/blackfriday/blackfriday_config"
"github.com/gohugoio/hugo/markup/converter"
"github.com/miekg/mmark"
)
func TestGetMmarkExtensions(t *testing.T) {
c := qt.New(t)
b, err := internal.NewBlackfriday(converter.ProviderConfig{Cfg: viper.New()})
c.Assert(err, qt.IsNil)
b := blackfriday_config.Default
//TODO: This is doing the same just with different marks...
type data struct {

14
markup/mmark/renderer.go
View file

@ -17,26 +17,24 @@ import (
"bytes"
"strings"
"github.com/gohugoio/hugo/markup/blackfriday/blackfriday_config"
"github.com/gohugoio/hugo/markup/converter"
"github.com/miekg/mmark"
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/markup/internal"
)
// hugoHTMLRenderer wraps a blackfriday.Renderer, typically a blackfriday.Html
// adding some custom behaviour.
type mmarkRenderer struct {
Cfg config.Provider
Config *internal.BlackFriday
highlight func(code, lang, optsStr string) (string, error)
Config converter.ProviderConfig
BlackfridayConfig blackfriday_config.Config
mmark.Renderer
}
// BlockCode renders a given text as a block of code.
func (r *mmarkRenderer) BlockCode(out *bytes.Buffer, text []byte, lang string, caption []byte, subfigure bool, callouts bool) {
if r.Cfg.GetBool("pygmentsCodeFences") && (lang != "" || r.Cfg.GetBool("pygmentsCodeFencesGuessSyntax")) {
if r.Config.MarkupConfig.Highlight.CodeFences {
str := strings.Trim(string(text), "\n\r")
highlighted, _ := r.highlight(str, lang, "")
highlighted, _ := r.Config.Highlight(str, lang, "")
out.WriteString(highlighted)
} else {
r.Renderer.BlockCode(out, text, lang, caption, subfigure, callouts)

7
markup/org/convert.go
View file

@ -23,19 +23,18 @@ import (
)
// Provider is the package entry point.
var Provider converter.NewProvider = provide{}
var Provider converter.ProviderProvider = provide{}
type provide struct {
}
func (p provide) New(cfg converter.ProviderConfig) (converter.Provider, error) {
var n converter.NewConverter = func(ctx converter.DocumentContext) (converter.Converter, error) {
return converter.NewProvider("org", func(ctx converter.DocumentContext) (converter.Converter, error) {
return &orgConverter{
ctx: ctx,
cfg: cfg,
}, nil
}
return n, nil
}), nil
}
type orgConverter struct {

7
markup/pandoc/convert.go
View file

@ -23,19 +23,18 @@ import (
)
// Provider is the package entry point.
var Provider converter.NewProvider = provider{}
var Provider converter.ProviderProvider = provider{}
type provider struct {
}
func (p provider) New(cfg converter.ProviderConfig) (converter.Provider, error) {
var n converter.NewConverter = func(ctx converter.DocumentContext) (converter.Converter, error) {
return converter.NewProvider("pandoc", func(ctx converter.DocumentContext) (converter.Converter, error) {
return &pandocConverter{
ctx: ctx,
cfg: cfg,
}, nil
}
return n, nil
}), nil
}

8
markup/rst/convert.go
View file

@ -25,20 +25,18 @@ import (
)
// Provider is the package entry point.
var Provider converter.NewProvider = provider{}
var Provider converter.ProviderProvider = provider{}
type provider struct {
}
func (p provider) New(cfg converter.ProviderConfig) (converter.Provider, error) {
var n converter.NewConverter = func(ctx converter.DocumentContext) (converter.Converter, error) {
return converter.NewProvider("rst", func(ctx converter.DocumentContext) (converter.Converter, error) {
return &rstConverter{
ctx: ctx,
cfg: cfg,
}, nil
}
return n, nil
}), nil
}
type rstConverter struct {

148
markup/tableofcontents/tableofcontents.go Normal file
View file

@ -0,0 +1,148 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tableofcontents
import (
"strings"
)
type Headers []Header
type Header struct {
ID string
Text string
Headers Headers
}
func (h Header) IsZero() bool {
return h.ID == "" && h.Text == ""
}
type Root struct {
Headers Headers
}
func (toc *Root) AddAt(h Header, y, x int) {
for i := len(toc.Headers); i <= y; i++ {
toc.Headers = append(toc.Headers, Header{})
}
if x == 0 {
toc.Headers[y] = h
return
}
header := &toc.Headers[y]
for i := 1; i < x; i++ {
if len(header.Headers) == 0 {
header.Headers = append(header.Headers, Header{})
}
header = &header.Headers[len(header.Headers)-1]
}
header.Headers = append(header.Headers, h)
}
func (toc Root) ToHTML(startLevel, stopLevel int) string {
b := &tocBuilder{
s: strings.Builder{},
h: toc.Headers,
startLevel: startLevel,
stopLevel: stopLevel,
}
b.Build()
return b.s.String()
}
type tocBuilder struct {
s strings.Builder
h Headers
startLevel int
stopLevel int
}
func (b *tocBuilder) Build() {
b.buildHeaders2(b.h)
}
func (b *tocBuilder) buildHeaders2(h Headers) {
b.s.WriteString("<nav id=\"TableOfContents\">")
b.buildHeaders(1, 0, b.h)
b.s.WriteString("</nav>")
}
func (b *tocBuilder) buildHeaders(level, indent int, h Headers) {
if level < b.startLevel {
for _, h := range h {
b.buildHeaders(level+1, indent, h.Headers)
}
return
}
if b.stopLevel != -1 && level > b.stopLevel {
return
}
hasChildren := len(h) > 0
if hasChildren {
b.s.WriteString("\n")
b.indent(indent + 1)
b.s.WriteString("<ul>\n")
}
for _, h := range h {
b.buildHeader(level+1, indent+2, h)
}
if hasChildren {
b.indent(indent + 1)
b.s.WriteString("</ul>")
b.s.WriteString("\n")
b.indent(indent)
}
}
func (b *tocBuilder) buildHeader(level, indent int, h Header) {
b.indent(indent)
b.s.WriteString("<li>")
if !h.IsZero() {
b.s.WriteString("<a href=\"#" + h.ID + "\">" + h.Text + "</a>")
}
b.buildHeaders(level, indent, h.Headers)
b.s.WriteString("</li>\n")
}
func (b *tocBuilder) indent(n int) {
for i := 0; i < n; i++ {
b.s.WriteString(" ")
}
}
var DefaultConfig = Config{
StartLevel: 2,
EndLevel: 3,
}
type Config struct {
// Heading start level to include in the table of contents, starting
// at h1 (inclusive).
StartLevel int
// Heading end level, inclusive, to include in the table of contents.
// Default is 3, a value of -1 will include everything.
EndLevel int
}

119
markup/tableofcontents/tableofcontents_test.go Normal file
View file

@ -0,0 +1,119 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tableofcontents
import (
"testing"
qt "github.com/frankban/quicktest"
)
func TestToc(t *testing.T) {
c := qt.New(t)
toc := &Root{}
toc.AddAt(Header{Text: "Header 1", ID: "h1-1"}, 0, 0)
toc.AddAt(Header{Text: "1-H2-1", ID: "1-h2-1"}, 0, 1)
toc.AddAt(Header{Text: "1-H2-2", ID: "1-h2-2"}, 0, 1)
toc.AddAt(Header{Text: "1-H3-1", ID: "1-h2-2"}, 0, 2)
toc.AddAt(Header{Text: "Header 2", ID: "h1-2"}, 1, 0)
got := toc.ToHTML(1, -1)
c.Assert(got, qt.Equals, `<nav id="TableOfContents">
<ul>
<li><a href="#h1-1">Header 1</a>
<ul>
<li><a href="#1-h2-1">1-H2-1</a></li>
<li><a href="#1-h2-2">1-H2-2</a>
<ul>
<li><a href="#1-h2-2">1-H3-1</a></li>
</ul>
</li>
</ul>
</li>
<li><a href="#h1-2">Header 2</a></li>
</ul>
</nav>`, qt.Commentf(got))
got = toc.ToHTML(1, 1)
c.Assert(got, qt.Equals, `<nav id="TableOfContents">
<ul>
<li><a href="#h1-1">Header 1</a></li>
<li><a href="#h1-2">Header 2</a></li>
</ul>
</nav>`, qt.Commentf(got))
got = toc.ToHTML(1, 2)
c.Assert(got, qt.Equals, `<nav id="TableOfContents">
<ul>
<li><a href="#h1-1">Header 1</a>
<ul>
<li><a href="#1-h2-1">1-H2-1</a></li>
<li><a href="#1-h2-2">1-H2-2</a></li>
</ul>
</li>
<li><a href="#h1-2">Header 2</a></li>
</ul>
</nav>`, qt.Commentf(got))
got = toc.ToHTML(2, 2)
c.Assert(got, qt.Equals, `<nav id="TableOfContents">
<ul>
<li><a href="#1-h2-1">1-H2-1</a></li>
<li><a href="#1-h2-2">1-H2-2</a></li>
</ul>
</nav>`, qt.Commentf(got))
}
func TestTocMissingParent(t *testing.T) {
c := qt.New(t)
toc := &Root{}
toc.AddAt(Header{Text: "H2", ID: "h2"}, 0, 1)
toc.AddAt(Header{Text: "H3", ID: "h3"}, 1, 2)
toc.AddAt(Header{Text: "H3", ID: "h3"}, 1, 2)
got := toc.ToHTML(1, -1)
c.Assert(got, qt.Equals, `<nav id="TableOfContents">
<ul>
<li>
<ul>
<li><a href="#h2">H2</a></li>
</ul>
</li>
<li>
<ul>
<li>
<ul>
<li><a href="#h3">H3</a></li>
<li><a href="#h3">H3</a></li>
</ul>
</li>
</ul>
</li>
</ul>
</nav>`, qt.Commentf(got))
got = toc.ToHTML(3, 3)
c.Assert(got, qt.Equals, `<nav id="TableOfContents">
<ul>
<li><a href="#h3">H3</a></li>
<li><a href="#h3">H3</a></li>
</ul>
</nav>`, qt.Commentf(got))
}

51
parser/lowercase_camel_json.go Normal file
View file

@ -0,0 +1,51 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package parser
import (
"encoding/json"
"regexp"
"unicode"
"unicode/utf8"
)
// Regexp definitions
var keyMatchRegex = regexp.MustCompile(`\"(\w+)\":`)
var wordBarrierRegex = regexp.MustCompile(`(\w)([A-Z])`)
// Code adapted from https://gist.github.com/piersy/b9934790a8892db1a603820c0c23e4a7
type LowerCaseCamelJSONMarshaller struct {
Value interface{}
}
func (c LowerCaseCamelJSONMarshaller) MarshalJSON() ([]byte, error) {
marshalled, err := json.Marshal(c.Value)
converted := keyMatchRegex.ReplaceAllFunc(
marshalled,
func(match []byte) []byte {
// Empty keys are valid JSON, only lowercase if we do not have an
// empty key.
if len(match) > 2 {
// Decode first rune after the double quotes
r, width := utf8.DecodeRune(match[1:])
r = unicode.ToLower(r)
utf8.EncodeRune(match[1:width+1], r)
}
return match
},
)
return converted, err
}

58
tpl/transform/remarshal.go
View file

@ -18,32 +18,41 @@ import (
// change without notice if it serves a purpose in the docs.
// Format is one of json, yaml or toml.
func (ns *Namespace) Remarshal(format string, data interface{}) (string, error) {
from, err := cast.ToStringE(data)
if err != nil {
return "", err
}
var meta map[string]interface{}
from = strings.TrimSpace(from)
format = strings.TrimSpace(strings.ToLower(format))
if from == "" {
return "", nil
}
mark, err := toFormatMark(format)
if err != nil {
return "", err
}
fromFormat := metadecoders.Default.FormatFromContentString(from)
if fromFormat == "" {
return "", errors.New("failed to detect format from content")
if m, ok := data.(map[string]interface{}); ok {
meta = m
} else {
from, err := cast.ToStringE(data)
if err != nil {
return "", err
}
from = strings.TrimSpace(from)
if from == "" {
return "", nil
}
fromFormat := metadecoders.Default.FormatFromContentString(from)
if fromFormat == "" {
return "", errors.New("failed to detect format from content")
}
meta, err = metadecoders.Default.UnmarshalToMap([]byte(from), fromFormat)
if err != nil {
return "", err
}
}
meta, err := metadecoders.Default.UnmarshalToMap([]byte(from), fromFormat)
if err != nil {
return "", err
}
// Make it so 1.0 float64 prints as 1 etc.
applyMarshalTypes(meta)
var result bytes.Buffer
if err := parser.InterfaceToConfig(meta, mark, &result); err != nil {
@ -53,6 +62,23 @@ func (ns *Namespace) Remarshal(format string, data interface{}) (string, error)
return result.String(), nil
}
// The unmarshal/marshal dance is extremely type lossy, and we need
// to make sure that integer types prints as "43" and not "43.0" in
// all formats, hence this hack.
func applyMarshalTypes(m map[string]interface{}) {
for k, v := range m {
switch t := v.(type) {
case map[string]interface{}:
applyMarshalTypes(t)
case float64:
i := int64(t)
if t == float64(i) {
m[k] = i
}
}
}
}
func toFormatMark(format string) (metadecoders.Format, error) {
if f := metadecoders.FormatFromString(format); f != "" {
return f, nil

18
tpl/transform/remarshal_test.go
View file

@ -156,11 +156,11 @@ Hugo = "Rules"
func TestTestRemarshalError(t *testing.T) {
t.Parallel()
c := qt.New(t)
v := viper.New()
v.Set("contentDir", "content")
ns := New(newDeps(v))
c := qt.New(t)
_, err := ns.Remarshal("asdf", "asdf")
c.Assert(err, qt.Not(qt.IsNil))
@ -169,3 +169,19 @@ func TestTestRemarshalError(t *testing.T) {
c.Assert(err, qt.Not(qt.IsNil))
}
func TestTestRemarshalMapInput(t *testing.T) {
t.Parallel()
c := qt.New(t)
v := viper.New()
v.Set("contentDir", "content")
ns := New(newDeps(v))
input := map[string]interface{}{
"hello": "world",
}
output, err := ns.Remarshal("toml", input)
c.Assert(err, qt.IsNil)
c.Assert(output, qt.Equals, "hello = \"world\"\n")
}

2
tpl/transform/transform.go
View file

@ -65,7 +65,7 @@ func (ns *Namespace) Highlight(s interface{}, lang, opts string) (template.HTML,
return "", err
}
highlighted, _ := ns.deps.ContentSpec.Highlight(ss, lang, opts)
highlighted, _ := ns.deps.ContentSpec.Converters.Highlight(ss, lang, opts)
return template.HTML(highlighted), nil
}

2
tpl/transform/transform_test.go
View file

@ -204,7 +204,7 @@ And then some.
result, err := ns.Markdownify(text)
c.Assert(err, qt.IsNil)
c.Assert(result, qt.Equals, template.HTML(
"<p>#First</p>\n\n<p>This is some <em>bold</em> text.</p>\n\n<h2 id=\"second\">Second</h2>\n\n<p>This is some more text.</p>\n\n<p>And then some.</p>\n"))
"<p>#First</p>\n<p>This is some <em>bold</em> text.</p>\n<h2 id=\"second\">Second</h2>\n<p>This is some more text.</p>\n<p>And then some.</p>\n"))
}