github/hugo
1
0
Fork 0
mirror of https://github.com/gohugoio/hugo.git synced 2024-11-28 23:02:05 -05:00
Code Issues Projects Releases Packages Wiki Activity

Prepare for Goldmark

Browse source 
This commmit prepares for the addition of Goldmark as the new Markdown renderer in Hugo.

This introduces a new `markup` package with some common interfaces and each implementation in its own package.

See #5963
This commit is contained in:
Bjørn Erik Pedersen 2019-08-16 15:55:03 +02:00
parent 366ee4d8da
commit 5f6b6ec689
No known key found for this signature in database
GPG key ID: 330E6E2BD4859D8F
39 changed files with 1739 additions and 986 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
deps/deps.go vendored
View file

@ -223,7 +223,7 @@ func New(cfg DepsCfg) (*Deps, error) {
return nil, err return nil, err
} }
contentSpec, err := helpers.NewContentSpec(cfg.Language) contentSpec, err := helpers.NewContentSpec(cfg.Language, logger, ps.BaseFs.Content.Fs)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -277,7 +277,7 @@ func (d Deps) ForLanguage(cfg DepsCfg, onCreated func(d *Deps) error) (*Deps, er
return nil, err return nil, err
} }
d.ContentSpec, err = helpers.NewContentSpec(l) d.ContentSpec, err = helpers.NewContentSpec(l, d.Log, d.BaseFs.Content.Fs)
if err != nil { if err != nil {
return nil, err return nil, err
} }

522
helpers/content.go
View file

@ -19,22 +19,18 @@ package helpers
import ( import (
"bytes" "bytes"
"fmt"
"html/template" "html/template"
"os/exec"
"runtime"
"unicode" "unicode"
"unicode/utf8" "unicode/utf8"
"github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/hugolib/filesystems"
"github.com/niklasfasching/go-org/org" "github.com/gohugoio/hugo/markup/converter"
"github.com/gohugoio/hugo/markup"
bp "github.com/gohugoio/hugo/bufferpool" bp "github.com/gohugoio/hugo/bufferpool"
"github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/config"
"github.com/miekg/mmark"
"github.com/mitchellh/mapstructure"
"github.com/russross/blackfriday"
"github.com/spf13/afero" "github.com/spf13/afero"
jww "github.com/spf13/jwalterweatherman" jww "github.com/spf13/jwalterweatherman"
@ -52,9 +48,9 @@ var (
// ContentSpec provides functionality to render markdown content. // ContentSpec provides functionality to render markdown content.
type ContentSpec struct { type ContentSpec struct {
BlackFriday *BlackFriday Converters markup.ConverterProvider
footnoteAnchorPrefix string MardownConverter converter.Converter // Markdown converter with no document context
footnoteReturnLinkContents string
// SummaryLength is the length of the summary that Hugo extracts from a content. // SummaryLength is the length of the summary that Hugo extracts from a content.
summaryLength int summaryLength int
@ -70,12 +66,9 @@ type ContentSpec struct {
// NewContentSpec returns a ContentSpec initialized // NewContentSpec returns a ContentSpec initialized
// with the appropriate fields from the given config.Provider. // with the appropriate fields from the given config.Provider.
func NewContentSpec(cfg config.Provider) (*ContentSpec, error) { func NewContentSpec(cfg config.Provider, logger *loggers.Logger, contentFs afero.Fs) (*ContentSpec, error) {
bf := newBlackfriday(cfg.GetStringMap("blackfriday"))
spec := &ContentSpec{ spec := &ContentSpec{
BlackFriday: bf,
footnoteAnchorPrefix: cfg.GetString("footnoteAnchorPrefix"),
footnoteReturnLinkContents: cfg.GetString("footnoteReturnLinkContents"),
summaryLength: cfg.GetInt("summaryLength"), summaryLength: cfg.GetInt("summaryLength"),
BuildFuture: cfg.GetBool("buildFuture"), BuildFuture: cfg.GetBool("buildFuture"),
BuildExpired: cfg.GetBool("buildExpired"), BuildExpired: cfg.GetBool("buildExpired"),
@ -109,99 +102,29 @@ func NewContentSpec(cfg config.Provider) (*ContentSpec, error) {
spec.Highlight = h.chromaHighlight spec.Highlight = h.chromaHighlight
} }
converterProvider, err := markup.NewConverterProvider(converter.ProviderConfig{
Cfg: cfg,
ContentFs: contentFs,
Logger: logger,
Highlight: spec.Highlight,
})
if err != nil {
return nil, err
}
spec.Converters = converterProvider
p := converterProvider.Get("markdown")
conv, err := p.New(converter.DocumentContext{})
if err != nil {
return nil, err
}
spec.MardownConverter = conv
return spec, nil return spec, nil
} }
// BlackFriday holds configuration values for BlackFriday rendering.
type BlackFriday struct {
Smartypants bool
SmartypantsQuotesNBSP bool
AngledQuotes bool
Fractions bool
HrefTargetBlank bool
NofollowLinks bool
NoreferrerLinks bool
SmartDashes bool
LatexDashes bool
TaskLists bool
PlainIDAnchors bool
Extensions []string
ExtensionsMask []string
SkipHTML bool
}
// NewBlackfriday creates a new Blackfriday filled with site config or some sane defaults.
func newBlackfriday(config map[string]interface{}) *BlackFriday {
defaultParam := map[string]interface{}{
"smartypants": true,
"angledQuotes": false,
"smartypantsQuotesNBSP": false,
"fractions": true,
"hrefTargetBlank": false,
"nofollowLinks": false,
"noreferrerLinks": false,
"smartDashes": true,
"latexDashes": true,
"plainIDAnchors": true,
"taskLists": true,
"skipHTML": false,
}
maps.ToLower(defaultParam)
siteConfig := make(map[string]interface{})
for k, v := range defaultParam {
siteConfig[k] = v
}
for k, v := range config {
siteConfig[k] = v
}
combinedConfig := &BlackFriday{}
if err := mapstructure.Decode(siteConfig, combinedConfig); err != nil {
jww.FATAL.Printf("Failed to get site rendering config\n%s", err.Error())
}
return combinedConfig
}
var blackfridayExtensionMap = map[string]int{
"noIntraEmphasis": blackfriday.EXTENSION_NO_INTRA_EMPHASIS,
"tables": blackfriday.EXTENSION_TABLES,
"fencedCode": blackfriday.EXTENSION_FENCED_CODE,
"autolink": blackfriday.EXTENSION_AUTOLINK,
"strikethrough": blackfriday.EXTENSION_STRIKETHROUGH,
"laxHtmlBlocks": blackfriday.EXTENSION_LAX_HTML_BLOCKS,
"spaceHeaders": blackfriday.EXTENSION_SPACE_HEADERS,
"hardLineBreak": blackfriday.EXTENSION_HARD_LINE_BREAK,
"tabSizeEight": blackfriday.EXTENSION_TAB_SIZE_EIGHT,
"footnotes": blackfriday.EXTENSION_FOOTNOTES,
"noEmptyLineBeforeBlock": blackfriday.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK,
"headerIds": blackfriday.EXTENSION_HEADER_IDS,
"titleblock": blackfriday.EXTENSION_TITLEBLOCK,
"autoHeaderIds": blackfriday.EXTENSION_AUTO_HEADER_IDS,
"backslashLineBreak": blackfriday.EXTENSION_BACKSLASH_LINE_BREAK,
"definitionLists": blackfriday.EXTENSION_DEFINITION_LISTS,
"joinLines": blackfriday.EXTENSION_JOIN_LINES,
}
var stripHTMLReplacer = strings.NewReplacer("\n", " ", "</p>", "\n", "<br>", "\n", "<br />", "\n") var stripHTMLReplacer = strings.NewReplacer("\n", " ", "</p>", "\n", "<br>", "\n", "<br />", "\n")
var mmarkExtensionMap = map[string]int{
"tables": mmark.EXTENSION_TABLES,
"fencedCode": mmark.EXTENSION_FENCED_CODE,
"autolink": mmark.EXTENSION_AUTOLINK,
"laxHtmlBlocks": mmark.EXTENSION_LAX_HTML_BLOCKS,
"spaceHeaders": mmark.EXTENSION_SPACE_HEADERS,
"hardLineBreak": mmark.EXTENSION_HARD_LINE_BREAK,
"footnotes": mmark.EXTENSION_FOOTNOTES,
"noEmptyLineBeforeBlock": mmark.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK,
"headerIds": mmark.EXTENSION_HEADER_IDS,
"autoHeaderIds": mmark.EXTENSION_AUTO_HEADER_IDS,
}
// StripHTML accepts a string, strips out all HTML tags and returns it. // StripHTML accepts a string, strips out all HTML tags and returns it.
func StripHTML(s string) string { func StripHTML(s string) string {
@ -250,181 +173,6 @@ func BytesToHTML(b []byte) template.HTML {
return template.HTML(string(b)) return template.HTML(string(b))
} }
// getHTMLRenderer creates a new Blackfriday HTML Renderer with the given configuration.
func (c *ContentSpec) getHTMLRenderer(defaultFlags int, ctx *RenderingContext) blackfriday.Renderer {
renderParameters := blackfriday.HtmlRendererParameters{
FootnoteAnchorPrefix: c.footnoteAnchorPrefix,
FootnoteReturnLinkContents: c.footnoteReturnLinkContents,
}
b := len(ctx.DocumentID) != 0
if ctx.Config == nil {
panic(fmt.Sprintf("RenderingContext of %q doesn't have a config", ctx.DocumentID))
}
if b && !ctx.Config.PlainIDAnchors {
renderParameters.FootnoteAnchorPrefix = ctx.DocumentID + ":" + renderParameters.FootnoteAnchorPrefix
renderParameters.HeaderIDSuffix = ":" + ctx.DocumentID
}
htmlFlags := defaultFlags
htmlFlags |= blackfriday.HTML_USE_XHTML
htmlFlags |= blackfriday.HTML_FOOTNOTE_RETURN_LINKS
if ctx.Config.Smartypants {
htmlFlags |= blackfriday.HTML_USE_SMARTYPANTS
}
if ctx.Config.SmartypantsQuotesNBSP {
htmlFlags |= blackfriday.HTML_SMARTYPANTS_QUOTES_NBSP
}
if ctx.Config.AngledQuotes {
htmlFlags |= blackfriday.HTML_SMARTYPANTS_ANGLED_QUOTES
}
if ctx.Config.Fractions {
htmlFlags |= blackfriday.HTML_SMARTYPANTS_FRACTIONS
}
if ctx.Config.HrefTargetBlank {
htmlFlags |= blackfriday.HTML_HREF_TARGET_BLANK
}
if ctx.Config.NofollowLinks {
htmlFlags |= blackfriday.HTML_NOFOLLOW_LINKS
}
if ctx.Config.NoreferrerLinks {
htmlFlags |= blackfriday.HTML_NOREFERRER_LINKS
}
if ctx.Config.SmartDashes {
htmlFlags |= blackfriday.HTML_SMARTYPANTS_DASHES
}
if ctx.Config.LatexDashes {
htmlFlags |= blackfriday.HTML_SMARTYPANTS_LATEX_DASHES
}
if ctx.Config.SkipHTML {
htmlFlags |= blackfriday.HTML_SKIP_HTML
}
return &HugoHTMLRenderer{
cs: c,
RenderingContext: ctx,
Renderer: blackfriday.HtmlRendererWithParameters(htmlFlags, "", "", renderParameters),
}
}
func getMarkdownExtensions(ctx *RenderingContext) int {
// Default Blackfriday common extensions
commonExtensions := 0 |
blackfriday.EXTENSION_NO_INTRA_EMPHASIS |
blackfriday.EXTENSION_TABLES |
blackfriday.EXTENSION_FENCED_CODE |
blackfriday.EXTENSION_AUTOLINK |
blackfriday.EXTENSION_STRIKETHROUGH |
blackfriday.EXTENSION_SPACE_HEADERS |
blackfriday.EXTENSION_HEADER_IDS |
blackfriday.EXTENSION_BACKSLASH_LINE_BREAK |
blackfriday.EXTENSION_DEFINITION_LISTS
// Extra Blackfriday extensions that Hugo enables by default
flags := commonExtensions |
blackfriday.EXTENSION_AUTO_HEADER_IDS |
blackfriday.EXTENSION_FOOTNOTES
if ctx.Config == nil {
panic(fmt.Sprintf("RenderingContext of %q doesn't have a config", ctx.DocumentID))
}
for _, extension := range ctx.Config.Extensions {
if flag, ok := blackfridayExtensionMap[extension]; ok {
flags |= flag
}
}
for _, extension := range ctx.Config.ExtensionsMask {
if flag, ok := blackfridayExtensionMap[extension]; ok {
flags &= ^flag
}
}
return flags
}
func (c *ContentSpec) markdownRender(ctx *RenderingContext) []byte {
if ctx.RenderTOC {
return blackfriday.Markdown(ctx.Content,
c.getHTMLRenderer(blackfriday.HTML_TOC, ctx),
getMarkdownExtensions(ctx))
}
return blackfriday.Markdown(ctx.Content, c.getHTMLRenderer(0, ctx),
getMarkdownExtensions(ctx))
}
// getMmarkHTMLRenderer creates a new mmark HTML Renderer with the given configuration.
func (c *ContentSpec) getMmarkHTMLRenderer(defaultFlags int, ctx *RenderingContext) mmark.Renderer {
renderParameters := mmark.HtmlRendererParameters{
FootnoteAnchorPrefix: c.footnoteAnchorPrefix,
FootnoteReturnLinkContents: c.footnoteReturnLinkContents,
}
b := len(ctx.DocumentID) != 0
if ctx.Config == nil {
panic(fmt.Sprintf("RenderingContext of %q doesn't have a config", ctx.DocumentID))
}
if b && !ctx.Config.PlainIDAnchors {
renderParameters.FootnoteAnchorPrefix = ctx.DocumentID + ":" + renderParameters.FootnoteAnchorPrefix
// renderParameters.HeaderIDSuffix = ":" + ctx.DocumentId
}
htmlFlags := defaultFlags
htmlFlags |= mmark.HTML_FOOTNOTE_RETURN_LINKS
return &HugoMmarkHTMLRenderer{
cs: c,
Renderer: mmark.HtmlRendererWithParameters(htmlFlags, "", "", renderParameters),
Cfg: c.Cfg,
}
}
func getMmarkExtensions(ctx *RenderingContext) int {
flags := 0
flags |= mmark.EXTENSION_TABLES
flags |= mmark.EXTENSION_FENCED_CODE
flags |= mmark.EXTENSION_AUTOLINK
flags |= mmark.EXTENSION_SPACE_HEADERS
flags |= mmark.EXTENSION_CITATION
flags |= mmark.EXTENSION_TITLEBLOCK_TOML
flags |= mmark.EXTENSION_HEADER_IDS
flags |= mmark.EXTENSION_AUTO_HEADER_IDS
flags |= mmark.EXTENSION_UNIQUE_HEADER_IDS
flags |= mmark.EXTENSION_FOOTNOTES
flags |= mmark.EXTENSION_SHORT_REF
flags |= mmark.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK
flags |= mmark.EXTENSION_INCLUDE
if ctx.Config == nil {
panic(fmt.Sprintf("RenderingContext of %q doesn't have a config", ctx.DocumentID))
}
for _, extension := range ctx.Config.Extensions {
if flag, ok := mmarkExtensionMap[extension]; ok {
flags |= flag
}
}
return flags
}
func (c *ContentSpec) mmarkRender(ctx *RenderingContext) []byte {
return mmark.Parse(ctx.Content, c.getMmarkHTMLRenderer(0, ctx),
getMmarkExtensions(ctx)).Bytes()
}
// ExtractTOC extracts Table of Contents from content. // ExtractTOC extracts Table of Contents from content.
func ExtractTOC(content []byte) (newcontent []byte, toc []byte) { func ExtractTOC(content []byte) (newcontent []byte, toc []byte) {
if !bytes.Contains(content, []byte("<nav>")) { if !bytes.Contains(content, []byte("<nav>")) {
@ -464,38 +212,12 @@ func ExtractTOC(content []byte) (newcontent []byte, toc []byte) {
return return
} }
// RenderingContext holds contextual information, like content and configuration, func (c *ContentSpec) RenderMarkdown(src []byte) ([]byte, error) {
// for a given content rendering. b, err := c.MardownConverter.Convert(converter.RenderContext{Src: src})
// By creating you must set the Config, otherwise it will panic. if err != nil {
type RenderingContext struct { return nil, err
BaseFs *filesystems.BaseFs
Content []byte
PageFmt string
DocumentID string
DocumentName string
Config *BlackFriday
RenderTOC bool
Cfg config.Provider
}
// RenderBytes renders a []byte.
func (c *ContentSpec) RenderBytes(ctx *RenderingContext) []byte {
switch ctx.PageFmt {
default:
return c.markdownRender(ctx)
case "markdown":
return c.markdownRender(ctx)
case "asciidoc":
return getAsciidocContent(ctx)
case "mmark":
return c.mmarkRender(ctx)
case "rst":
return getRstContent(ctx)
case "org":
return orgRender(ctx, c)
case "pandoc":
return getPandocContent(ctx)
} }
return b.Bytes(), nil
} }
// TotalWords counts instance of one or more consecutive white space // TotalWords counts instance of one or more consecutive white space
@ -622,181 +344,3 @@ func (c *ContentSpec) truncateWordsToWholeSentenceOld(content string) (string, b
return strings.Join(words[:c.summaryLength], " "), true return strings.Join(words[:c.summaryLength], " "), true
} }
func getAsciidocExecPath() string {
path, err := exec.LookPath("asciidoc")
if err != nil {
return ""
}
return path
}
func getAsciidoctorExecPath() string {
path, err := exec.LookPath("asciidoctor")
if err != nil {
return ""
}
return path
}
// HasAsciidoc returns whether Asciidoc or Asciidoctor is installed on this computer.
func HasAsciidoc() bool {
return (getAsciidoctorExecPath() != "" ||
getAsciidocExecPath() != "")
}
// getAsciidocContent calls asciidoctor or asciidoc as an external helper
// to convert AsciiDoc content to HTML.
func getAsciidocContent(ctx *RenderingContext) []byte {
var isAsciidoctor bool
path := getAsciidoctorExecPath()
if path == "" {
path = getAsciidocExecPath()
if path == "" {
jww.ERROR.Println("asciidoctor / asciidoc not found in $PATH: Please install.\n",
" Leaving AsciiDoc content unrendered.")
return ctx.Content
}
} else {
isAsciidoctor = true
}
jww.INFO.Println("Rendering", ctx.DocumentName, "with", path, "...")
args := []string{"--no-header-footer", "--safe"}
if isAsciidoctor {
// asciidoctor-specific arg to show stack traces on errors
args = append(args, "--trace")
}
args = append(args, "-")
return externallyRenderContent(ctx, path, args)
}
// HasRst returns whether rst2html is installed on this computer.
func HasRst() bool {
return getRstExecPath() != ""
}
func getRstExecPath() string {
path, err := exec.LookPath("rst2html")
if err != nil {
path, err = exec.LookPath("rst2html.py")
if err != nil {
return ""
}
}
return path
}
func getPythonExecPath() string {
path, err := exec.LookPath("python")
if err != nil {
path, err = exec.LookPath("python.exe")
if err != nil {
return ""
}
}
return path
}
// getRstContent calls the Python script rst2html as an external helper
// to convert reStructuredText content to HTML.
func getRstContent(ctx *RenderingContext) []byte {
path := getRstExecPath()
if path == "" {
jww.ERROR.Println("rst2html / rst2html.py not found in $PATH: Please install.\n",
" Leaving reStructuredText content unrendered.")
return ctx.Content
}
jww.INFO.Println("Rendering", ctx.DocumentName, "with", path, "...")
var result []byte
// certain *nix based OSs wrap executables in scripted launchers
// invoking binaries on these OSs via python interpreter causes SyntaxError
// invoke directly so that shebangs work as expected
// handle Windows manually because it doesn't do shebangs
if runtime.GOOS == "windows" {
python := getPythonExecPath()
args := []string{path, "--leave-comments", "--initial-header-level=2"}
result = externallyRenderContent(ctx, python, args)
} else {
args := []string{"--leave-comments", "--initial-header-level=2"}
result = externallyRenderContent(ctx, path, args)
}
// TODO(bep) check if rst2html has a body only option.
bodyStart := bytes.Index(result, []byte("<body>\n"))
if bodyStart < 0 {
bodyStart = -7 //compensate for length
}
bodyEnd := bytes.Index(result, []byte("\n</body>"))
if bodyEnd < 0 || bodyEnd >= len(result) {
bodyEnd = len(result) - 1
if bodyEnd < 0 {
bodyEnd = 0
}
}
return result[bodyStart+7 : bodyEnd]
}
// getPandocContent calls pandoc as an external helper to convert pandoc markdown to HTML.
func getPandocContent(ctx *RenderingContext) []byte {
path, err := exec.LookPath("pandoc")
if err != nil {
jww.ERROR.Println("pandoc not found in $PATH: Please install.\n",
" Leaving pandoc content unrendered.")
return ctx.Content
}
args := []string{"--mathjax"}
return externallyRenderContent(ctx, path, args)
}
func orgRender(ctx *RenderingContext, c *ContentSpec) []byte {
config := org.New()
config.Log = jww.WARN
config.ReadFile = func(filename string) ([]byte, error) {
return afero.ReadFile(ctx.BaseFs.Content.Fs, filename)
}
writer := org.NewHTMLWriter()
writer.HighlightCodeBlock = func(source, lang string) string {
highlightedSource, err := c.Highlight(source, lang, "")
if err != nil {
jww.ERROR.Printf("Could not highlight source as lang %s. Using raw source.", lang)
return source
}
return highlightedSource
}
html, err := config.Parse(bytes.NewReader(ctx.Content), ctx.DocumentName).Write(writer)
if err != nil {
jww.ERROR.Printf("Could not render org: %s. Using unrendered content.", err)
return ctx.Content
}
return []byte(html)
}
func externallyRenderContent(ctx *RenderingContext, path string, args []string) []byte {
content := ctx.Content
cleanContent := bytes.Replace(content, SummaryDivider, []byte(""), 1)
cmd := exec.Command(path, args...)
cmd.Stdin = bytes.NewReader(cleanContent)
var out, cmderr bytes.Buffer
cmd.Stdout = &out
cmd.Stderr = &cmderr
err := cmd.Run()
// Most external helpers exit w/ non-zero exit code only if severe, i.e.
// halting errors occurred. -> log stderr output regardless of state of err
for _, item := range strings.Split(cmderr.String(), "\n") {
item := strings.TrimSpace(item)
if item != "" {
jww.ERROR.Printf("%s: %s", ctx.DocumentName, item)
}
}
if err != nil {
jww.ERROR.Printf("%s rendering %s: %v", path, ctx.DocumentName, err)
}
return normalizeExternalHelperLineFeeds(out.Bytes())
}

141
helpers/content_renderer_test.go
View file

@ -1,141 +0,0 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package helpers
import (
"bytes"
"regexp"
"testing"
qt "github.com/frankban/quicktest"
"github.com/spf13/viper"
)
// Renders a codeblock using Blackfriday
func (c *ContentSpec) render(input string) string {
ctx := &RenderingContext{Cfg: c.Cfg, Config: c.BlackFriday}
render := c.getHTMLRenderer(0, ctx)
buf := &bytes.Buffer{}
render.BlockCode(buf, []byte(input), "html")
return buf.String()
}
// Renders a codeblock using Mmark
func (c *ContentSpec) renderWithMmark(input string) string {
ctx := &RenderingContext{Cfg: c.Cfg, Config: c.BlackFriday}
render := c.getMmarkHTMLRenderer(0, ctx)
buf := &bytes.Buffer{}
render.BlockCode(buf, []byte(input), "html", []byte(""), false, false)
return buf.String()
}
func TestCodeFence(t *testing.T) {
c := qt.New(t)
type test struct {
enabled bool
input, expected string
}
// Pygments 2.0 and 2.1 have slightly different outputs so only do partial matching
data := []test{
{true, "<html></html>", `(?s)^<div class="highlight">\n?<pre.*><code class="language-html" data-lang="html">.*?</code></pre>\n?</div>\n?$`},
{false, "<html></html>", `(?s)^<pre.*><code class="language-html">.*?</code></pre>\n$`},
}
for _, useClassic := range []bool{false, true} {
for i, d := range data {
v := viper.New()
v.Set("pygmentsStyle", "monokai")
v.Set("pygmentsUseClasses", true)
v.Set("pygmentsCodeFences", d.enabled)
v.Set("pygmentsUseClassic", useClassic)
cs, err := NewContentSpec(v)
c.Assert(err, qt.IsNil)
result := cs.render(d.input)
expectedRe, err := regexp.Compile(d.expected)
if err != nil {
t.Fatal("Invalid regexp", err)
}
matched := expectedRe.MatchString(result)
if !matched {
t.Errorf("Test %d failed. BlackFriday enabled:%t, Expected:\n%q got:\n%q", i, d.enabled, d.expected, result)
}
result = cs.renderWithMmark(d.input)
matched = expectedRe.MatchString(result)
if !matched {
t.Errorf("Test %d failed. Mmark enabled:%t, Expected:\n%q got:\n%q", i, d.enabled, d.expected, result)
}
}
}
}
func TestBlackfridayTaskList(t *testing.T) {
c := newTestContentSpec()
for i, this := range []struct {
markdown string
taskListEnabled bool
expect string
}{
{`
TODO:
- [x] On1
- [X] On2
- [ ] Off
END
`, true, `<p>TODO:</p>
<ul class="task-list">
<li><label><input type="checkbox" checked disabled class="task-list-item"> On1</label></li>
<li><label><input type="checkbox" checked disabled class="task-list-item"> On2</label></li>
<li><label><input type="checkbox" disabled class="task-list-item"> Off</label></li>
</ul>
<p>END</p>
`},
{`- [x] On1`, false, `<ul>
<li>[x] On1</li>
</ul>
`},
{`* [ ] Off
END`, true, `<ul class="task-list">
<li><label><input type="checkbox" disabled class="task-list-item"> Off</label></li>
</ul>
<p>END</p>
`},
} {
blackFridayConfig := c.BlackFriday
blackFridayConfig.TaskLists = this.taskListEnabled
ctx := &RenderingContext{Content: []byte(this.markdown), PageFmt: "markdown", Config: blackFridayConfig}
result := string(c.RenderBytes(ctx))
if result != this.expect {
t.Errorf("[%d] got \n%v but expected \n%v", i, result, this.expect)
}
}
}

235
helpers/content_test.go
View file

@ -19,11 +19,13 @@ import (
"strings" "strings"
"testing" "testing"
"github.com/spf13/afero"
"github.com/gohugoio/hugo/common/loggers"
"github.com/spf13/viper" "github.com/spf13/viper"
qt "github.com/frankban/quicktest" qt "github.com/frankban/quicktest"
"github.com/miekg/mmark"
"github.com/russross/blackfriday"
) )
const tstHTMLContent = "<!DOCTYPE html><html><head><script src=\"http://two/foobar.js\"></script></head><body><nav><ul><li hugo-nav=\"section_0\"></li><li hugo-nav=\"section_1\"></li></ul></nav><article>content <a href=\"http://two/foobar\">foobar</a>. Follow up</article><p>This is some text.<br>And some more.</p></body></html>" const tstHTMLContent = "<!DOCTYPE html><html><head><script src=\"http://two/foobar.js\"></script></head><body><nav><ul><li hugo-nav=\"section_0\"></li><li hugo-nav=\"section_1\"></li></ul></nav><article>content <a href=\"http://two/foobar\">foobar</a>. Follow up</article><p>This is some text.<br>And some more.</p></body></html>"
@ -108,7 +110,7 @@ func TestNewContentSpec(t *testing.T) {
cfg.Set("buildExpired", true) cfg.Set("buildExpired", true)
cfg.Set("buildDrafts", true) cfg.Set("buildDrafts", true)
spec, err := NewContentSpec(cfg) spec, err := NewContentSpec(cfg, loggers.NewErrorLogger(), afero.NewMemMapFs())
c.Assert(err, qt.IsNil) c.Assert(err, qt.IsNil)
c.Assert(spec.summaryLength, qt.Equals, 32) c.Assert(spec.summaryLength, qt.Equals, 32)
@ -202,233 +204,6 @@ func TestTruncateWordsByRune(t *testing.T) {
} }
} }
func TestGetHTMLRendererFlags(t *testing.T) {
c := newTestContentSpec()
ctx := &RenderingContext{Cfg: c.Cfg, Config: c.BlackFriday}
renderer := c.getHTMLRenderer(blackfriday.HTML_USE_XHTML, ctx)
flags := renderer.GetFlags()
if flags&blackfriday.HTML_USE_XHTML != blackfriday.HTML_USE_XHTML {
t.Errorf("Test flag: %d was not found amongs set flags:%d; Result: %d", blackfriday.HTML_USE_XHTML, flags, flags&blackfriday.HTML_USE_XHTML)
}
}
func TestGetHTMLRendererAllFlags(t *testing.T) {
c := newTestContentSpec()
type data struct {
testFlag int
}
allFlags := []data{
{blackfriday.HTML_USE_XHTML},
{blackfriday.HTML_FOOTNOTE_RETURN_LINKS},
{blackfriday.HTML_USE_SMARTYPANTS},
{blackfriday.HTML_SMARTYPANTS_QUOTES_NBSP},
{blackfriday.HTML_SMARTYPANTS_ANGLED_QUOTES},
{blackfriday.HTML_SMARTYPANTS_FRACTIONS},
{blackfriday.HTML_HREF_TARGET_BLANK},
{blackfriday.HTML_NOFOLLOW_LINKS},
{blackfriday.HTML_NOREFERRER_LINKS},
{blackfriday.HTML_SMARTYPANTS_DASHES},
{blackfriday.HTML_SMARTYPANTS_LATEX_DASHES},
}
defaultFlags := blackfriday.HTML_USE_XHTML
ctx := &RenderingContext{Cfg: c.Cfg, Config: c.BlackFriday}
ctx.Config.AngledQuotes = true
ctx.Config.Fractions = true
ctx.Config.HrefTargetBlank = true
ctx.Config.NofollowLinks = true
ctx.Config.NoreferrerLinks = true
ctx.Config.LatexDashes = true
ctx.Config.PlainIDAnchors = true
ctx.Config.SmartDashes = true
ctx.Config.Smartypants = true
ctx.Config.SmartypantsQuotesNBSP = true
renderer := c.getHTMLRenderer(defaultFlags, ctx)
actualFlags := renderer.GetFlags()
var expectedFlags int
//OR-ing flags together...
for _, d := range allFlags {
expectedFlags |= d.testFlag
}
if expectedFlags != actualFlags {
t.Errorf("Expected flags (%d) did not equal actual (%d) flags.", expectedFlags, actualFlags)
}
}
func TestGetHTMLRendererAnchors(t *testing.T) {
c := newTestContentSpec()
ctx := &RenderingContext{Cfg: c.Cfg, Config: c.BlackFriday}
ctx.DocumentID = "testid"
ctx.Config.PlainIDAnchors = false
actualRenderer := c.getHTMLRenderer(0, ctx)
headerBuffer := &bytes.Buffer{}
footnoteBuffer := &bytes.Buffer{}
expectedFootnoteHref := []byte("href=\"#fn:testid:href\"")
expectedHeaderID := []byte("<h1 id=\"id:testid\"></h1>\n")
actualRenderer.Header(headerBuffer, func() bool { return true }, 1, "id")
actualRenderer.FootnoteRef(footnoteBuffer, []byte("href"), 1)
if !bytes.Contains(footnoteBuffer.Bytes(), expectedFootnoteHref) {
t.Errorf("Footnote anchor prefix not applied. Actual:%s Expected:%s", footnoteBuffer.String(), expectedFootnoteHref)
}
if !bytes.Equal(headerBuffer.Bytes(), expectedHeaderID) {
t.Errorf("Header Id Postfix not applied. Actual:%s Expected:%s", headerBuffer.String(), expectedHeaderID)
}
}
func TestGetMmarkHTMLRenderer(t *testing.T) {
c := newTestContentSpec()
ctx := &RenderingContext{Cfg: c.Cfg, Config: c.BlackFriday}
ctx.DocumentID = "testid"
ctx.Config.PlainIDAnchors = false
actualRenderer := c.getMmarkHTMLRenderer(0, ctx)
headerBuffer := &bytes.Buffer{}
footnoteBuffer := &bytes.Buffer{}
expectedFootnoteHref := []byte("href=\"#fn:testid:href\"")
expectedHeaderID := []byte("<h1 id=\"id\"></h1>")
actualRenderer.FootnoteRef(footnoteBuffer, []byte("href"), 1)
actualRenderer.Header(headerBuffer, func() bool { return true }, 1, "id")
if !bytes.Contains(footnoteBuffer.Bytes(), expectedFootnoteHref) {
t.Errorf("Footnote anchor prefix not applied. Actual:%s Expected:%s", footnoteBuffer.String(), expectedFootnoteHref)
}
if bytes.Equal(headerBuffer.Bytes(), expectedHeaderID) {
t.Errorf("Header Id Postfix applied. Actual:%s Expected:%s", headerBuffer.String(), expectedHeaderID)
}
}
func TestGetMarkdownExtensionsMasksAreRemovedFromExtensions(t *testing.T) {
c := newTestContentSpec()
ctx := &RenderingContext{Cfg: c.Cfg, Config: c.BlackFriday}
ctx.Config.Extensions = []string{"headerId"}
ctx.Config.ExtensionsMask = []string{"noIntraEmphasis"}
actualFlags := getMarkdownExtensions(ctx)
if actualFlags&blackfriday.EXTENSION_NO_INTRA_EMPHASIS == blackfriday.EXTENSION_NO_INTRA_EMPHASIS {
t.Errorf("Masked out flag {%v} found amongst returned extensions.", blackfriday.EXTENSION_NO_INTRA_EMPHASIS)
}
}
func TestGetMarkdownExtensionsByDefaultAllExtensionsAreEnabled(t *testing.T) {
type data struct {
testFlag int
}
c := newTestContentSpec()
ctx := &RenderingContext{Cfg: c.Cfg, Config: c.BlackFriday}
ctx.Config.Extensions = []string{""}
ctx.Config.ExtensionsMask = []string{""}
allExtensions := []data{
{blackfriday.EXTENSION_NO_INTRA_EMPHASIS},
{blackfriday.EXTENSION_TABLES},
{blackfriday.EXTENSION_FENCED_CODE},
{blackfriday.EXTENSION_AUTOLINK},
{blackfriday.EXTENSION_STRIKETHROUGH},
// {blackfriday.EXTENSION_LAX_HTML_BLOCKS},
{blackfriday.EXTENSION_SPACE_HEADERS},
// {blackfriday.EXTENSION_HARD_LINE_BREAK},
// {blackfriday.EXTENSION_TAB_SIZE_EIGHT},
{blackfriday.EXTENSION_FOOTNOTES},
// {blackfriday.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK},
{blackfriday.EXTENSION_HEADER_IDS},
// {blackfriday.EXTENSION_TITLEBLOCK},
{blackfriday.EXTENSION_AUTO_HEADER_IDS},
{blackfriday.EXTENSION_BACKSLASH_LINE_BREAK},
{blackfriday.EXTENSION_DEFINITION_LISTS},
}
actualFlags := getMarkdownExtensions(ctx)
for _, e := range allExtensions {
if actualFlags&e.testFlag != e.testFlag {
t.Errorf("Flag %v was not found in the list of extensions.", e)
}
}
}
func TestGetMarkdownExtensionsAddingFlagsThroughRenderingContext(t *testing.T) {
c := newTestContentSpec()
ctx := &RenderingContext{Cfg: c.Cfg, Config: c.BlackFriday}
ctx.Config.Extensions = []string{"definitionLists"}
ctx.Config.ExtensionsMask = []string{""}
actualFlags := getMarkdownExtensions(ctx)
if actualFlags&blackfriday.EXTENSION_DEFINITION_LISTS != blackfriday.EXTENSION_DEFINITION_LISTS {
t.Errorf("Masked out flag {%v} found amongst returned extensions.", blackfriday.EXTENSION_DEFINITION_LISTS)
}
}
func TestGetMarkdownRenderer(t *testing.T) {
c := newTestContentSpec()
ctx := &RenderingContext{Cfg: c.Cfg, Config: c.BlackFriday}
ctx.Content = []byte("testContent")
actualRenderedMarkdown := c.markdownRender(ctx)
expectedRenderedMarkdown := []byte("<p>testContent</p>\n")
if !bytes.Equal(actualRenderedMarkdown, expectedRenderedMarkdown) {
t.Errorf("Actual rendered Markdown (%s) did not match expected markdown (%s)", actualRenderedMarkdown, expectedRenderedMarkdown)
}
}
func TestGetMarkdownRendererWithTOC(t *testing.T) {
c := newTestContentSpec()
ctx := &RenderingContext{RenderTOC: true, Cfg: c.Cfg, Config: c.BlackFriday}
ctx.Content = []byte("testContent")
actualRenderedMarkdown := c.markdownRender(ctx)
expectedRenderedMarkdown := []byte("<nav>\n</nav>\n\n<p>testContent</p>\n")
if !bytes.Equal(actualRenderedMarkdown, expectedRenderedMarkdown) {
t.Errorf("Actual rendered Markdown (%s) did not match expected markdown (%s)", actualRenderedMarkdown, expectedRenderedMarkdown)
}
}
func TestGetMmarkExtensions(t *testing.T) {
//TODO: This is doing the same just with different marks...
type data struct {
testFlag int
}
c := newTestContentSpec()
ctx := &RenderingContext{Cfg: c.Cfg, Config: c.BlackFriday}
ctx.Config.Extensions = []string{"tables"}
ctx.Config.ExtensionsMask = []string{""}
allExtensions := []data{
{mmark.EXTENSION_TABLES},
{mmark.EXTENSION_FENCED_CODE},
{mmark.EXTENSION_AUTOLINK},
{mmark.EXTENSION_SPACE_HEADERS},
{mmark.EXTENSION_CITATION},
{mmark.EXTENSION_TITLEBLOCK_TOML},
{mmark.EXTENSION_HEADER_IDS},
{mmark.EXTENSION_AUTO_HEADER_IDS},
{mmark.EXTENSION_UNIQUE_HEADER_IDS},
{mmark.EXTENSION_FOOTNOTES},
{mmark.EXTENSION_SHORT_REF},
{mmark.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK},
{mmark.EXTENSION_INCLUDE},
}
actualFlags := getMmarkExtensions(ctx)
for _, e := range allExtensions {
if actualFlags&e.testFlag != e.testFlag {
t.Errorf("Flag %v was not found in the list of extensions.", e)
}
}
}
func TestMmarkRender(t *testing.T) {
c := newTestContentSpec()
ctx := &RenderingContext{Cfg: c.Cfg, Config: c.BlackFriday}
ctx.Content = []byte("testContent")
actualRenderedMarkdown := c.mmarkRender(ctx)
expectedRenderedMarkdown := []byte("<p>testContent</p>\n")
if !bytes.Equal(actualRenderedMarkdown, expectedRenderedMarkdown) {
t.Errorf("Actual rendered Markdown (%s) did not match expected markdown (%s)", actualRenderedMarkdown, expectedRenderedMarkdown)
}
}
func TestExtractTOCNormalContent(t *testing.T) { func TestExtractTOCNormalContent(t *testing.T) {
content := []byte("<nav>\n<ul>\nTOC<li><a href=\"#") content := []byte("<nav>\n<ul>\nTOC<li><a href=\"#")

10
helpers/pygments_test.go
View file

@ -45,7 +45,7 @@ func TestParsePygmentsArgs(t *testing.T) {
v := viper.New() v := viper.New()
v.Set("pygmentsStyle", this.pygmentsStyle) v.Set("pygmentsStyle", this.pygmentsStyle)
v.Set("pygmentsUseClasses", this.pygmentsUseClasses) v.Set("pygmentsUseClasses", this.pygmentsUseClasses)
spec, err := NewContentSpec(v) spec, err := NewContentSpec(v, nil, nil)
c.Assert(err, qt.IsNil) c.Assert(err, qt.IsNil)
result1, err := spec.createPygmentsOptionsString(this.in) result1, err := spec.createPygmentsOptionsString(this.in)
@ -94,7 +94,7 @@ func TestParseDefaultPygmentsArgs(t *testing.T) {
v.Set("pygmentsUseClasses", b) v.Set("pygmentsUseClasses", b)
} }
spec, err := NewContentSpec(v) spec, err := NewContentSpec(v, nil, nil)
c.Assert(err, qt.IsNil) c.Assert(err, qt.IsNil)
result, err := spec.createPygmentsOptionsString(this.in) result, err := spec.createPygmentsOptionsString(this.in)
@ -138,7 +138,7 @@ func TestChromaHTMLHighlight(t *testing.T) {
v := viper.New() v := viper.New()
v.Set("pygmentsUseClasses", true) v.Set("pygmentsUseClasses", true)
spec, err := NewContentSpec(v) spec, err := NewContentSpec(v, nil, nil)
c.Assert(err, qt.IsNil) c.Assert(err, qt.IsNil)
result, err := spec.Highlight(`echo "Hello"`, "bash", "") result, err := spec.Highlight(`echo "Hello"`, "bash", "")
@ -206,7 +206,7 @@ func TestChromaHTMLFormatterFromOptions(t *testing.T) {
v.Set("pygmentsUseClasses", b) v.Set("pygmentsUseClasses", b)
} }
spec, err := NewContentSpec(v) spec, err := NewContentSpec(v, nil, nil)
c.Assert(err, qt.IsNil) c.Assert(err, qt.IsNil)
opts, err := spec.parsePygmentsOpts(this.in) opts, err := spec.parsePygmentsOpts(this.in)
@ -288,7 +288,7 @@ func GetTitleFunc(style string) func(s string) string {
} }
` `
spec, err := NewContentSpec(v) spec, err := NewContentSpec(v, nil, nil)
c.Assert(err, qt.IsNil) c.Assert(err, qt.IsNil)
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {

4
helpers/testhelpers_test.go
View file

@ -1,6 +1,8 @@
package helpers package helpers
import ( import (
"github.com/gohugoio/hugo/common/loggers"
"github.com/spf13/afero"
"github.com/spf13/viper" "github.com/spf13/viper"
"github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/hugofs"
@ -56,7 +58,7 @@ func newTestCfg() *viper.Viper {
func newTestContentSpec() *ContentSpec { func newTestContentSpec() *ContentSpec {
v := viper.New() v := viper.New()
spec, err := NewContentSpec(v) spec, err := NewContentSpec(v, loggers.NewErrorLogger(), afero.NewMemMapFs())
if err != nil { if err != nil {
panic(err) panic(err)
} }

6
hugolib/config.go
View file

@ -564,11 +564,6 @@ func (configLoader) mergeStringMapKeepLeft(rootKey, key string, v1, v2 config.Pr
func loadDefaultSettingsFor(v *viper.Viper) error { func loadDefaultSettingsFor(v *viper.Viper) error {
c, err := helpers.NewContentSpec(v)
if err != nil {
return err
}
v.RegisterAlias("indexes", "taxonomies") v.RegisterAlias("indexes", "taxonomies")
/* /*
@ -616,7 +611,6 @@ func loadDefaultSettingsFor(v *viper.Viper) error {
v.SetDefault("paginate", 10) v.SetDefault("paginate", 10)
v.SetDefault("paginatePath", "page") v.SetDefault("paginatePath", "page")
v.SetDefault("summaryLength", 70) v.SetDefault("summaryLength", 70)
v.SetDefault("blackfriday", c.BlackFriday)
v.SetDefault("rssLimit", -1) v.SetDefault("rssLimit", -1)
v.SetDefault("sectionPagesMenu", "") v.SetDefault("sectionPagesMenu", "")
v.SetDefault("disablePathToLower", false) v.SetDefault("disablePathToLower", false)

15
hugolib/page.go
View file

@ -23,6 +23,8 @@ import (
"sort" "sort"
"strings" "strings"
"github.com/gohugoio/hugo/markup/converter"
"github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/common/maps"
"github.com/gohugoio/hugo/hugofs/files" "github.com/gohugoio/hugo/hugofs/files"
@ -65,7 +67,7 @@ var (
type pageContext interface { type pageContext interface {
posOffset(offset int) text.Position posOffset(offset int) text.Position
wrapError(err error) error wrapError(err error) error
getRenderingConfig() *helpers.BlackFriday getContentConverter() converter.Converter
} }
// wrapErr adds some context to the given error if possible. // wrapErr adds some context to the given error if possible.
@ -299,13 +301,6 @@ func (p *pageState) Translations() page.Pages {
return p.translations return p.translations
} }
func (p *pageState) getRenderingConfig() *helpers.BlackFriday {
if p.m.renderingConfig == nil {
return p.s.ContentSpec.BlackFriday
}
return p.m.renderingConfig
}
func (ps *pageState) initCommonProviders(pp pagePaths) error { func (ps *pageState) initCommonProviders(pp pagePaths) error {
if ps.IsPage() { if ps.IsPage() {
ps.posNextPrev = &nextPrev{init: ps.s.init.prevNext} ps.posNextPrev = &nextPrev{init: ps.s.init.prevNext}
@ -516,6 +511,10 @@ func (p *pageState) wrapError(err error) error {
return err return err
} }
func (p *pageState) getContentConverter() converter.Converter {
return p.m.contentConverter
}
func (p *pageState) addResources(r ...resource.Resource) { func (p *pageState) addResources(r ...resource.Resource) {
p.resources = append(p.resources, r...) p.resources = append(p.resources, r...)
} }

34
hugolib/page__meta.go
View file

@ -21,6 +21,8 @@ import (
"strings" "strings"
"time" "time"
"github.com/gohugoio/hugo/markup/converter"
"github.com/gohugoio/hugo/hugofs/files" "github.com/gohugoio/hugo/hugofs/files"
"github.com/gohugoio/hugo/common/hugo" "github.com/gohugoio/hugo/common/hugo"
@ -29,7 +31,6 @@ import (
"github.com/gohugoio/hugo/source" "github.com/gohugoio/hugo/source"
"github.com/markbates/inflect" "github.com/markbates/inflect"
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/common/maps"
@ -123,7 +124,7 @@ type pageMeta struct {
s *Site s *Site
renderingConfig *helpers.BlackFriday contentConverter converter.Converter
} }
func (p *pageMeta) Aliases() []string { func (p *pageMeta) Aliases() []string {
@ -598,7 +599,7 @@ func (p *pageMeta) applyDefaultValues() error {
p.markup = helpers.GuessType(p.File().Ext()) p.markup = helpers.GuessType(p.File().Ext())
} }
if p.markup == "" { if p.markup == "" {
p.markup = "unknown" p.markup = "markdown"
} }
} }
@ -637,17 +638,28 @@ func (p *pageMeta) applyDefaultValues() error {
} }
} }
if !p.f.IsZero() && p.markup != "html" {
var renderingConfigOverrides map[string]interface{}
bfParam := getParamToLower(p, "blackfriday") bfParam := getParamToLower(p, "blackfriday")
if bfParam != nil { if bfParam != nil {
p.renderingConfig = p.s.ContentSpec.BlackFriday renderingConfigOverrides = cast.ToStringMap(bfParam)
// Create a copy so we can modify it.
bf := *p.s.ContentSpec.BlackFriday
p.renderingConfig = &bf
pageParam := cast.ToStringMap(bfParam)
if err := mapstructure.Decode(pageParam, &p.renderingConfig); err != nil {
return errors.WithMessage(err, "failed to decode rendering config")
} }
cp := p.s.ContentSpec.Converters.Get(p.markup)
if cp == nil {
return errors.Errorf("no content renderer found for markup %q", p.markup)
}
cpp, err := cp.New(converter.DocumentContext{
DocumentID: p.f.UniqueID(),
DocumentName: p.f.Path(),
ConfigOverrides: renderingConfigOverrides,
})
if err != nil {
return err
}
p.contentConverter = cpp
} }
return nil return nil

6
hugolib/page__output.go
View file

@ -45,8 +45,10 @@ func newPageOutput(
paginatorProvider = pag paginatorProvider = pag
} }
var contentProvider page.ContentProvider = page.NopPage var (
var tableOfContentsProvider page.TableOfContentsProvider = page.NopPage contentProvider page.ContentProvider = page.NopPage
tableOfContentsProvider page.TableOfContentsProvider = page.NopPage
)
if cp != nil { if cp != nil {
contentProvider = cp contentProvider = cp

39
hugolib/page__per_output.go
View file

@ -23,6 +23,8 @@ import (
"sync" "sync"
"unicode/utf8" "unicode/utf8"
"github.com/gohugoio/hugo/markup/converter"
"github.com/gohugoio/hugo/lazy" "github.com/gohugoio/hugo/lazy"
bp "github.com/gohugoio/hugo/bufferpool" bp "github.com/gohugoio/hugo/bufferpool"
@ -97,7 +99,12 @@ func newPageContentOutput(p *pageState) func(f output.Format) (*pageContentOutpu
if p.renderable { if p.renderable {
if !isHTML { if !isHTML {
cp.workContent = cp.renderContent(p, cp.workContent) r, err := cp.renderContent(cp.workContent)
if err != nil {
return err
}
cp.workContent = r.Bytes()
tmpContent, tmpTableOfContents := helpers.ExtractTOC(cp.workContent) tmpContent, tmpTableOfContents := helpers.ExtractTOC(cp.workContent)
cp.tableOfContents = helpers.BytesToHTML(tmpTableOfContents) cp.tableOfContents = helpers.BytesToHTML(tmpTableOfContents)
cp.workContent = tmpContent cp.workContent = tmpContent
@ -140,13 +147,16 @@ func newPageContentOutput(p *pageState) func(f output.Format) (*pageContentOutpu
} }
} }
} else if cp.p.m.summary != "" { } else if cp.p.m.summary != "" {
html := cp.p.s.ContentSpec.RenderBytes(&helpers.RenderingContext{ b, err := cp.p.getContentConverter().Convert(
Content: []byte(cp.p.m.summary), RenderTOC: false, PageFmt: cp.p.m.markup, converter.RenderContext{
Cfg: p.Language(), Src: []byte(cp.p.m.summary),
BaseFs: p.s.BaseFs, },
DocumentID: p.File().UniqueID(), DocumentName: p.File().Path(), )
Config: cp.p.getRenderingConfig()})
html = cp.p.s.ContentSpec.TrimShortHTML(html) if err != nil {
return err
}
html := cp.p.s.ContentSpec.TrimShortHTML(b.Bytes())
cp.summary = helpers.BytesToHTML(html) cp.summary = helpers.BytesToHTML(html)
} }
} }
@ -311,13 +321,12 @@ func (p *pageContentOutput) setAutoSummary() error {
} }
func (cp *pageContentOutput) renderContent(p page.Page, content []byte) []byte { func (cp *pageContentOutput) renderContent(content []byte) (converter.Result, error) {
return cp.p.s.ContentSpec.RenderBytes(&helpers.RenderingContext{ return cp.p.getContentConverter().Convert(
Content: content, RenderTOC: true, PageFmt: cp.p.m.markup, converter.RenderContext{
Cfg: p.Language(), Src: content,
BaseFs: cp.p.s.BaseFs, RenderTOC: true,
DocumentID: p.File().UniqueID(), DocumentName: p.File().Path(), })
Config: cp.p.getRenderingConfig()})
} }
func (p *pageContentOutput) setWordCounts(isCJKLanguage bool) { func (p *pageContentOutput) setWordCounts(isCJKLanguage bool) {

8
hugolib/page_test.go
View file

@ -18,6 +18,10 @@ import (
"html/template" "html/template"
"os" "os"
"github.com/gohugoio/hugo/markup/rst"
"github.com/gohugoio/hugo/markup/asciidoc"
"github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/common/loggers"
@ -378,8 +382,8 @@ func testAllMarkdownEnginesForPages(t *testing.T,
}{ }{
{"md", func() bool { return true }}, {"md", func() bool { return true }},
{"mmark", func() bool { return true }}, {"mmark", func() bool { return true }},
{"ad", func() bool { return helpers.HasAsciidoc() }}, {"ad", func() bool { return asciidoc.Supports() }},
{"rst", func() bool { return helpers.HasRst() }}, {"rst", func() bool { return rst.Supports() }},
} }
for _, e := range engines { for _, e := range engines {

23
hugolib/shortcode.go
View file

@ -21,6 +21,8 @@ import (
"html/template" "html/template"
"path" "path"
"github.com/gohugoio/hugo/markup/converter"
"github.com/gohugoio/hugo/common/herrors" "github.com/gohugoio/hugo/common/herrors"
"github.com/pkg/errors" "github.com/pkg/errors"
@ -43,7 +45,6 @@ import (
"github.com/gohugoio/hugo/output" "github.com/gohugoio/hugo/output"
bp "github.com/gohugoio/hugo/bufferpool" bp "github.com/gohugoio/hugo/bufferpool"
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/tpl" "github.com/gohugoio/hugo/tpl"
) )
@ -347,13 +348,19 @@ func renderShortcode(
// Pre Hugo 0.55 this was the behaviour even for the outer-most // Pre Hugo 0.55 this was the behaviour even for the outer-most
// shortcode. // shortcode.
if sc.doMarkup && (level > 0 || sc.info.Config.Version == 1) { if sc.doMarkup && (level > 0 || sc.info.Config.Version == 1) {
newInner := s.ContentSpec.RenderBytes(&helpers.RenderingContext{ var err error
Content: []byte(inner),
PageFmt: p.m.markup, b, err := p.getContentConverter().Convert(
Cfg: p.Language(), converter.RenderContext{
DocumentID: p.File().UniqueID(), Src: []byte(inner),
DocumentName: p.File().Path(), },
Config: p.getRenderingConfig()}) )
if err != nil {
return "", false, err
}
newInner := b.Bytes()
// If the type is “” (unknown) or “markdown”, we assume the markdown // If the type is “” (unknown) or “markdown”, we assume the markdown
// generation has been performed. Given the input: `a line`, markdown // generation has been performed. Given the input: `a line`, markdown

22
hugolib/shortcode_test.go
View file

@ -18,6 +18,9 @@ import (
"path/filepath" "path/filepath"
"reflect" "reflect"
"github.com/gohugoio/hugo/markup/asciidoc"
"github.com/gohugoio/hugo/markup/rst"
"github.com/spf13/viper" "github.com/spf13/viper"
"github.com/gohugoio/hugo/parser/pageparser" "github.com/gohugoio/hugo/parser/pageparser"
@ -27,7 +30,6 @@ import (
"testing" "testing"
"github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/deps"
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/tpl" "github.com/gohugoio/hugo/tpl"
"github.com/spf13/cast" "github.com/spf13/cast"
@ -538,6 +540,19 @@ title: "Foo"
"<h1>Hugo!</h1>"}, "<h1>Hugo!</h1>"},
} }
temp := tests[:0]
for _, test := range tests {
if strings.HasSuffix(test.contentPath, ".ad") && !asciidoc.Supports() {
t.Log("Skip Asciidoc test case as no Asciidoc present.")
continue
} else if strings.HasSuffix(test.contentPath, ".rst") && !rst.Supports() {
t.Log("Skip Rst test case as no rst2html present.")
continue
}
temp = append(temp, test)
}
tests = temp
sources := make([][2]string, len(tests)) sources := make([][2]string, len(tests))
for i, test := range tests { for i, test := range tests {
@ -578,11 +593,6 @@ title: "Foo"
test := test test := test
t.Run(fmt.Sprintf("test=%d;contentPath=%s", i, test.contentPath), func(t *testing.T) { t.Run(fmt.Sprintf("test=%d;contentPath=%s", i, test.contentPath), func(t *testing.T) {
t.Parallel() t.Parallel()
if strings.HasSuffix(test.contentPath, ".ad") && !helpers.HasAsciidoc() {
t.Skip("Skip Asciidoc test case as no Asciidoc present.")
} else if strings.HasSuffix(test.contentPath, ".rst") && !helpers.HasRst() {
t.Skip("Skip Rst test case as no rst2html present.")
}
th := newTestHelper(s.Cfg, s.Fs, t) th := newTestHelper(s.Cfg, s.Fs, t)

16
hugolib/site.go
View file

@ -28,6 +28,8 @@ import (
"strings" "strings"
"time" "time"
"github.com/gohugoio/hugo/markup/converter"
"github.com/gohugoio/hugo/hugofs/files" "github.com/gohugoio/hugo/hugofs/files"
"github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/common/maps"
@ -758,17 +760,23 @@ func (s *siteRefLinker) refLink(ref string, source interface{}, relative bool, o
} }
if refURL.Fragment != "" { if refURL.Fragment != "" {
_ = target
link = link + "#" + refURL.Fragment link = link + "#" + refURL.Fragment
if pctx, ok := target.(pageContext); ok && !target.File().IsZero() && !pctx.getRenderingConfig().PlainIDAnchors { if pctx, ok := target.(pageContext); ok {
if refURL.Path != "" { if refURL.Path != "" {
link = link + ":" + target.File().UniqueID() if di, ok := pctx.getContentConverter().(converter.DocumentInfo); ok {
link = link + di.AnchorSuffix()
}
}
} else if pctx, ok := p.(pageContext); ok {
if di, ok := pctx.getContentConverter().(converter.DocumentInfo); ok {
link = link + di.AnchorSuffix()
} }
} else if pctx, ok := p.(pageContext); ok && !p.File().IsZero() && !pctx.getRenderingConfig().PlainIDAnchors {
link = link + ":" + p.File().UniqueID()
} }
} }
return link, nil return link, nil
} }

1
hugolib/site_test.go
View file

@ -1018,6 +1018,7 @@ func TestRefLinking(t *testing.T) {
} }
func checkLinkCase(site *Site, link string, currentPage page.Page, relative bool, outputFormat string, expected string, t *testing.T, i int) { func checkLinkCase(site *Site, link string, currentPage page.Page, relative bool, outputFormat string, expected string, t *testing.T, i int) {
t.Helper()
if out, err := site.refLink(link, currentPage, relative, outputFormat); err != nil || out != expected { if out, err := site.refLink(link, currentPage, relative, outputFormat); err != nil || out != expected {
t.Fatalf("[%d] Expected %q from %q to resolve to %q, got %q - error: %s", i, link, currentPage.Path(), expected, out, err) t.Fatalf("[%d] Expected %q from %q to resolve to %q, got %q - error: %s", i, link, currentPage.Path(), expected, out, err)
} }

97
markup/asciidoc/convert.go Normal file
View file

@ -0,0 +1,97 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package asciidoc converts Asciidoc to HTML using Asciidoc or Asciidoctor
// external binaries.
package asciidoc
import (
"os/exec"
"github.com/gohugoio/hugo/markup/internal"
"github.com/gohugoio/hugo/markup/converter"
)
// Provider is the package entry point.
var Provider converter.NewProvider = provider{}
type provider struct {
}
func (p provider) New(cfg converter.ProviderConfig) (converter.Provider, error) {
var n converter.NewConverter = func(ctx converter.DocumentContext) (converter.Converter, error) {
return &asciidocConverter{
ctx: ctx,
cfg: cfg,
}, nil
}
return n, nil
}
type asciidocConverter struct {
ctx converter.DocumentContext
cfg converter.ProviderConfig
}
func (a *asciidocConverter) Convert(ctx converter.RenderContext) (converter.Result, error) {
return converter.Bytes(a.getAsciidocContent(ctx.Src, a.ctx)), nil
}
// getAsciidocContent calls asciidoctor or asciidoc as an external helper
// to convert AsciiDoc content to HTML.
func (a *asciidocConverter) getAsciidocContent(src []byte, ctx converter.DocumentContext) []byte {
var isAsciidoctor bool
path := getAsciidoctorExecPath()
if path == "" {
path = getAsciidocExecPath()
if path == "" {
a.cfg.Logger.ERROR.Println("asciidoctor / asciidoc not found in $PATH: Please install.\n",
" Leaving AsciiDoc content unrendered.")
return src
}
} else {
isAsciidoctor = true
}
a.cfg.Logger.INFO.Println("Rendering", ctx.DocumentName, "with", path, "...")
args := []string{"--no-header-footer", "--safe"}
if isAsciidoctor {
// asciidoctor-specific arg to show stack traces on errors
args = append(args, "--trace")
}
args = append(args, "-")
return internal.ExternallyRenderContent(a.cfg, ctx, src, path, args)
}
func getAsciidocExecPath() string {
path, err := exec.LookPath("asciidoc")
if err != nil {
return ""
}
return path
}
func getAsciidoctorExecPath() string {
path, err := exec.LookPath("asciidoctor")
if err != nil {
return ""
}
return path
}
// Supports returns whether Asciidoc or Asciidoctor is installed on this computer.
func Supports() bool {
return (getAsciidoctorExecPath() != "" ||
getAsciidocExecPath() != "")
}

38
markup/asciidoc/convert_test.go Normal file
View file

@ -0,0 +1,38 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package asciidoc
import (
"testing"
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/markup/converter"
qt "github.com/frankban/quicktest"
)
func TestConvert(t *testing.T) {
if !Supports() {
t.Skip("asciidoc/asciidoctor not installed")
}
c := qt.New(t)
p, err := Provider.New(converter.ProviderConfig{Logger: loggers.NewErrorLogger()})
c.Assert(err, qt.IsNil)
conv, err := p.New(converter.DocumentContext{})
c.Assert(err, qt.IsNil)
b, err := conv.Convert(converter.RenderContext{Src: []byte("testContent")})
c.Assert(err, qt.IsNil)
c.Assert(string(b.Bytes()), qt.Equals, "<div class=\"paragraph\">\n<p>testContent</p>\n</div>\n")
}

224
markup/blackfriday/convert.go Normal file
View file

@ -0,0 +1,224 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package blackfriday converts Markdown to HTML using Blackfriday v1.
package blackfriday
import (
"github.com/gohugoio/hugo/markup/converter"
"github.com/gohugoio/hugo/markup/internal"
"github.com/russross/blackfriday"
)
// Provider is the package entry point.
var Provider converter.NewProvider = provider{}
type provider struct {
}
func (p provider) New(cfg converter.ProviderConfig) (converter.Provider, error) {
defaultBlackFriday, err := internal.NewBlackfriday(cfg)
if err != nil {
return nil, err
}
defaultExtensions := getMarkdownExtensions(defaultBlackFriday)
pygmentsCodeFences := cfg.Cfg.GetBool("pygmentsCodeFences")
pygmentsCodeFencesGuessSyntax := cfg.Cfg.GetBool("pygmentsCodeFencesGuessSyntax")
pygmentsOptions := cfg.Cfg.GetString("pygmentsOptions")
var n converter.NewConverter = func(ctx converter.DocumentContext) (converter.Converter, error) {
b := defaultBlackFriday
extensions := defaultExtensions
if ctx.ConfigOverrides != nil {
var err error
b, err = internal.UpdateBlackFriday(b, ctx.ConfigOverrides)
if err != nil {
return nil, err
}
extensions = getMarkdownExtensions(b)
}
return &blackfridayConverter{
ctx: ctx,
bf: b,
extensions: extensions,
cfg: cfg,
pygmentsCodeFences: pygmentsCodeFences,
pygmentsCodeFencesGuessSyntax: pygmentsCodeFencesGuessSyntax,
pygmentsOptions: pygmentsOptions,
}, nil
}
return n, nil
}
type blackfridayConverter struct {
ctx converter.DocumentContext
bf *internal.BlackFriday
extensions int
pygmentsCodeFences bool
pygmentsCodeFencesGuessSyntax bool
pygmentsOptions string
cfg converter.ProviderConfig
}
func (c *blackfridayConverter) AnchorSuffix() string {
if c.bf.PlainIDAnchors {
return ""
}
return ":" + c.ctx.DocumentID
}
func (c *blackfridayConverter) Convert(ctx converter.RenderContext) (converter.Result, error) {
r := c.getHTMLRenderer(ctx.RenderTOC)
return converter.Bytes(blackfriday.Markdown(ctx.Src, r, c.extensions)), nil
}
func (c *blackfridayConverter) getHTMLRenderer(renderTOC bool) blackfriday.Renderer {
flags := getFlags(renderTOC, c.bf)
documentID := c.ctx.DocumentID
renderParameters := blackfriday.HtmlRendererParameters{
FootnoteAnchorPrefix: c.bf.FootnoteAnchorPrefix,
FootnoteReturnLinkContents: c.bf.FootnoteReturnLinkContents,
}
if documentID != "" && !c.bf.PlainIDAnchors {
renderParameters.FootnoteAnchorPrefix = documentID + ":" + renderParameters.FootnoteAnchorPrefix
renderParameters.HeaderIDSuffix = ":" + documentID
}
return &hugoHTMLRenderer{
c: c,
Renderer: blackfriday.HtmlRendererWithParameters(flags, "", "", renderParameters),
}
}
func getFlags(renderTOC bool, cfg *internal.BlackFriday) int {
var flags int
if renderTOC {
flags = blackfriday.HTML_TOC
}
flags |= blackfriday.HTML_USE_XHTML
flags |= blackfriday.HTML_FOOTNOTE_RETURN_LINKS
if cfg.Smartypants {
flags |= blackfriday.HTML_USE_SMARTYPANTS
}
if cfg.SmartypantsQuotesNBSP {
flags |= blackfriday.HTML_SMARTYPANTS_QUOTES_NBSP
}
if cfg.AngledQuotes {
flags |= blackfriday.HTML_SMARTYPANTS_ANGLED_QUOTES
}
if cfg.Fractions {
flags |= blackfriday.HTML_SMARTYPANTS_FRACTIONS
}
if cfg.HrefTargetBlank {
flags |= blackfriday.HTML_HREF_TARGET_BLANK
}
if cfg.NofollowLinks {
flags |= blackfriday.HTML_NOFOLLOW_LINKS
}
if cfg.NoreferrerLinks {
flags |= blackfriday.HTML_NOREFERRER_LINKS
}
if cfg.SmartDashes {
flags |= blackfriday.HTML_SMARTYPANTS_DASHES
}
if cfg.LatexDashes {
flags |= blackfriday.HTML_SMARTYPANTS_LATEX_DASHES
}
if cfg.SkipHTML {
flags |= blackfriday.HTML_SKIP_HTML
}
return flags
}
func getMarkdownExtensions(cfg *internal.BlackFriday) int {
// Default Blackfriday common extensions
commonExtensions := 0 |
blackfriday.EXTENSION_NO_INTRA_EMPHASIS |
blackfriday.EXTENSION_TABLES |
blackfriday.EXTENSION_FENCED_CODE |
blackfriday.EXTENSION_AUTOLINK |
blackfriday.EXTENSION_STRIKETHROUGH |
blackfriday.EXTENSION_SPACE_HEADERS |
blackfriday.EXTENSION_HEADER_IDS |
blackfriday.EXTENSION_BACKSLASH_LINE_BREAK |
blackfriday.EXTENSION_DEFINITION_LISTS
// Extra Blackfriday extensions that Hugo enables by default
flags := commonExtensions |
blackfriday.EXTENSION_AUTO_HEADER_IDS |
blackfriday.EXTENSION_FOOTNOTES
for _, extension := range cfg.Extensions {
if flag, ok := blackfridayExtensionMap[extension]; ok {
flags |= flag
}
}
for _, extension := range cfg.ExtensionsMask {
if flag, ok := blackfridayExtensionMap[extension]; ok {
flags &= ^flag
}
}
return flags
}
var blackfridayExtensionMap = map[string]int{
"noIntraEmphasis": blackfriday.EXTENSION_NO_INTRA_EMPHASIS,
"tables": blackfriday.EXTENSION_TABLES,
"fencedCode": blackfriday.EXTENSION_FENCED_CODE,
"autolink": blackfriday.EXTENSION_AUTOLINK,
"strikethrough": blackfriday.EXTENSION_STRIKETHROUGH,
"laxHtmlBlocks": blackfriday.EXTENSION_LAX_HTML_BLOCKS,
"spaceHeaders": blackfriday.EXTENSION_SPACE_HEADERS,
"hardLineBreak": blackfriday.EXTENSION_HARD_LINE_BREAK,
"tabSizeEight": blackfriday.EXTENSION_TAB_SIZE_EIGHT,
"footnotes": blackfriday.EXTENSION_FOOTNOTES,
"noEmptyLineBeforeBlock": blackfriday.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK,
"headerIds": blackfriday.EXTENSION_HEADER_IDS,
"titleblock": blackfriday.EXTENSION_TITLEBLOCK,
"autoHeaderIds": blackfriday.EXTENSION_AUTO_HEADER_IDS,
"backslashLineBreak": blackfriday.EXTENSION_BACKSLASH_LINE_BREAK,
"definitionLists": blackfriday.EXTENSION_DEFINITION_LISTS,
"joinLines": blackfriday.EXTENSION_JOIN_LINES,
}
var (
_ converter.DocumentInfo = (*blackfridayConverter)(nil)
)

194
markup/blackfriday/convert_test.go Normal file
View file

@ -0,0 +1,194 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package blackfriday
import (
"testing"
"github.com/spf13/viper"
"github.com/gohugoio/hugo/markup/internal"
"github.com/gohugoio/hugo/markup/converter"
qt "github.com/frankban/quicktest"
"github.com/russross/blackfriday"
)
func TestGetMarkdownExtensionsMasksAreRemovedFromExtensions(t *testing.T) {
c := qt.New(t)
b, err := internal.NewBlackfriday(converter.ProviderConfig{Cfg: viper.New()})
c.Assert(err, qt.IsNil)
b.Extensions = []string{"headerId"}
b.ExtensionsMask = []string{"noIntraEmphasis"}
actualFlags := getMarkdownExtensions(b)
if actualFlags&blackfriday.EXTENSION_NO_INTRA_EMPHASIS == blackfriday.EXTENSION_NO_INTRA_EMPHASIS {
t.Errorf("Masked out flag {%v} found amongst returned extensions.", blackfriday.EXTENSION_NO_INTRA_EMPHASIS)
}
}
func TestGetMarkdownExtensionsByDefaultAllExtensionsAreEnabled(t *testing.T) {
type data struct {
testFlag int
}
c := qt.New(t)
b, err := internal.NewBlackfriday(converter.ProviderConfig{Cfg: viper.New()})
c.Assert(err, qt.IsNil)
b.Extensions = []string{""}
b.ExtensionsMask = []string{""}
allExtensions := []data{
{blackfriday.EXTENSION_NO_INTRA_EMPHASIS},
{blackfriday.EXTENSION_TABLES},
{blackfriday.EXTENSION_FENCED_CODE},
{blackfriday.EXTENSION_AUTOLINK},
{blackfriday.EXTENSION_STRIKETHROUGH},
// {blackfriday.EXTENSION_LAX_HTML_BLOCKS},
{blackfriday.EXTENSION_SPACE_HEADERS},
// {blackfriday.EXTENSION_HARD_LINE_BREAK},
// {blackfriday.EXTENSION_TAB_SIZE_EIGHT},
{blackfriday.EXTENSION_FOOTNOTES},
// {blackfriday.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK},
{blackfriday.EXTENSION_HEADER_IDS},
// {blackfriday.EXTENSION_TITLEBLOCK},
{blackfriday.EXTENSION_AUTO_HEADER_IDS},
{blackfriday.EXTENSION_BACKSLASH_LINE_BREAK},
{blackfriday.EXTENSION_DEFINITION_LISTS},
}
actualFlags := getMarkdownExtensions(b)
for _, e := range allExtensions {
if actualFlags&e.testFlag != e.testFlag {
t.Errorf("Flag %v was not found in the list of extensions.", e)
}
}
}
func TestGetMarkdownExtensionsAddingFlagsThroughRenderingContext(t *testing.T) {
c := qt.New(t)
b, err := internal.NewBlackfriday(converter.ProviderConfig{Cfg: viper.New()})
c.Assert(err, qt.IsNil)
b.Extensions = []string{"definitionLists"}
b.ExtensionsMask = []string{""}
actualFlags := getMarkdownExtensions(b)
if actualFlags&blackfriday.EXTENSION_DEFINITION_LISTS != blackfriday.EXTENSION_DEFINITION_LISTS {
t.Errorf("Masked out flag {%v} found amongst returned extensions.", blackfriday.EXTENSION_DEFINITION_LISTS)
}
}
func TestGetFlags(t *testing.T) {
c := qt.New(t)
cfg := converter.ProviderConfig{Cfg: viper.New()}
b, err := internal.NewBlackfriday(cfg)
c.Assert(err, qt.IsNil)
flags := getFlags(false, b)
if flags&blackfriday.HTML_USE_XHTML != blackfriday.HTML_USE_XHTML {
t.Errorf("Test flag: %d was not found amongs set flags:%d; Result: %d", blackfriday.HTML_USE_XHTML, flags, flags&blackfriday.HTML_USE_XHTML)
}
}
func TestGetAllFlags(t *testing.T) {
c := qt.New(t)
cfg := converter.ProviderConfig{Cfg: viper.New()}
b, err := internal.NewBlackfriday(cfg)
c.Assert(err, qt.IsNil)
type data struct {
testFlag int
}
allFlags := []data{
{blackfriday.HTML_USE_XHTML},
{blackfriday.HTML_FOOTNOTE_RETURN_LINKS},
{blackfriday.HTML_USE_SMARTYPANTS},
{blackfriday.HTML_SMARTYPANTS_QUOTES_NBSP},
{blackfriday.HTML_SMARTYPANTS_ANGLED_QUOTES},
{blackfriday.HTML_SMARTYPANTS_FRACTIONS},
{blackfriday.HTML_HREF_TARGET_BLANK},
{blackfriday.HTML_NOFOLLOW_LINKS},
{blackfriday.HTML_NOREFERRER_LINKS},
{blackfriday.HTML_SMARTYPANTS_DASHES},
{blackfriday.HTML_SMARTYPANTS_LATEX_DASHES},
}
b.AngledQuotes = true
b.Fractions = true
b.HrefTargetBlank = true
b.NofollowLinks = true
b.NoreferrerLinks = true
b.LatexDashes = true
b.PlainIDAnchors = true
b.SmartDashes = true
b.Smartypants = true
b.SmartypantsQuotesNBSP = true
actualFlags := getFlags(false, b)
var expectedFlags int
//OR-ing flags together...
for _, d := range allFlags {
expectedFlags |= d.testFlag
}
if expectedFlags != actualFlags {
t.Errorf("Expected flags (%d) did not equal actual (%d) flags.", expectedFlags, actualFlags)
}
}
func TestConvert(t *testing.T) {
c := qt.New(t)
p, err := Provider.New(converter.ProviderConfig{
Cfg: viper.New(),
})
c.Assert(err, qt.IsNil)
conv, err := p.New(converter.DocumentContext{})
c.Assert(err, qt.IsNil)
b, err := conv.Convert(converter.RenderContext{Src: []byte("testContent")})
c.Assert(err, qt.IsNil)
c.Assert(string(b.Bytes()), qt.Equals, "<p>testContent</p>\n")
}
func TestGetHTMLRendererAnchors(t *testing.T) {
c := qt.New(t)
p, err := Provider.New(converter.ProviderConfig{
Cfg: viper.New(),
})
c.Assert(err, qt.IsNil)
conv, err := p.New(converter.DocumentContext{
DocumentID: "testid",
ConfigOverrides: map[string]interface{}{
"plainIDAnchors": false,
"footnotes": true,
},
})
c.Assert(err, qt.IsNil)
b, err := conv.Convert(converter.RenderContext{Src: []byte(`# Header
This is a footnote.[^1] And then some.
[^1]: Footnote text.
`)})
c.Assert(err, qt.IsNil)
s := string(b.Bytes())
c.Assert(s, qt.Contains, "<h1 id=\"header:testid\">Header</h1>")
c.Assert(s, qt.Contains, "This is a footnote.<sup class=\"footnote-ref\" id=\"fnref:testid:1\"><a href=\"#fn:testid:1\">1</a></sup>")
c.Assert(s, qt.Contains, "<a class=\"footnote-return\" href=\"#fnref:testid:1\"><sup>[return]</sup></a>")
}

51
helpers/content_renderer.go → markup/blackfriday/renderer.go
View file

@ -1,4 +1,4 @@
// Copyright 2016 The Hugo Authors. All rights reserved. // Copyright 2019 The Hugo Authors. All rights reserved.
// //
// Licensed under the Apache License, Version 2.0 (the "License"); // Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. // you may not use this file except in compliance with the License.
@ -11,32 +11,29 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
package helpers package blackfriday
import ( import (
"bytes" "bytes"
"strings" "strings"
"github.com/gohugoio/hugo/config"
"github.com/miekg/mmark"
"github.com/russross/blackfriday" "github.com/russross/blackfriday"
) )
// HugoHTMLRenderer wraps a blackfriday.Renderer, typically a blackfriday.Html // hugoHTMLRenderer wraps a blackfriday.Renderer, typically a blackfriday.Html
// Enabling Hugo to customise the rendering experience // adding some custom behaviour.
type HugoHTMLRenderer struct { type hugoHTMLRenderer struct {
cs *ContentSpec c *blackfridayConverter
*RenderingContext
blackfriday.Renderer blackfriday.Renderer
} }
// BlockCode renders a given text as a block of code. // BlockCode renders a given text as a block of code.
// Pygments is used if it is setup to handle code fences. // Pygments is used if it is setup to handle code fences.
func (r *HugoHTMLRenderer) BlockCode(out *bytes.Buffer, text []byte, lang string) { func (r *hugoHTMLRenderer) BlockCode(out *bytes.Buffer, text []byte, lang string) {
if r.Cfg.GetBool("pygmentsCodeFences") && (lang != "" || r.Cfg.GetBool("pygmentsCodeFencesGuessSyntax")) { if r.c.pygmentsCodeFences && (lang != "" || r.c.pygmentsCodeFencesGuessSyntax) {
opts := r.Cfg.GetString("pygmentsOptions") opts := r.c.pygmentsOptions
str := strings.Trim(string(text), "\n\r") str := strings.Trim(string(text), "\n\r")
highlighted, _ := r.cs.Highlight(str, lang, opts) highlighted, _ := r.c.cfg.Highlight(str, lang, opts)
out.WriteString(highlighted) out.WriteString(highlighted)
} else { } else {
r.Renderer.BlockCode(out, text, lang) r.Renderer.BlockCode(out, text, lang)
@ -44,8 +41,8 @@ func (r *HugoHTMLRenderer) BlockCode(out *bytes.Buffer, text []byte, lang string
} }
// ListItem adds task list support to the Blackfriday renderer. // ListItem adds task list support to the Blackfriday renderer.
func (r *HugoHTMLRenderer) ListItem(out *bytes.Buffer, text []byte, flags int) { func (r *hugoHTMLRenderer) ListItem(out *bytes.Buffer, text []byte, flags int) {
if !r.Config.TaskLists { if !r.c.bf.TaskLists {
r.Renderer.ListItem(out, text, flags) r.Renderer.ListItem(out, text, flags)
return return
} }
@ -64,8 +61,8 @@ func (r *HugoHTMLRenderer) ListItem(out *bytes.Buffer, text []byte, flags int) {
} }
// List adds task list support to the Blackfriday renderer. // List adds task list support to the Blackfriday renderer.
func (r *HugoHTMLRenderer) List(out *bytes.Buffer, text func() bool, flags int) { func (r *hugoHTMLRenderer) List(out *bytes.Buffer, text func() bool, flags int) {
if !r.Config.TaskLists { if !r.c.bf.TaskLists {
r.Renderer.List(out, text, flags) r.Renderer.List(out, text, flags)
return return
} }
@ -86,23 +83,3 @@ func (r *HugoHTMLRenderer) List(out *bytes.Buffer, text func() bool, flags int)
} }
} }
} }
// HugoMmarkHTMLRenderer wraps a mmark.Renderer, typically a mmark.html,
// enabling Hugo to customise the rendering experience.
type HugoMmarkHTMLRenderer struct {
cs *ContentSpec
mmark.Renderer
Cfg config.Provider
}
// BlockCode renders a given text as a block of code.
// Pygments is used if it is setup to handle code fences.
func (r *HugoMmarkHTMLRenderer) BlockCode(out *bytes.Buffer, text []byte, lang string, caption []byte, subfigure bool, callouts bool) {
if r.Cfg.GetBool("pygmentsCodeFences") && (lang != "" || r.Cfg.GetBool("pygmentsCodeFencesGuessSyntax")) {
str := strings.Trim(string(text), "\n\r")
highlighted, _ := r.cs.Highlight(str, lang, "")
out.WriteString(highlighted)
} else {
r.Renderer.BlockCode(out, text, lang, caption, subfigure, callouts)
}
}

83
markup/converter/converter.go Normal file
View file

@ -0,0 +1,83 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package converter
import (
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/config"
"github.com/spf13/afero"
)
// ProviderConfig configures a new Provider.
type ProviderConfig struct {
Cfg config.Provider // Site config
ContentFs afero.Fs
Logger *loggers.Logger
Highlight func(code, lang, optsStr string) (string, error)
}
// NewProvider creates converter providers.
type NewProvider interface {
New(cfg ProviderConfig) (Provider, error)
}
// Provider creates converters.
type Provider interface {
New(ctx DocumentContext) (Converter, error)
}
// NewConverter is an adapter that can be used as a ConverterProvider.
type NewConverter func(ctx DocumentContext) (Converter, error)
// New creates a new Converter for the given ctx.
func (n NewConverter) New(ctx DocumentContext) (Converter, error) {
return n(ctx)
}
// Converter wraps the Convert method that converts some markup into
// another format, e.g. Markdown to HTML.
type Converter interface {
Convert(ctx RenderContext) (Result, error)
}
// Result represents the minimum returned from Convert.
type Result interface {
Bytes() []byte
}
// DocumentInfo holds additional information provided by some converters.
type DocumentInfo interface {
AnchorSuffix() string
}
// Bytes holds a byte slice and implements the Result interface.
type Bytes []byte
// Bytes returns itself
func (b Bytes) Bytes() []byte {
return b
}
// DocumentContext holds contextual information about the document to convert.
type DocumentContext struct {
DocumentID string
DocumentName string
ConfigOverrides map[string]interface{}
}
// RenderContext holds contextual information about the content to render.
type RenderContext struct {
Src []byte
RenderTOC bool
}

108
markup/internal/blackfriday.go Normal file
View file

@ -0,0 +1,108 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package helpers implements general utility functions that work with
// and on content. The helper functions defined here lay down the
// foundation of how Hugo works with files and filepaths, and perform
// string operations on content.
package internal
import (
"github.com/gohugoio/hugo/common/maps"
"github.com/gohugoio/hugo/markup/converter"
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
)
// BlackFriday holds configuration values for BlackFriday rendering.
// It is kept here because it's used in several packages.
type BlackFriday struct {
Smartypants bool
SmartypantsQuotesNBSP bool
AngledQuotes bool
Fractions bool
HrefTargetBlank bool
NofollowLinks bool
NoreferrerLinks bool
SmartDashes bool
LatexDashes bool
TaskLists bool
PlainIDAnchors bool
Extensions []string
ExtensionsMask []string
SkipHTML bool
FootnoteAnchorPrefix string
FootnoteReturnLinkContents string
}
func UpdateBlackFriday(old *BlackFriday, m map[string]interface{}) (*BlackFriday, error) {
// Create a copy so we can modify it.
bf := *old
if err := mapstructure.Decode(m, &bf); err != nil {
return nil, errors.WithMessage(err, "failed to decode rendering config")
}
return &bf, nil
}
// NewBlackfriday creates a new Blackfriday filled with site config or some sane defaults.
func NewBlackfriday(cfg converter.ProviderConfig) (*BlackFriday, error) {
var siteConfig map[string]interface{}
if cfg.Cfg != nil {
siteConfig = cfg.Cfg.GetStringMap("blackfriday")
}
defaultParam := map[string]interface{}{
"smartypants": true,
"angledQuotes": false,
"smartypantsQuotesNBSP": false,
"fractions": true,
"hrefTargetBlank": false,
"nofollowLinks": false,
"noreferrerLinks": false,
"smartDashes": true,
"latexDashes": true,
"plainIDAnchors": true,
"taskLists": true,
"skipHTML": false,
}
maps.ToLower(defaultParam)
config := make(map[string]interface{})
for k, v := range defaultParam {
config[k] = v
}
for k, v := range siteConfig {
config[k] = v
}
combinedConfig := &BlackFriday{}
if err := mapstructure.Decode(config, combinedConfig); err != nil {
return nil, errors.Errorf("failed to decode Blackfriday config: %s", err)
}
// TODO(bep) update/consolidate docs
if combinedConfig.FootnoteAnchorPrefix == "" {
combinedConfig.FootnoteAnchorPrefix = cfg.Cfg.GetString("footnoteAnchorPrefix")
}
if combinedConfig.FootnoteReturnLinkContents == "" {
combinedConfig.FootnoteReturnLinkContents = cfg.Cfg.GetString("footnoteReturnLinkContents")
}
return combinedConfig, nil
}

52
markup/internal/external.go Normal file
View file

@ -0,0 +1,52 @@
package internal
import (
"bytes"
"os/exec"
"strings"
"github.com/gohugoio/hugo/markup/converter"
)
func ExternallyRenderContent(
cfg converter.ProviderConfig,
ctx converter.DocumentContext,
content []byte, path string, args []string) []byte {
logger := cfg.Logger
cmd := exec.Command(path, args...)
cmd.Stdin = bytes.NewReader(content)
var out, cmderr bytes.Buffer
cmd.Stdout = &out
cmd.Stderr = &cmderr
err := cmd.Run()
// Most external helpers exit w/ non-zero exit code only if severe, i.e.
// halting errors occurred. -> log stderr output regardless of state of err
for _, item := range strings.Split(cmderr.String(), "\n") {
item := strings.TrimSpace(item)
if item != "" {
logger.ERROR.Printf("%s: %s", ctx.DocumentName, item)
}
}
if err != nil {
logger.ERROR.Printf("%s rendering %s: %v", path, ctx.DocumentName, err)
}
return normalizeExternalHelperLineFeeds(out.Bytes())
}
// Strips carriage returns from third-party / external processes (useful for Windows)
func normalizeExternalHelperLineFeeds(content []byte) []byte {
return bytes.Replace(content, []byte("\r"), []byte(""), -1)
}
func GetPythonExecPath() string {
path, err := exec.LookPath("python")
if err != nil {
path, err = exec.LookPath("python.exe")
if err != nil {
return ""
}
}
return path
}

83
markup/markup.go Normal file
View file

@ -0,0 +1,83 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package markup
import (
"strings"
"github.com/gohugoio/hugo/markup/org"
"github.com/gohugoio/hugo/markup/asciidoc"
"github.com/gohugoio/hugo/markup/blackfriday"
"github.com/gohugoio/hugo/markup/converter"
"github.com/gohugoio/hugo/markup/mmark"
"github.com/gohugoio/hugo/markup/pandoc"
"github.com/gohugoio/hugo/markup/rst"
)
func NewConverterProvider(cfg converter.ProviderConfig) (ConverterProvider, error) {
converters := make(map[string]converter.Provider)
add := func(p converter.NewProvider, aliases ...string) error {
c, err := p.New(cfg)
if err != nil {
return err
}
addConverter(converters, c, aliases...)
return nil
}
if err := add(blackfriday.Provider, "md", "markdown", "blackfriday"); err != nil {
return nil, err
}
if err := add(mmark.Provider, "mmark"); err != nil {
return nil, err
}
if err := add(asciidoc.Provider, "asciidoc"); err != nil {
return nil, err
}
if err := add(rst.Provider, "rst"); err != nil {
return nil, err
}
if err := add(pandoc.Provider, "pandoc"); err != nil {
return nil, err
}
if err := add(org.Provider, "org"); err != nil {
return nil, err
}
return &converterRegistry{converters: converters}, nil
}
type ConverterProvider interface {
Get(name string) converter.Provider
}
type converterRegistry struct {
// Maps name (md, markdown, blackfriday etc.) to a converter provider.
// Note that this is also used for aliasing, so the same converter
// may be registered multiple times.
// All names are lower case.
converters map[string]converter.Provider
}
func (r *converterRegistry) Get(name string) converter.Provider {
return r.converters[strings.ToLower(name)]
}
func addConverter(m map[string]converter.Provider, c converter.Provider, aliases ...string) {
for _, alias := range aliases {
m[alias] = c
}
}

41
markup/markup_test.go Normal file
View file

@ -0,0 +1,41 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package markup
import (
"testing"
"github.com/spf13/viper"
"github.com/gohugoio/hugo/markup/converter"
qt "github.com/frankban/quicktest"
)
func TestConverterRegistry(t *testing.T) {
c := qt.New(t)
r, err := NewConverterProvider(converter.ProviderConfig{Cfg: viper.New()})
c.Assert(err, qt.IsNil)
c.Assert(r.Get("foo"), qt.IsNil)
c.Assert(r.Get("markdown"), qt.Not(qt.IsNil))
c.Assert(r.Get("mmark"), qt.Not(qt.IsNil))
c.Assert(r.Get("asciidoc"), qt.Not(qt.IsNil))
c.Assert(r.Get("rst"), qt.Not(qt.IsNil))
c.Assert(r.Get("pandoc"), qt.Not(qt.IsNil))
c.Assert(r.Get("org"), qt.Not(qt.IsNil))
}

143
markup/mmark/convert.go Normal file
View file

@ -0,0 +1,143 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package mmark converts Markdown to HTML using MMark v1.
package mmark
import (
"github.com/gohugoio/hugo/markup/internal"
"github.com/gohugoio/hugo/markup/converter"
"github.com/miekg/mmark"
)
// Provider is the package entry point.
var Provider converter.NewProvider = provider{}
type provider struct {
}
func (p provider) New(cfg converter.ProviderConfig) (converter.Provider, error) {
defaultBlackFriday, err := internal.NewBlackfriday(cfg)
if err != nil {
return nil, err
}
defaultExtensions := getMmarkExtensions(defaultBlackFriday)
var n converter.NewConverter = func(ctx converter.DocumentContext) (converter.Converter, error) {
b := defaultBlackFriday
extensions := defaultExtensions
if ctx.ConfigOverrides != nil {
var err error
b, err = internal.UpdateBlackFriday(b, ctx.ConfigOverrides)
if err != nil {
return nil, err
}
extensions = getMmarkExtensions(b)
}
return &mmarkConverter{
ctx: ctx,
b: b,
extensions: extensions,
cfg: cfg,
}, nil
}
return n, nil
}
type mmarkConverter struct {
ctx converter.DocumentContext
extensions int
b *internal.BlackFriday
cfg converter.ProviderConfig
}
func (c *mmarkConverter) Convert(ctx converter.RenderContext) (converter.Result, error) {
r := getHTMLRenderer(c.ctx, c.b, c.cfg)
return mmark.Parse(ctx.Src, r, c.extensions), nil
}
func getHTMLRenderer(
ctx converter.DocumentContext,
cfg *internal.BlackFriday,
pcfg converter.ProviderConfig) mmark.Renderer {
var (
flags int
documentID string
)
documentID = ctx.DocumentID
renderParameters := mmark.HtmlRendererParameters{
FootnoteAnchorPrefix: cfg.FootnoteAnchorPrefix,
FootnoteReturnLinkContents: cfg.FootnoteReturnLinkContents,
}
if documentID != "" && !cfg.PlainIDAnchors {
renderParameters.FootnoteAnchorPrefix = documentID + ":" + renderParameters.FootnoteAnchorPrefix
}
htmlFlags := flags
htmlFlags |= mmark.HTML_FOOTNOTE_RETURN_LINKS
return &mmarkRenderer{
Config: cfg,
Cfg: pcfg.Cfg,
highlight: pcfg.Highlight,
Renderer: mmark.HtmlRendererWithParameters(htmlFlags, "", "", renderParameters),
}
}
func getMmarkExtensions(cfg *internal.BlackFriday) int {
flags := 0
flags |= mmark.EXTENSION_TABLES
flags |= mmark.EXTENSION_FENCED_CODE
flags |= mmark.EXTENSION_AUTOLINK
flags |= mmark.EXTENSION_SPACE_HEADERS
flags |= mmark.EXTENSION_CITATION
flags |= mmark.EXTENSION_TITLEBLOCK_TOML
flags |= mmark.EXTENSION_HEADER_IDS
flags |= mmark.EXTENSION_AUTO_HEADER_IDS
flags |= mmark.EXTENSION_UNIQUE_HEADER_IDS
flags |= mmark.EXTENSION_FOOTNOTES
flags |= mmark.EXTENSION_SHORT_REF
flags |= mmark.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK
flags |= mmark.EXTENSION_INCLUDE
for _, extension := range cfg.Extensions {
if flag, ok := mmarkExtensionMap[extension]; ok {
flags |= flag
}
}
return flags
}
var mmarkExtensionMap = map[string]int{
"tables": mmark.EXTENSION_TABLES,
"fencedCode": mmark.EXTENSION_FENCED_CODE,
"autolink": mmark.EXTENSION_AUTOLINK,
"laxHtmlBlocks": mmark.EXTENSION_LAX_HTML_BLOCKS,
"spaceHeaders": mmark.EXTENSION_SPACE_HEADERS,
"hardLineBreak": mmark.EXTENSION_HARD_LINE_BREAK,
"footnotes": mmark.EXTENSION_FOOTNOTES,
"noEmptyLineBeforeBlock": mmark.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK,
"headerIds": mmark.EXTENSION_HEADER_IDS,
"autoHeaderIds": mmark.EXTENSION_AUTO_HEADER_IDS,
}

77
markup/mmark/convert_test.go Normal file
View file

@ -0,0 +1,77 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package mmark
import (
"testing"
"github.com/spf13/viper"
"github.com/gohugoio/hugo/common/loggers"
"github.com/miekg/mmark"
"github.com/gohugoio/hugo/markup/internal"
"github.com/gohugoio/hugo/markup/converter"
qt "github.com/frankban/quicktest"
)
func TestGetMmarkExtensions(t *testing.T) {
c := qt.New(t)
b, err := internal.NewBlackfriday(converter.ProviderConfig{Cfg: viper.New()})
c.Assert(err, qt.IsNil)
//TODO: This is doing the same just with different marks...
type data struct {
testFlag int
}
b.Extensions = []string{"tables"}
b.ExtensionsMask = []string{""}
allExtensions := []data{
{mmark.EXTENSION_TABLES},
{mmark.EXTENSION_FENCED_CODE},
{mmark.EXTENSION_AUTOLINK},
{mmark.EXTENSION_SPACE_HEADERS},
{mmark.EXTENSION_CITATION},
{mmark.EXTENSION_TITLEBLOCK_TOML},
{mmark.EXTENSION_HEADER_IDS},
{mmark.EXTENSION_AUTO_HEADER_IDS},
{mmark.EXTENSION_UNIQUE_HEADER_IDS},
{mmark.EXTENSION_FOOTNOTES},
{mmark.EXTENSION_SHORT_REF},
{mmark.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK},
{mmark.EXTENSION_INCLUDE},
}
actualFlags := getMmarkExtensions(b)
for _, e := range allExtensions {
if actualFlags&e.testFlag != e.testFlag {
t.Errorf("Flag %v was not found in the list of extensions.", e)
}
}
}
func TestConvert(t *testing.T) {
c := qt.New(t)
p, err := Provider.New(converter.ProviderConfig{Cfg: viper.New(), Logger: loggers.NewErrorLogger()})
c.Assert(err, qt.IsNil)
conv, err := p.New(converter.DocumentContext{})
c.Assert(err, qt.IsNil)
b, err := conv.Convert(converter.RenderContext{Src: []byte("testContent")})
c.Assert(err, qt.IsNil)
c.Assert(string(b.Bytes()), qt.Equals, "<p>testContent</p>\n")
}

44
markup/mmark/renderer.go Normal file
View file

@ -0,0 +1,44 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package mmark
import (
"bytes"
"strings"
"github.com/miekg/mmark"
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/markup/internal"
)
// hugoHTMLRenderer wraps a blackfriday.Renderer, typically a blackfriday.Html
// adding some custom behaviour.
type mmarkRenderer struct {
Cfg config.Provider
Config *internal.BlackFriday
highlight func(code, lang, optsStr string) (string, error)
mmark.Renderer
}
// BlockCode renders a given text as a block of code.
func (r *mmarkRenderer) BlockCode(out *bytes.Buffer, text []byte, lang string, caption []byte, subfigure bool, callouts bool) {
if r.Cfg.GetBool("pygmentsCodeFences") && (lang != "" || r.Cfg.GetBool("pygmentsCodeFencesGuessSyntax")) {
str := strings.Trim(string(text), "\n\r")
highlighted, _ := r.highlight(str, lang, "")
out.WriteString(highlighted)
} else {
r.Renderer.BlockCode(out, text, lang, caption, subfigure, callouts)
}
}

69
markup/org/convert.go Normal file
View file

@ -0,0 +1,69 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package org converts Emacs Org-Mode to HTML.
package org
import (
"bytes"
"github.com/gohugoio/hugo/markup/converter"
"github.com/niklasfasching/go-org/org"
"github.com/spf13/afero"
)
// Provider is the package entry point.
var Provider converter.NewProvider = provide{}
type provide struct {
}
func (p provide) New(cfg converter.ProviderConfig) (converter.Provider, error) {
var n converter.NewConverter = func(ctx converter.DocumentContext) (converter.Converter, error) {
return &orgConverter{
ctx: ctx,
cfg: cfg,
}, nil
}
return n, nil
}
type orgConverter struct {
ctx converter.DocumentContext
cfg converter.ProviderConfig
}
func (c *orgConverter) Convert(ctx converter.RenderContext) (converter.Result, error) {
logger := c.cfg.Logger
config := org.New()
config.Log = logger.WARN
config.ReadFile = func(filename string) ([]byte, error) {
return afero.ReadFile(c.cfg.ContentFs, filename)
}
writer := org.NewHTMLWriter()
writer.HighlightCodeBlock = func(source, lang string) string {
highlightedSource, err := c.cfg.Highlight(source, lang, "")
if err != nil {
logger.ERROR.Printf("Could not highlight source as lang %s. Using raw source.", lang)
return source
}
return highlightedSource
}
html, err := config.Parse(bytes.NewReader(ctx.Src), c.ctx.DocumentName).Write(writer)
if err != nil {
logger.ERROR.Printf("Could not render org: %s. Using unrendered content.", err)
return converter.Bytes(ctx.Src), nil
}
return converter.Bytes([]byte(html)), nil
}

35
markup/org/convert_test.go Normal file
View file

@ -0,0 +1,35 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org
import (
"testing"
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/markup/converter"
qt "github.com/frankban/quicktest"
)
func TestConvert(t *testing.T) {
c := qt.New(t)
p, err := Provider.New(converter.ProviderConfig{Logger: loggers.NewErrorLogger()})
c.Assert(err, qt.IsNil)
conv, err := p.New(converter.DocumentContext{})
c.Assert(err, qt.IsNil)
b, err := conv.Convert(converter.RenderContext{Src: []byte("testContent")})
c.Assert(err, qt.IsNil)
c.Assert(string(b.Bytes()), qt.Equals, "<p>\ntestContent\n</p>\n")
}

76
markup/pandoc/convert.go Normal file
View file

@ -0,0 +1,76 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package pandoc converts content to HTML using Pandoc as an external helper.
package pandoc
import (
"os/exec"
"github.com/gohugoio/hugo/markup/internal"
"github.com/gohugoio/hugo/markup/converter"
)
// Provider is the package entry point.
var Provider converter.NewProvider = provider{}
type provider struct {
}
func (p provider) New(cfg converter.ProviderConfig) (converter.Provider, error) {
var n converter.NewConverter = func(ctx converter.DocumentContext) (converter.Converter, error) {
return &pandocConverter{
ctx: ctx,
cfg: cfg,
}, nil
}
return n, nil
}
type pandocConverter struct {
ctx converter.DocumentContext
cfg converter.ProviderConfig
}
func (c *pandocConverter) Convert(ctx converter.RenderContext) (converter.Result, error) {
return converter.Bytes(c.getPandocContent(ctx.Src, c.ctx)), nil
}
// getPandocContent calls pandoc as an external helper to convert pandoc markdown to HTML.
func (c *pandocConverter) getPandocContent(src []byte, ctx converter.DocumentContext) []byte {
logger := c.cfg.Logger
path := getPandocExecPath()
if path == "" {
logger.ERROR.Println("pandoc not found in $PATH: Please install.\n",
" Leaving pandoc content unrendered.")
return src
}
args := []string{"--mathjax"}
return internal.ExternallyRenderContent(c.cfg, ctx, src, path, args)
}
func getPandocExecPath() string {
path, err := exec.LookPath("pandoc")
if err != nil {
return ""
}
return path
}
// Supports returns whether Pandoc is installed on this computer.
func Supports() bool {
return getPandocExecPath() != ""
}

38
markup/pandoc/convert_test.go Normal file
View file

@ -0,0 +1,38 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pandoc
import (
"testing"
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/markup/converter"
qt "github.com/frankban/quicktest"
)
func TestConvert(t *testing.T) {
if !Supports() {
t.Skip("pandoc not installed")
}
c := qt.New(t)
p, err := Provider.New(converter.ProviderConfig{Logger: loggers.NewErrorLogger()})
c.Assert(err, qt.IsNil)
conv, err := p.New(converter.DocumentContext{})
c.Assert(err, qt.IsNil)
b, err := conv.Convert(converter.RenderContext{Src: []byte("testContent")})
c.Assert(err, qt.IsNil)
c.Assert(string(b.Bytes()), qt.Equals, "<p>testContent</p>\n")
}

109
markup/rst/convert.go Normal file
View file

@ -0,0 +1,109 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package rst converts content to HTML using the RST external helper.
package rst
import (
"bytes"
"os/exec"
"runtime"
"github.com/gohugoio/hugo/markup/internal"
"github.com/gohugoio/hugo/markup/converter"
)
// Provider is the package entry point.
var Provider converter.NewProvider = provider{}
type provider struct {
}
func (p provider) New(cfg converter.ProviderConfig) (converter.Provider, error) {
var n converter.NewConverter = func(ctx converter.DocumentContext) (converter.Converter, error) {
return &rstConverter{
ctx: ctx,
cfg: cfg,
}, nil
}
return n, nil
}
type rstConverter struct {
ctx converter.DocumentContext
cfg converter.ProviderConfig
}
func (c *rstConverter) Convert(ctx converter.RenderContext) (converter.Result, error) {
return converter.Bytes(c.getRstContent(ctx.Src, c.ctx)), nil
}
// getRstContent calls the Python script rst2html as an external helper
// to convert reStructuredText content to HTML.
func (c *rstConverter) getRstContent(src []byte, ctx converter.DocumentContext) []byte {
logger := c.cfg.Logger
path := getRstExecPath()
if path == "" {
logger.ERROR.Println("rst2html / rst2html.py not found in $PATH: Please install.\n",
" Leaving reStructuredText content unrendered.")
return src
}
logger.INFO.Println("Rendering", ctx.DocumentName, "with", path, "...")
var result []byte
// certain *nix based OSs wrap executables in scripted launchers
// invoking binaries on these OSs via python interpreter causes SyntaxError
// invoke directly so that shebangs work as expected
// handle Windows manually because it doesn't do shebangs
if runtime.GOOS == "windows" {
python := internal.GetPythonExecPath()
args := []string{path, "--leave-comments", "--initial-header-level=2"}
result = internal.ExternallyRenderContent(c.cfg, ctx, src, python, args)
} else {
args := []string{"--leave-comments", "--initial-header-level=2"}
result = internal.ExternallyRenderContent(c.cfg, ctx, src, path, args)
}
// TODO(bep) check if rst2html has a body only option.
bodyStart := bytes.Index(result, []byte("<body>\n"))
if bodyStart < 0 {
bodyStart = -7 //compensate for length
}
bodyEnd := bytes.Index(result, []byte("\n</body>"))
if bodyEnd < 0 || bodyEnd >= len(result) {
bodyEnd = len(result) - 1
if bodyEnd < 0 {
bodyEnd = 0
}
}
return result[bodyStart+7 : bodyEnd]
}
func getRstExecPath() string {
path, err := exec.LookPath("rst2html")
if err != nil {
path, err = exec.LookPath("rst2html.py")
if err != nil {
return ""
}
}
return path
}
// Supports returns whether rst is installed on this computer.
func Supports() bool {
return getRstExecPath() != ""
}

38
markup/rst/convert_test.go Normal file
View file

@ -0,0 +1,38 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package rst
import (
"testing"
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/markup/converter"
qt "github.com/frankban/quicktest"
)
func TestConvert(t *testing.T) {
if !Supports() {
t.Skip("rst not installed")
}
c := qt.New(t)
p, err := Provider.New(converter.ProviderConfig{Logger: loggers.NewErrorLogger()})
c.Assert(err, qt.IsNil)
conv, err := p.New(converter.DocumentContext{})
c.Assert(err, qt.IsNil)
b, err := conv.Convert(converter.RenderContext{Src: []byte("testContent")})
c.Assert(err, qt.IsNil)
c.Assert(string(b.Bytes()), qt.Equals, "<div class=\"document\">\n\n\n<p>testContent</p>\n</div>")
}

3
tpl/collections/collections_test.go
View file

@ -29,6 +29,7 @@ import (
"github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/hugofs"
"github.com/gohugoio/hugo/langs" "github.com/gohugoio/hugo/langs"
"github.com/spf13/afero"
"github.com/spf13/viper" "github.com/spf13/viper"
) )
@ -894,7 +895,7 @@ func ToTstXIs(slice interface{}) []TstXI {
func newDeps(cfg config.Provider) *deps.Deps { func newDeps(cfg config.Provider) *deps.Deps {
l := langs.NewLanguage("en", cfg) l := langs.NewLanguage("en", cfg)
l.Set("i18nDir", "i18n") l.Set("i18nDir", "i18n")
cs, err := helpers.NewContentSpec(l) cs, err := helpers.NewContentSpec(l, loggers.NewErrorLogger(), afero.NewMemMapFs())
if err != nil { if err != nil {
panic(err) panic(err)
} }

2
tpl/data/resources_test.go
View file

@ -195,7 +195,7 @@ func newDeps(cfg config.Provider) *deps.Deps {
} }
cfg.Set("allModules", modules.Modules{mod}) cfg.Set("allModules", modules.Modules{mod})
cs, err := helpers.NewContentSpec(cfg) cs, err := helpers.NewContentSpec(cfg, loggers.NewErrorLogger(), afero.NewMemMapFs())
if err != nil { if err != nil {
panic(err) panic(err)
} }

17
tpl/transform/transform.go
View file

@ -97,19 +97,16 @@ func (ns *Namespace) Markdownify(s interface{}) (template.HTML, error) {
return "", err return "", err
} }
m := ns.deps.ContentSpec.RenderBytes( b, err := ns.deps.ContentSpec.RenderMarkdown([]byte(ss))
&helpers.RenderingContext{
Cfg: ns.deps.Cfg, if err != nil {
Content: []byte(ss), return "", err
PageFmt: "markdown", }
Config: ns.deps.ContentSpec.BlackFriday,
},
)
// Strip if this is a short inline type of text. // Strip if this is a short inline type of text.
m = ns.deps.ContentSpec.TrimShortHTML(m) b = ns.deps.ContentSpec.TrimShortHTML(b)
return helpers.BytesToHTML(m), nil return helpers.BytesToHTML(b), nil
} }
// Plainify returns a copy of s with all HTML tags removed. // Plainify returns a copy of s with all HTML tags removed.

5
tpl/transform/transform_test.go
View file

@ -17,6 +17,9 @@ import (
"html/template" "html/template"
"testing" "testing"
"github.com/gohugoio/hugo/common/loggers"
"github.com/spf13/afero"
qt "github.com/frankban/quicktest" qt "github.com/frankban/quicktest"
"github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/deps"
@ -239,7 +242,7 @@ func newDeps(cfg config.Provider) *deps.Deps {
l := langs.NewLanguage("en", cfg) l := langs.NewLanguage("en", cfg)
cs, err := helpers.NewContentSpec(l) cs, err := helpers.NewContentSpec(l, loggers.NewErrorLogger(), afero.NewMemMapFs())
if err != nil { if err != nil {
panic(err) panic(err)
} }