Prepare for Goldmark

This commmit prepares for the addition of Goldmark as the new Markdown renderer in Hugo.

This introduces a new `markup` package with some common interfaces and each implementation in its own package.

See #5963

deps/deps.go

@@ -223,7 +223,7 @@ func New(cfg DepsCfg) (*Deps, error) {
 		return nil, err
 	}
 
-	contentSpec, err := helpers.NewContentSpec(cfg.Language)
+	contentSpec, err := helpers.NewContentSpec(cfg.Language, logger, ps.BaseFs.Content.Fs)
 	if err != nil {
 		return nil, err
 	}
@@ -277,7 +277,7 @@ func (d Deps) ForLanguage(cfg DepsCfg, onCreated func(d *Deps) error) (*Deps, er
 		return nil, err
 	}
 
-	d.ContentSpec, err = helpers.NewContentSpec(l)
+	d.ContentSpec, err = helpers.NewContentSpec(l, d.Log, d.BaseFs.Content.Fs)
 	if err != nil {
 		return nil, err
 	}

helpers/content.go

@@ -19,22 +19,18 @@ package helpers
 
 import (
 	"bytes"
-	"fmt"
 	"html/template"
-	"os/exec"
-	"runtime"
 	"unicode"
 	"unicode/utf8"
 
-	"github.com/gohugoio/hugo/common/maps"
+	"github.com/gohugoio/hugo/common/loggers"
-	"github.com/gohugoio/hugo/hugolib/filesystems"
+
-	"github.com/niklasfasching/go-org/org"
+	"github.com/gohugoio/hugo/markup/converter"
+
+	"github.com/gohugoio/hugo/markup"
 
 	bp "github.com/gohugoio/hugo/bufferpool"
 	"github.com/gohugoio/hugo/config"
-	"github.com/miekg/mmark"
-	"github.com/mitchellh/mapstructure"
-	"github.com/russross/blackfriday"
 	"github.com/spf13/afero"
 	jww "github.com/spf13/jwalterweatherman"
 
@@ -52,9 +48,9 @@ var (
 
 // ContentSpec provides functionality to render markdown content.
 type ContentSpec struct {
-	BlackFriday                *BlackFriday
+	Converters       markup.ConverterProvider
-	footnoteAnchorPrefix       string
+	MardownConverter converter.Converter // Markdown converter with no document context
-	footnoteReturnLinkContents string
+
 	// SummaryLength is the length of the summary that Hugo extracts from a content.
 	summaryLength int
 
@@ -70,16 +66,13 @@ type ContentSpec struct {
 
 // NewContentSpec returns a ContentSpec initialized
 // with the appropriate fields from the given config.Provider.
-func NewContentSpec(cfg config.Provider) (*ContentSpec, error) {
+func NewContentSpec(cfg config.Provider, logger *loggers.Logger, contentFs afero.Fs) (*ContentSpec, error) {
-	bf := newBlackfriday(cfg.GetStringMap("blackfriday"))
+
 	spec := &ContentSpec{
-		BlackFriday:                bf,
+		summaryLength: cfg.GetInt("summaryLength"),
-		footnoteAnchorPrefix:       cfg.GetString("footnoteAnchorPrefix"),
+		BuildFuture:   cfg.GetBool("buildFuture"),
-		footnoteReturnLinkContents: cfg.GetString("footnoteReturnLinkContents"),
+		BuildExpired:  cfg.GetBool("buildExpired"),
-		summaryLength:              cfg.GetInt("summaryLength"),
+		BuildDrafts:   cfg.GetBool("buildDrafts"),
-		BuildFuture:                cfg.GetBool("buildFuture"),
-		BuildExpired:               cfg.GetBool("buildExpired"),
-		BuildDrafts:                cfg.GetBool("buildDrafts"),
 
 		Cfg: cfg,
 	}
@@ -109,99 +102,29 @@ func NewContentSpec(cfg config.Provider) (*ContentSpec, error) {
 		spec.Highlight = h.chromaHighlight
 	}
 
+	converterProvider, err := markup.NewConverterProvider(converter.ProviderConfig{
+		Cfg:       cfg,
+		ContentFs: contentFs,
+		Logger:    logger,
+		Highlight: spec.Highlight,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	spec.Converters = converterProvider
+	p := converterProvider.Get("markdown")
+	conv, err := p.New(converter.DocumentContext{})
+	if err != nil {
+		return nil, err
+	}
+	spec.MardownConverter = conv
+
 	return spec, nil
 }
 
-// BlackFriday holds configuration values for BlackFriday rendering.
-type BlackFriday struct {
-	Smartypants           bool
-	SmartypantsQuotesNBSP bool
-	AngledQuotes          bool
-	Fractions             bool
-	HrefTargetBlank       bool
-	NofollowLinks         bool
-	NoreferrerLinks       bool
-	SmartDashes           bool
-	LatexDashes           bool
-	TaskLists             bool
-	PlainIDAnchors        bool
-	Extensions            []string
-	ExtensionsMask        []string
-	SkipHTML              bool
-}
-
-// NewBlackfriday creates a new Blackfriday filled with site config or some sane defaults.
-func newBlackfriday(config map[string]interface{}) *BlackFriday {
-	defaultParam := map[string]interface{}{
-		"smartypants":           true,
-		"angledQuotes":          false,
-		"smartypantsQuotesNBSP": false,
-		"fractions":             true,
-		"hrefTargetBlank":       false,
-		"nofollowLinks":         false,
-		"noreferrerLinks":       false,
-		"smartDashes":           true,
-		"latexDashes":           true,
-		"plainIDAnchors":        true,
-		"taskLists":             true,
-		"skipHTML":              false,
-	}
-
-	maps.ToLower(defaultParam)
-
-	siteConfig := make(map[string]interface{})
-
-	for k, v := range defaultParam {
-		siteConfig[k] = v
-	}
-
-	for k, v := range config {
-		siteConfig[k] = v
-	}
-
-	combinedConfig := &BlackFriday{}
-	if err := mapstructure.Decode(siteConfig, combinedConfig); err != nil {
-		jww.FATAL.Printf("Failed to get site rendering config\n%s", err.Error())
-	}
-
-	return combinedConfig
-}
-
-var blackfridayExtensionMap = map[string]int{
-	"noIntraEmphasis":        blackfriday.EXTENSION_NO_INTRA_EMPHASIS,
-	"tables":                 blackfriday.EXTENSION_TABLES,
-	"fencedCode":             blackfriday.EXTENSION_FENCED_CODE,
-	"autolink":               blackfriday.EXTENSION_AUTOLINK,
-	"strikethrough":          blackfriday.EXTENSION_STRIKETHROUGH,
-	"laxHtmlBlocks":          blackfriday.EXTENSION_LAX_HTML_BLOCKS,
-	"spaceHeaders":           blackfriday.EXTENSION_SPACE_HEADERS,
-	"hardLineBreak":          blackfriday.EXTENSION_HARD_LINE_BREAK,
-	"tabSizeEight":           blackfriday.EXTENSION_TAB_SIZE_EIGHT,
-	"footnotes":              blackfriday.EXTENSION_FOOTNOTES,
-	"noEmptyLineBeforeBlock": blackfriday.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK,
-	"headerIds":              blackfriday.EXTENSION_HEADER_IDS,
-	"titleblock":             blackfriday.EXTENSION_TITLEBLOCK,
-	"autoHeaderIds":          blackfriday.EXTENSION_AUTO_HEADER_IDS,
-	"backslashLineBreak":     blackfriday.EXTENSION_BACKSLASH_LINE_BREAK,
-	"definitionLists":        blackfriday.EXTENSION_DEFINITION_LISTS,
-	"joinLines":              blackfriday.EXTENSION_JOIN_LINES,
-}
-
 var stripHTMLReplacer = strings.NewReplacer("\n", " ", "</p>", "\n", "<br>", "\n", "<br />", "\n")
 
-var mmarkExtensionMap = map[string]int{
-	"tables":                 mmark.EXTENSION_TABLES,
-	"fencedCode":             mmark.EXTENSION_FENCED_CODE,
-	"autolink":               mmark.EXTENSION_AUTOLINK,
-	"laxHtmlBlocks":          mmark.EXTENSION_LAX_HTML_BLOCKS,
-	"spaceHeaders":           mmark.EXTENSION_SPACE_HEADERS,
-	"hardLineBreak":          mmark.EXTENSION_HARD_LINE_BREAK,
-	"footnotes":              mmark.EXTENSION_FOOTNOTES,
-	"noEmptyLineBeforeBlock": mmark.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK,
-	"headerIds":              mmark.EXTENSION_HEADER_IDS,
-	"autoHeaderIds":          mmark.EXTENSION_AUTO_HEADER_IDS,
-}
-
 // StripHTML accepts a string, strips out all HTML tags and returns it.
 func StripHTML(s string) string {
 
@@ -250,181 +173,6 @@ func BytesToHTML(b []byte) template.HTML {
 	return template.HTML(string(b))
 }
 
-// getHTMLRenderer creates a new Blackfriday HTML Renderer with the given configuration.
-func (c *ContentSpec) getHTMLRenderer(defaultFlags int, ctx *RenderingContext) blackfriday.Renderer {
-	renderParameters := blackfriday.HtmlRendererParameters{
-		FootnoteAnchorPrefix:       c.footnoteAnchorPrefix,
-		FootnoteReturnLinkContents: c.footnoteReturnLinkContents,
-	}
-
-	b := len(ctx.DocumentID) != 0
-
-	if ctx.Config == nil {
-		panic(fmt.Sprintf("RenderingContext of %q doesn't have a config", ctx.DocumentID))
-	}
-
-	if b && !ctx.Config.PlainIDAnchors {
-		renderParameters.FootnoteAnchorPrefix = ctx.DocumentID + ":" + renderParameters.FootnoteAnchorPrefix
-		renderParameters.HeaderIDSuffix = ":" + ctx.DocumentID
-	}
-
-	htmlFlags := defaultFlags
-	htmlFlags |= blackfriday.HTML_USE_XHTML
-	htmlFlags |= blackfriday.HTML_FOOTNOTE_RETURN_LINKS
-
-	if ctx.Config.Smartypants {
-		htmlFlags |= blackfriday.HTML_USE_SMARTYPANTS
-	}
-
-	if ctx.Config.SmartypantsQuotesNBSP {
-		htmlFlags |= blackfriday.HTML_SMARTYPANTS_QUOTES_NBSP
-	}
-
-	if ctx.Config.AngledQuotes {
-		htmlFlags |= blackfriday.HTML_SMARTYPANTS_ANGLED_QUOTES
-	}
-
-	if ctx.Config.Fractions {
-		htmlFlags |= blackfriday.HTML_SMARTYPANTS_FRACTIONS
-	}
-
-	if ctx.Config.HrefTargetBlank {
-		htmlFlags |= blackfriday.HTML_HREF_TARGET_BLANK
-	}
-
-	if ctx.Config.NofollowLinks {
-		htmlFlags |= blackfriday.HTML_NOFOLLOW_LINKS
-	}
-
-	if ctx.Config.NoreferrerLinks {
-		htmlFlags |= blackfriday.HTML_NOREFERRER_LINKS
-	}
-
-	if ctx.Config.SmartDashes {
-		htmlFlags |= blackfriday.HTML_SMARTYPANTS_DASHES
-	}
-
-	if ctx.Config.LatexDashes {
-		htmlFlags |= blackfriday.HTML_SMARTYPANTS_LATEX_DASHES
-	}
-
-	if ctx.Config.SkipHTML {
-		htmlFlags |= blackfriday.HTML_SKIP_HTML
-	}
-
-	return &HugoHTMLRenderer{
-		cs:               c,
-		RenderingContext: ctx,
-		Renderer:         blackfriday.HtmlRendererWithParameters(htmlFlags, "", "", renderParameters),
-	}
-}
-
-func getMarkdownExtensions(ctx *RenderingContext) int {
-	// Default Blackfriday common extensions
-	commonExtensions := 0 |
-		blackfriday.EXTENSION_NO_INTRA_EMPHASIS |
-		blackfriday.EXTENSION_TABLES |
-		blackfriday.EXTENSION_FENCED_CODE |
-		blackfriday.EXTENSION_AUTOLINK |
-		blackfriday.EXTENSION_STRIKETHROUGH |
-		blackfriday.EXTENSION_SPACE_HEADERS |
-		blackfriday.EXTENSION_HEADER_IDS |
-		blackfriday.EXTENSION_BACKSLASH_LINE_BREAK |
-		blackfriday.EXTENSION_DEFINITION_LISTS
-
-	// Extra Blackfriday extensions that Hugo enables by default
-	flags := commonExtensions |
-		blackfriday.EXTENSION_AUTO_HEADER_IDS |
-		blackfriday.EXTENSION_FOOTNOTES
-
-	if ctx.Config == nil {
-		panic(fmt.Sprintf("RenderingContext of %q doesn't have a config", ctx.DocumentID))
-	}
-
-	for _, extension := range ctx.Config.Extensions {
-		if flag, ok := blackfridayExtensionMap[extension]; ok {
-			flags |= flag
-		}
-	}
-	for _, extension := range ctx.Config.ExtensionsMask {
-		if flag, ok := blackfridayExtensionMap[extension]; ok {
-			flags &= ^flag
-		}
-	}
-	return flags
-}
-
-func (c *ContentSpec) markdownRender(ctx *RenderingContext) []byte {
-	if ctx.RenderTOC {
-		return blackfriday.Markdown(ctx.Content,
-			c.getHTMLRenderer(blackfriday.HTML_TOC, ctx),
-			getMarkdownExtensions(ctx))
-	}
-	return blackfriday.Markdown(ctx.Content, c.getHTMLRenderer(0, ctx),
-		getMarkdownExtensions(ctx))
-}
-
-// getMmarkHTMLRenderer creates a new mmark HTML Renderer with the given configuration.
-func (c *ContentSpec) getMmarkHTMLRenderer(defaultFlags int, ctx *RenderingContext) mmark.Renderer {
-	renderParameters := mmark.HtmlRendererParameters{
-		FootnoteAnchorPrefix:       c.footnoteAnchorPrefix,
-		FootnoteReturnLinkContents: c.footnoteReturnLinkContents,
-	}
-
-	b := len(ctx.DocumentID) != 0
-
-	if ctx.Config == nil {
-		panic(fmt.Sprintf("RenderingContext of %q doesn't have a config", ctx.DocumentID))
-	}
-
-	if b && !ctx.Config.PlainIDAnchors {
-		renderParameters.FootnoteAnchorPrefix = ctx.DocumentID + ":" + renderParameters.FootnoteAnchorPrefix
-		// renderParameters.HeaderIDSuffix = ":" + ctx.DocumentId
-	}
-
-	htmlFlags := defaultFlags
-	htmlFlags |= mmark.HTML_FOOTNOTE_RETURN_LINKS
-
-	return &HugoMmarkHTMLRenderer{
-		cs:       c,
-		Renderer: mmark.HtmlRendererWithParameters(htmlFlags, "", "", renderParameters),
-		Cfg:      c.Cfg,
-	}
-}
-
-func getMmarkExtensions(ctx *RenderingContext) int {
-	flags := 0
-	flags |= mmark.EXTENSION_TABLES
-	flags |= mmark.EXTENSION_FENCED_CODE
-	flags |= mmark.EXTENSION_AUTOLINK
-	flags |= mmark.EXTENSION_SPACE_HEADERS
-	flags |= mmark.EXTENSION_CITATION
-	flags |= mmark.EXTENSION_TITLEBLOCK_TOML
-	flags |= mmark.EXTENSION_HEADER_IDS
-	flags |= mmark.EXTENSION_AUTO_HEADER_IDS
-	flags |= mmark.EXTENSION_UNIQUE_HEADER_IDS
-	flags |= mmark.EXTENSION_FOOTNOTES
-	flags |= mmark.EXTENSION_SHORT_REF
-	flags |= mmark.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK
-	flags |= mmark.EXTENSION_INCLUDE
-
-	if ctx.Config == nil {
-		panic(fmt.Sprintf("RenderingContext of %q doesn't have a config", ctx.DocumentID))
-	}
-
-	for _, extension := range ctx.Config.Extensions {
-		if flag, ok := mmarkExtensionMap[extension]; ok {
-			flags |= flag
-		}
-	}
-	return flags
-}
-
-func (c *ContentSpec) mmarkRender(ctx *RenderingContext) []byte {
-	return mmark.Parse(ctx.Content, c.getMmarkHTMLRenderer(0, ctx),
-		getMmarkExtensions(ctx)).Bytes()
-}
-
 // ExtractTOC extracts Table of Contents from content.
 func ExtractTOC(content []byte) (newcontent []byte, toc []byte) {
 	if !bytes.Contains(content, []byte("<nav>")) {
@@ -464,38 +212,12 @@ func ExtractTOC(content []byte) (newcontent []byte, toc []byte) {
 	return
 }
 
-// RenderingContext holds contextual information, like content and configuration,
+func (c *ContentSpec) RenderMarkdown(src []byte) ([]byte, error) {
-// for a given content rendering.
+	b, err := c.MardownConverter.Convert(converter.RenderContext{Src: src})
-// By creating you must set the Config, otherwise it will panic.
+	if err != nil {
-type RenderingContext struct {
+		return nil, err
-	BaseFs       *filesystems.BaseFs
-	Content      []byte
-	PageFmt      string
-	DocumentID   string
-	DocumentName string
-	Config       *BlackFriday
-	RenderTOC    bool
-	Cfg          config.Provider
-}
-
-// RenderBytes renders a []byte.
-func (c *ContentSpec) RenderBytes(ctx *RenderingContext) []byte {
-	switch ctx.PageFmt {
-	default:
-		return c.markdownRender(ctx)
-	case "markdown":
-		return c.markdownRender(ctx)
-	case "asciidoc":
-		return getAsciidocContent(ctx)
-	case "mmark":
-		return c.mmarkRender(ctx)
-	case "rst":
-		return getRstContent(ctx)
-	case "org":
-		return orgRender(ctx, c)
-	case "pandoc":
-		return getPandocContent(ctx)
 	}
+	return b.Bytes(), nil
 }
 
 // TotalWords counts instance of one or more consecutive white space
@@ -622,181 +344,3 @@ func (c *ContentSpec) truncateWordsToWholeSentenceOld(content string) (string, b
 
 	return strings.Join(words[:c.summaryLength], " "), true
 }
-
-func getAsciidocExecPath() string {
-	path, err := exec.LookPath("asciidoc")
-	if err != nil {
-		return ""
-	}
-	return path
-}
-
-func getAsciidoctorExecPath() string {
-	path, err := exec.LookPath("asciidoctor")
-	if err != nil {
-		return ""
-	}
-	return path
-}
-
-// HasAsciidoc returns whether Asciidoc or Asciidoctor is installed on this computer.
-func HasAsciidoc() bool {
-	return (getAsciidoctorExecPath() != "" ||
-		getAsciidocExecPath() != "")
-}
-
-// getAsciidocContent calls asciidoctor or asciidoc as an external helper
-// to convert AsciiDoc content to HTML.
-func getAsciidocContent(ctx *RenderingContext) []byte {
-	var isAsciidoctor bool
-	path := getAsciidoctorExecPath()
-	if path == "" {
-		path = getAsciidocExecPath()
-		if path == "" {
-			jww.ERROR.Println("asciidoctor / asciidoc not found in $PATH: Please install.\n",
-				"                 Leaving AsciiDoc content unrendered.")
-			return ctx.Content
-		}
-	} else {
-		isAsciidoctor = true
-	}
-
-	jww.INFO.Println("Rendering", ctx.DocumentName, "with", path, "...")
-	args := []string{"--no-header-footer", "--safe"}
-	if isAsciidoctor {
-		// asciidoctor-specific arg to show stack traces on errors
-		args = append(args, "--trace")
-	}
-	args = append(args, "-")
-	return externallyRenderContent(ctx, path, args)
-}
-
-// HasRst returns whether rst2html is installed on this computer.
-func HasRst() bool {
-	return getRstExecPath() != ""
-}
-
-func getRstExecPath() string {
-	path, err := exec.LookPath("rst2html")
-	if err != nil {
-		path, err = exec.LookPath("rst2html.py")
-		if err != nil {
-			return ""
-		}
-	}
-	return path
-}
-
-func getPythonExecPath() string {
-	path, err := exec.LookPath("python")
-	if err != nil {
-		path, err = exec.LookPath("python.exe")
-		if err != nil {
-			return ""
-		}
-	}
-	return path
-}
-
-// getRstContent calls the Python script rst2html as an external helper
-// to convert reStructuredText content to HTML.
-func getRstContent(ctx *RenderingContext) []byte {
-	path := getRstExecPath()
-
-	if path == "" {
-		jww.ERROR.Println("rst2html / rst2html.py not found in $PATH: Please install.\n",
-			"                 Leaving reStructuredText content unrendered.")
-		return ctx.Content
-
-	}
-	jww.INFO.Println("Rendering", ctx.DocumentName, "with", path, "...")
-	var result []byte
-	// certain *nix based OSs wrap executables in scripted launchers
-	// invoking binaries on these OSs via python interpreter causes SyntaxError
-	// invoke directly so that shebangs work as expected
-	// handle Windows manually because it doesn't do shebangs
-	if runtime.GOOS == "windows" {
-		python := getPythonExecPath()
-		args := []string{path, "--leave-comments", "--initial-header-level=2"}
-		result = externallyRenderContent(ctx, python, args)
-	} else {
-		args := []string{"--leave-comments", "--initial-header-level=2"}
-		result = externallyRenderContent(ctx, path, args)
-	}
-	// TODO(bep) check if rst2html has a body only option.
-	bodyStart := bytes.Index(result, []byte("<body>\n"))
-	if bodyStart < 0 {
-		bodyStart = -7 //compensate for length
-	}
-
-	bodyEnd := bytes.Index(result, []byte("\n</body>"))
-	if bodyEnd < 0 || bodyEnd >= len(result) {
-		bodyEnd = len(result) - 1
-		if bodyEnd < 0 {
-			bodyEnd = 0
-		}
-	}
-
-	return result[bodyStart+7 : bodyEnd]
-}
-
-// getPandocContent calls pandoc as an external helper to convert pandoc markdown to HTML.
-func getPandocContent(ctx *RenderingContext) []byte {
-	path, err := exec.LookPath("pandoc")
-	if err != nil {
-		jww.ERROR.Println("pandoc not found in $PATH: Please install.\n",
-			"                 Leaving pandoc content unrendered.")
-		return ctx.Content
-	}
-	args := []string{"--mathjax"}
-	return externallyRenderContent(ctx, path, args)
-}
-
-func orgRender(ctx *RenderingContext, c *ContentSpec) []byte {
-	config := org.New()
-	config.Log = jww.WARN
-	config.ReadFile = func(filename string) ([]byte, error) {
-		return afero.ReadFile(ctx.BaseFs.Content.Fs, filename)
-	}
-	writer := org.NewHTMLWriter()
-	writer.HighlightCodeBlock = func(source, lang string) string {
-		highlightedSource, err := c.Highlight(source, lang, "")
-		if err != nil {
-			jww.ERROR.Printf("Could not highlight source as lang %s. Using raw source.", lang)
-			return source
-		}
-		return highlightedSource
-	}
-
-	html, err := config.Parse(bytes.NewReader(ctx.Content), ctx.DocumentName).Write(writer)
-	if err != nil {
-		jww.ERROR.Printf("Could not render org: %s. Using unrendered content.", err)
-		return ctx.Content
-	}
-	return []byte(html)
-}
-
-func externallyRenderContent(ctx *RenderingContext, path string, args []string) []byte {
-	content := ctx.Content
-	cleanContent := bytes.Replace(content, SummaryDivider, []byte(""), 1)
-
-	cmd := exec.Command(path, args...)
-	cmd.Stdin = bytes.NewReader(cleanContent)
-	var out, cmderr bytes.Buffer
-	cmd.Stdout = &out
-	cmd.Stderr = &cmderr
-	err := cmd.Run()
-	// Most external helpers exit w/ non-zero exit code only if severe, i.e.
-	// halting errors occurred. -> log stderr output regardless of state of err
-	for _, item := range strings.Split(cmderr.String(), "\n") {
-		item := strings.TrimSpace(item)
-		if item != "" {
-			jww.ERROR.Printf("%s: %s", ctx.DocumentName, item)
-		}
-	}
-	if err != nil {
-		jww.ERROR.Printf("%s rendering %s: %v", path, ctx.DocumentName, err)
-	}
-
-	return normalizeExternalHelperLineFeeds(out.Bytes())
-}

helpers/content_renderer_test.go

@@ -1,141 +0,0 @@
-// Copyright 2019 The Hugo Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package helpers
-
-import (
-	"bytes"
-	"regexp"
-	"testing"
-
-	qt "github.com/frankban/quicktest"
-	"github.com/spf13/viper"
-)
-
-// Renders a codeblock using Blackfriday
-func (c *ContentSpec) render(input string) string {
-	ctx := &RenderingContext{Cfg: c.Cfg, Config: c.BlackFriday}
-	render := c.getHTMLRenderer(0, ctx)
-
-	buf := &bytes.Buffer{}
-	render.BlockCode(buf, []byte(input), "html")
-	return buf.String()
-}
-
-// Renders a codeblock using Mmark
-func (c *ContentSpec) renderWithMmark(input string) string {
-	ctx := &RenderingContext{Cfg: c.Cfg, Config: c.BlackFriday}
-	render := c.getMmarkHTMLRenderer(0, ctx)
-
-	buf := &bytes.Buffer{}
-	render.BlockCode(buf, []byte(input), "html", []byte(""), false, false)
-	return buf.String()
-}
-
-func TestCodeFence(t *testing.T) {
-	c := qt.New(t)
-
-	type test struct {
-		enabled         bool
-		input, expected string
-	}
-
-	// Pygments 2.0 and 2.1 have slightly different outputs so only do partial matching
-	data := []test{
-		{true, "<html></html>", `(?s)^<div class="highlight">\n?<pre.*><code class="language-html" data-lang="html">.*?</code></pre>\n?</div>\n?$`},
-		{false, "<html></html>", `(?s)^<pre.*><code class="language-html">.*?</code></pre>\n$`},
-	}
-
-	for _, useClassic := range []bool{false, true} {
-		for i, d := range data {
-			v := viper.New()
-			v.Set("pygmentsStyle", "monokai")
-			v.Set("pygmentsUseClasses", true)
-			v.Set("pygmentsCodeFences", d.enabled)
-			v.Set("pygmentsUseClassic", useClassic)
-
-			cs, err := NewContentSpec(v)
-			c.Assert(err, qt.IsNil)
-
-			result := cs.render(d.input)
-
-			expectedRe, err := regexp.Compile(d.expected)
-
-			if err != nil {
-				t.Fatal("Invalid regexp", err)
-			}
-			matched := expectedRe.MatchString(result)
-
-			if !matched {
-				t.Errorf("Test %d failed. BlackFriday enabled:%t, Expected:\n%q got:\n%q", i, d.enabled, d.expected, result)
-			}
-
-			result = cs.renderWithMmark(d.input)
-			matched = expectedRe.MatchString(result)
-			if !matched {
-				t.Errorf("Test %d failed. Mmark enabled:%t, Expected:\n%q got:\n%q", i, d.enabled, d.expected, result)
-			}
-		}
-	}
-}
-
-func TestBlackfridayTaskList(t *testing.T) {
-	c := newTestContentSpec()
-
-	for i, this := range []struct {
-		markdown        string
-		taskListEnabled bool
-		expect          string
-	}{
-		{`
-TODO:
-
-- [x] On1
-- [X] On2
-- [ ] Off
-
-END
-`, true, `<p>TODO:</p>
-
-<ul class="task-list">
-<li><label><input type="checkbox" checked disabled class="task-list-item"> On1</label></li>
-<li><label><input type="checkbox" checked disabled class="task-list-item"> On2</label></li>
-<li><label><input type="checkbox" disabled class="task-list-item"> Off</label></li>
-</ul>
-
-<p>END</p>
-`},
-		{`- [x] On1`, false, `<ul>
-<li>[x] On1</li>
-</ul>
-`},
-		{`* [ ] Off
-
-END`, true, `<ul class="task-list">
-<li><label><input type="checkbox" disabled class="task-list-item"> Off</label></li>
-</ul>
-
-<p>END</p>
-`},
-	} {
-		blackFridayConfig := c.BlackFriday
-		blackFridayConfig.TaskLists = this.taskListEnabled
-		ctx := &RenderingContext{Content: []byte(this.markdown), PageFmt: "markdown", Config: blackFridayConfig}
-
-		result := string(c.RenderBytes(ctx))
-
-		if result != this.expect {
-			t.Errorf("[%d] got \n%v but expected \n%v", i, result, this.expect)
-		}
-	}
-}

helpers/content_test.go

@@ -19,11 +19,13 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/spf13/afero"
+
+	"github.com/gohugoio/hugo/common/loggers"
+
 	"github.com/spf13/viper"
 
 	qt "github.com/frankban/quicktest"
-	"github.com/miekg/mmark"
-	"github.com/russross/blackfriday"
 )
 
 const tstHTMLContent = "<!DOCTYPE html><html><head><script src=\"http://two/foobar.js\"></script></head><body><nav><ul><li hugo-nav=\"section_0\"></li><li hugo-nav=\"section_1\"></li></ul></nav><article>content <a href=\"http://two/foobar\">foobar</a>. Follow up</article><p>This is some text.<br>And some more.</p></body></html>"
@@ -108,7 +110,7 @@ func TestNewContentSpec(t *testing.T) {
 	cfg.Set("buildExpired", true)
 	cfg.Set("buildDrafts", true)
 
-	spec, err := NewContentSpec(cfg)
+	spec, err := NewContentSpec(cfg, loggers.NewErrorLogger(), afero.NewMemMapFs())
 
 	c.Assert(err, qt.IsNil)
 	c.Assert(spec.summaryLength, qt.Equals, 32)
@@ -202,233 +204,6 @@ func TestTruncateWordsByRune(t *testing.T) {
 	}
 }
 
-func TestGetHTMLRendererFlags(t *testing.T) {
-	c := newTestContentSpec()
-	ctx := &RenderingContext{Cfg: c.Cfg, Config: c.BlackFriday}
-	renderer := c.getHTMLRenderer(blackfriday.HTML_USE_XHTML, ctx)
-	flags := renderer.GetFlags()
-	if flags&blackfriday.HTML_USE_XHTML != blackfriday.HTML_USE_XHTML {
-		t.Errorf("Test flag: %d was not found amongs set flags:%d; Result: %d", blackfriday.HTML_USE_XHTML, flags, flags&blackfriday.HTML_USE_XHTML)
-	}
-}
-
-func TestGetHTMLRendererAllFlags(t *testing.T) {
-	c := newTestContentSpec()
-
-	type data struct {
-		testFlag int
-	}
-
-	allFlags := []data{
-		{blackfriday.HTML_USE_XHTML},
-		{blackfriday.HTML_FOOTNOTE_RETURN_LINKS},
-		{blackfriday.HTML_USE_SMARTYPANTS},
-		{blackfriday.HTML_SMARTYPANTS_QUOTES_NBSP},
-		{blackfriday.HTML_SMARTYPANTS_ANGLED_QUOTES},
-		{blackfriday.HTML_SMARTYPANTS_FRACTIONS},
-		{blackfriday.HTML_HREF_TARGET_BLANK},
-		{blackfriday.HTML_NOFOLLOW_LINKS},
-		{blackfriday.HTML_NOREFERRER_LINKS},
-		{blackfriday.HTML_SMARTYPANTS_DASHES},
-		{blackfriday.HTML_SMARTYPANTS_LATEX_DASHES},
-	}
-	defaultFlags := blackfriday.HTML_USE_XHTML
-	ctx := &RenderingContext{Cfg: c.Cfg, Config: c.BlackFriday}
-	ctx.Config.AngledQuotes = true
-	ctx.Config.Fractions = true
-	ctx.Config.HrefTargetBlank = true
-	ctx.Config.NofollowLinks = true
-	ctx.Config.NoreferrerLinks = true
-	ctx.Config.LatexDashes = true
-	ctx.Config.PlainIDAnchors = true
-	ctx.Config.SmartDashes = true
-	ctx.Config.Smartypants = true
-	ctx.Config.SmartypantsQuotesNBSP = true
-	renderer := c.getHTMLRenderer(defaultFlags, ctx)
-	actualFlags := renderer.GetFlags()
-	var expectedFlags int
-	//OR-ing flags together...
-	for _, d := range allFlags {
-		expectedFlags |= d.testFlag
-	}
-	if expectedFlags != actualFlags {
-		t.Errorf("Expected flags (%d) did not equal actual (%d) flags.", expectedFlags, actualFlags)
-	}
-}
-
-func TestGetHTMLRendererAnchors(t *testing.T) {
-	c := newTestContentSpec()
-	ctx := &RenderingContext{Cfg: c.Cfg, Config: c.BlackFriday}
-	ctx.DocumentID = "testid"
-	ctx.Config.PlainIDAnchors = false
-
-	actualRenderer := c.getHTMLRenderer(0, ctx)
-	headerBuffer := &bytes.Buffer{}
-	footnoteBuffer := &bytes.Buffer{}
-	expectedFootnoteHref := []byte("href=\"#fn:testid:href\"")
-	expectedHeaderID := []byte("<h1 id=\"id:testid\"></h1>\n")
-
-	actualRenderer.Header(headerBuffer, func() bool { return true }, 1, "id")
-	actualRenderer.FootnoteRef(footnoteBuffer, []byte("href"), 1)
-
-	if !bytes.Contains(footnoteBuffer.Bytes(), expectedFootnoteHref) {
-		t.Errorf("Footnote anchor prefix not applied. Actual:%s Expected:%s", footnoteBuffer.String(), expectedFootnoteHref)
-	}
-
-	if !bytes.Equal(headerBuffer.Bytes(), expectedHeaderID) {
-		t.Errorf("Header Id Postfix not applied. Actual:%s Expected:%s", headerBuffer.String(), expectedHeaderID)
-	}
-}
-
-func TestGetMmarkHTMLRenderer(t *testing.T) {
-	c := newTestContentSpec()
-	ctx := &RenderingContext{Cfg: c.Cfg, Config: c.BlackFriday}
-	ctx.DocumentID = "testid"
-	ctx.Config.PlainIDAnchors = false
-	actualRenderer := c.getMmarkHTMLRenderer(0, ctx)
-
-	headerBuffer := &bytes.Buffer{}
-	footnoteBuffer := &bytes.Buffer{}
-	expectedFootnoteHref := []byte("href=\"#fn:testid:href\"")
-	expectedHeaderID := []byte("<h1 id=\"id\"></h1>")
-
-	actualRenderer.FootnoteRef(footnoteBuffer, []byte("href"), 1)
-	actualRenderer.Header(headerBuffer, func() bool { return true }, 1, "id")
-
-	if !bytes.Contains(footnoteBuffer.Bytes(), expectedFootnoteHref) {
-		t.Errorf("Footnote anchor prefix not applied. Actual:%s Expected:%s", footnoteBuffer.String(), expectedFootnoteHref)
-	}
-
-	if bytes.Equal(headerBuffer.Bytes(), expectedHeaderID) {
-		t.Errorf("Header Id Postfix applied. Actual:%s Expected:%s", headerBuffer.String(), expectedHeaderID)
-	}
-}
-
-func TestGetMarkdownExtensionsMasksAreRemovedFromExtensions(t *testing.T) {
-	c := newTestContentSpec()
-	ctx := &RenderingContext{Cfg: c.Cfg, Config: c.BlackFriday}
-	ctx.Config.Extensions = []string{"headerId"}
-	ctx.Config.ExtensionsMask = []string{"noIntraEmphasis"}
-
-	actualFlags := getMarkdownExtensions(ctx)
-	if actualFlags&blackfriday.EXTENSION_NO_INTRA_EMPHASIS == blackfriday.EXTENSION_NO_INTRA_EMPHASIS {
-		t.Errorf("Masked out flag {%v} found amongst returned extensions.", blackfriday.EXTENSION_NO_INTRA_EMPHASIS)
-	}
-}
-
-func TestGetMarkdownExtensionsByDefaultAllExtensionsAreEnabled(t *testing.T) {
-	type data struct {
-		testFlag int
-	}
-	c := newTestContentSpec()
-	ctx := &RenderingContext{Cfg: c.Cfg, Config: c.BlackFriday}
-	ctx.Config.Extensions = []string{""}
-	ctx.Config.ExtensionsMask = []string{""}
-	allExtensions := []data{
-		{blackfriday.EXTENSION_NO_INTRA_EMPHASIS},
-		{blackfriday.EXTENSION_TABLES},
-		{blackfriday.EXTENSION_FENCED_CODE},
-		{blackfriday.EXTENSION_AUTOLINK},
-		{blackfriday.EXTENSION_STRIKETHROUGH},
-		// {blackfriday.EXTENSION_LAX_HTML_BLOCKS},
-		{blackfriday.EXTENSION_SPACE_HEADERS},
-		// {blackfriday.EXTENSION_HARD_LINE_BREAK},
-		// {blackfriday.EXTENSION_TAB_SIZE_EIGHT},
-		{blackfriday.EXTENSION_FOOTNOTES},
-		// {blackfriday.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK},
-		{blackfriday.EXTENSION_HEADER_IDS},
-		// {blackfriday.EXTENSION_TITLEBLOCK},
-		{blackfriday.EXTENSION_AUTO_HEADER_IDS},
-		{blackfriday.EXTENSION_BACKSLASH_LINE_BREAK},
-		{blackfriday.EXTENSION_DEFINITION_LISTS},
-	}
-
-	actualFlags := getMarkdownExtensions(ctx)
-	for _, e := range allExtensions {
-		if actualFlags&e.testFlag != e.testFlag {
-			t.Errorf("Flag %v was not found in the list of extensions.", e)
-		}
-	}
-}
-
-func TestGetMarkdownExtensionsAddingFlagsThroughRenderingContext(t *testing.T) {
-	c := newTestContentSpec()
-	ctx := &RenderingContext{Cfg: c.Cfg, Config: c.BlackFriday}
-	ctx.Config.Extensions = []string{"definitionLists"}
-	ctx.Config.ExtensionsMask = []string{""}
-
-	actualFlags := getMarkdownExtensions(ctx)
-	if actualFlags&blackfriday.EXTENSION_DEFINITION_LISTS != blackfriday.EXTENSION_DEFINITION_LISTS {
-		t.Errorf("Masked out flag {%v} found amongst returned extensions.", blackfriday.EXTENSION_DEFINITION_LISTS)
-	}
-}
-
-func TestGetMarkdownRenderer(t *testing.T) {
-	c := newTestContentSpec()
-	ctx := &RenderingContext{Cfg: c.Cfg, Config: c.BlackFriday}
-	ctx.Content = []byte("testContent")
-	actualRenderedMarkdown := c.markdownRender(ctx)
-	expectedRenderedMarkdown := []byte("<p>testContent</p>\n")
-	if !bytes.Equal(actualRenderedMarkdown, expectedRenderedMarkdown) {
-		t.Errorf("Actual rendered Markdown (%s) did not match expected markdown (%s)", actualRenderedMarkdown, expectedRenderedMarkdown)
-	}
-}
-
-func TestGetMarkdownRendererWithTOC(t *testing.T) {
-	c := newTestContentSpec()
-	ctx := &RenderingContext{RenderTOC: true, Cfg: c.Cfg, Config: c.BlackFriday}
-	ctx.Content = []byte("testContent")
-	actualRenderedMarkdown := c.markdownRender(ctx)
-	expectedRenderedMarkdown := []byte("<nav>\n</nav>\n\n<p>testContent</p>\n")
-	if !bytes.Equal(actualRenderedMarkdown, expectedRenderedMarkdown) {
-		t.Errorf("Actual rendered Markdown (%s) did not match expected markdown (%s)", actualRenderedMarkdown, expectedRenderedMarkdown)
-	}
-}
-
-func TestGetMmarkExtensions(t *testing.T) {
-	//TODO: This is doing the same just with different marks...
-	type data struct {
-		testFlag int
-	}
-	c := newTestContentSpec()
-	ctx := &RenderingContext{Cfg: c.Cfg, Config: c.BlackFriday}
-	ctx.Config.Extensions = []string{"tables"}
-	ctx.Config.ExtensionsMask = []string{""}
-	allExtensions := []data{
-		{mmark.EXTENSION_TABLES},
-		{mmark.EXTENSION_FENCED_CODE},
-		{mmark.EXTENSION_AUTOLINK},
-		{mmark.EXTENSION_SPACE_HEADERS},
-		{mmark.EXTENSION_CITATION},
-		{mmark.EXTENSION_TITLEBLOCK_TOML},
-		{mmark.EXTENSION_HEADER_IDS},
-		{mmark.EXTENSION_AUTO_HEADER_IDS},
-		{mmark.EXTENSION_UNIQUE_HEADER_IDS},
-		{mmark.EXTENSION_FOOTNOTES},
-		{mmark.EXTENSION_SHORT_REF},
-		{mmark.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK},
-		{mmark.EXTENSION_INCLUDE},
-	}
-
-	actualFlags := getMmarkExtensions(ctx)
-	for _, e := range allExtensions {
-		if actualFlags&e.testFlag != e.testFlag {
-			t.Errorf("Flag %v was not found in the list of extensions.", e)
-		}
-	}
-}
-
-func TestMmarkRender(t *testing.T) {
-	c := newTestContentSpec()
-	ctx := &RenderingContext{Cfg: c.Cfg, Config: c.BlackFriday}
-	ctx.Content = []byte("testContent")
-	actualRenderedMarkdown := c.mmarkRender(ctx)
-	expectedRenderedMarkdown := []byte("<p>testContent</p>\n")
-	if !bytes.Equal(actualRenderedMarkdown, expectedRenderedMarkdown) {
-		t.Errorf("Actual rendered Markdown (%s) did not match expected markdown (%s)", actualRenderedMarkdown, expectedRenderedMarkdown)
-	}
-}
-
 func TestExtractTOCNormalContent(t *testing.T) {
 	content := []byte("<nav>\n<ul>\nTOC<li><a href=\"#")
 

helpers/pygments_test.go

@@ -45,7 +45,7 @@ func TestParsePygmentsArgs(t *testing.T) {
 		v := viper.New()
 		v.Set("pygmentsStyle", this.pygmentsStyle)
 		v.Set("pygmentsUseClasses", this.pygmentsUseClasses)
-		spec, err := NewContentSpec(v)
+		spec, err := NewContentSpec(v, nil, nil)
 		c.Assert(err, qt.IsNil)
 
 		result1, err := spec.createPygmentsOptionsString(this.in)
@@ -94,7 +94,7 @@ func TestParseDefaultPygmentsArgs(t *testing.T) {
 			v.Set("pygmentsUseClasses", b)
 		}
 
-		spec, err := NewContentSpec(v)
+		spec, err := NewContentSpec(v, nil, nil)
 		c.Assert(err, qt.IsNil)
 
 		result, err := spec.createPygmentsOptionsString(this.in)
@@ -138,7 +138,7 @@ func TestChromaHTMLHighlight(t *testing.T) {
 
 	v := viper.New()
 	v.Set("pygmentsUseClasses", true)
-	spec, err := NewContentSpec(v)
+	spec, err := NewContentSpec(v, nil, nil)
 	c.Assert(err, qt.IsNil)
 
 	result, err := spec.Highlight(`echo "Hello"`, "bash", "")
@@ -206,7 +206,7 @@ func TestChromaHTMLFormatterFromOptions(t *testing.T) {
 			v.Set("pygmentsUseClasses", b)
 		}
 
-		spec, err := NewContentSpec(v)
+		spec, err := NewContentSpec(v, nil, nil)
 		c.Assert(err, qt.IsNil)
 
 		opts, err := spec.parsePygmentsOpts(this.in)
@@ -288,7 +288,7 @@ func GetTitleFunc(style string) func(s string) string {
 }
 `
 
-	spec, err := NewContentSpec(v)
+	spec, err := NewContentSpec(v, nil, nil)
 	c.Assert(err, qt.IsNil)
 
 	for i := 0; i < b.N; i++ {

helpers/testhelpers_test.go

@@ -1,6 +1,8 @@
 package helpers
 
 import (
+	"github.com/gohugoio/hugo/common/loggers"
+	"github.com/spf13/afero"
 	"github.com/spf13/viper"
 
 	"github.com/gohugoio/hugo/hugofs"
@@ -56,7 +58,7 @@ func newTestCfg() *viper.Viper {
 
 func newTestContentSpec() *ContentSpec {
 	v := viper.New()
-	spec, err := NewContentSpec(v)
+	spec, err := NewContentSpec(v, loggers.NewErrorLogger(), afero.NewMemMapFs())
 	if err != nil {
 		panic(err)
 	}

hugolib/config.go

@@ -564,11 +564,6 @@ func (configLoader) mergeStringMapKeepLeft(rootKey, key string, v1, v2 config.Pr
 
 func loadDefaultSettingsFor(v *viper.Viper) error {
 
-	c, err := helpers.NewContentSpec(v)
-	if err != nil {
-		return err
-	}
-
 	v.RegisterAlias("indexes", "taxonomies")
 
 	/*
@@ -616,7 +611,6 @@ func loadDefaultSettingsFor(v *viper.Viper) error {
 	v.SetDefault("paginate", 10)
 	v.SetDefault("paginatePath", "page")
 	v.SetDefault("summaryLength", 70)
-	v.SetDefault("blackfriday", c.BlackFriday)
 	v.SetDefault("rssLimit", -1)
 	v.SetDefault("sectionPagesMenu", "")
 	v.SetDefault("disablePathToLower", false)

hugolib/page.go

@@ -23,6 +23,8 @@ import (
 	"sort"
 	"strings"
 
+	"github.com/gohugoio/hugo/markup/converter"
+
 	"github.com/gohugoio/hugo/common/maps"
 
 	"github.com/gohugoio/hugo/hugofs/files"
@@ -65,7 +67,7 @@ var (
 type pageContext interface {
 	posOffset(offset int) text.Position
 	wrapError(err error) error
-	getRenderingConfig() *helpers.BlackFriday
+	getContentConverter() converter.Converter
 }
 
 // wrapErr adds some context to the given error if possible.
@@ -299,13 +301,6 @@ func (p *pageState) Translations() page.Pages {
 	return p.translations
 }
 
-func (p *pageState) getRenderingConfig() *helpers.BlackFriday {
-	if p.m.renderingConfig == nil {
-		return p.s.ContentSpec.BlackFriday
-	}
-	return p.m.renderingConfig
-}
-
 func (ps *pageState) initCommonProviders(pp pagePaths) error {
 	if ps.IsPage() {
 		ps.posNextPrev = &nextPrev{init: ps.s.init.prevNext}
@@ -516,6 +511,10 @@ func (p *pageState) wrapError(err error) error {
 	return err
 }
 
+func (p *pageState) getContentConverter() converter.Converter {
+	return p.m.contentConverter
+}
+
 func (p *pageState) addResources(r ...resource.Resource) {
 	p.resources = append(p.resources, r...)
 }

hugolib/page__meta.go

@@ -21,6 +21,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/gohugoio/hugo/markup/converter"
+
 	"github.com/gohugoio/hugo/hugofs/files"
 
 	"github.com/gohugoio/hugo/common/hugo"
@@ -29,7 +31,6 @@ import (
 
 	"github.com/gohugoio/hugo/source"
 	"github.com/markbates/inflect"
-	"github.com/mitchellh/mapstructure"
 	"github.com/pkg/errors"
 
 	"github.com/gohugoio/hugo/common/maps"
@@ -123,7 +124,7 @@ type pageMeta struct {
 
 	s *Site
 
-	renderingConfig *helpers.BlackFriday
+	contentConverter converter.Converter
 }
 
 func (p *pageMeta) Aliases() []string {
@@ -598,7 +599,7 @@ func (p *pageMeta) applyDefaultValues() error {
 			p.markup = helpers.GuessType(p.File().Ext())
 		}
 		if p.markup == "" {
-			p.markup = "unknown"
+			p.markup = "markdown"
 		}
 	}
 
@@ -637,17 +638,28 @@ func (p *pageMeta) applyDefaultValues() error {
 		}
 	}
 
-	bfParam := getParamToLower(p, "blackfriday")
+	if !p.f.IsZero() && p.markup != "html" {
-	if bfParam != nil {
+		var renderingConfigOverrides map[string]interface{}
-		p.renderingConfig = p.s.ContentSpec.BlackFriday
+		bfParam := getParamToLower(p, "blackfriday")
-
+		if bfParam != nil {
-		// Create a copy so we can modify it.
+			renderingConfigOverrides = cast.ToStringMap(bfParam)
-		bf := *p.s.ContentSpec.BlackFriday
-		p.renderingConfig = &bf
-		pageParam := cast.ToStringMap(bfParam)
-		if err := mapstructure.Decode(pageParam, &p.renderingConfig); err != nil {
-			return errors.WithMessage(err, "failed to decode rendering config")
 		}
+
+		cp := p.s.ContentSpec.Converters.Get(p.markup)
+		if cp == nil {
+			return errors.Errorf("no content renderer found for markup %q", p.markup)
+		}
+
+		cpp, err := cp.New(converter.DocumentContext{
+			DocumentID:      p.f.UniqueID(),
+			DocumentName:    p.f.Path(),
+			ConfigOverrides: renderingConfigOverrides,
+		})
+
+		if err != nil {
+			return err
+		}
+		p.contentConverter = cpp
 	}
 
 	return nil

hugolib/page__output.go

@@ -45,8 +45,10 @@ func newPageOutput(
 		paginatorProvider = pag
 	}
 
-	var contentProvider page.ContentProvider = page.NopPage
+	var (
-	var tableOfContentsProvider page.TableOfContentsProvider = page.NopPage
+		contentProvider         page.ContentProvider         = page.NopPage
+		tableOfContentsProvider page.TableOfContentsProvider = page.NopPage
+	)
 
 	if cp != nil {
 		contentProvider = cp

hugolib/page__per_output.go

@@ -23,6 +23,8 @@ import (
 	"sync"
 	"unicode/utf8"
 
+	"github.com/gohugoio/hugo/markup/converter"
+
 	"github.com/gohugoio/hugo/lazy"
 
 	bp "github.com/gohugoio/hugo/bufferpool"
@@ -97,7 +99,12 @@ func newPageContentOutput(p *pageState) func(f output.Format) (*pageContentOutpu
 
 			if p.renderable {
 				if !isHTML {
-					cp.workContent = cp.renderContent(p, cp.workContent)
+					r, err := cp.renderContent(cp.workContent)
+					if err != nil {
+						return err
+					}
+					cp.workContent = r.Bytes()
+
 					tmpContent, tmpTableOfContents := helpers.ExtractTOC(cp.workContent)
 					cp.tableOfContents = helpers.BytesToHTML(tmpTableOfContents)
 					cp.workContent = tmpContent
@@ -140,13 +147,16 @@ func newPageContentOutput(p *pageState) func(f output.Format) (*pageContentOutpu
 						}
 					}
 				} else if cp.p.m.summary != "" {
-					html := cp.p.s.ContentSpec.RenderBytes(&helpers.RenderingContext{
+					b, err := cp.p.getContentConverter().Convert(
-						Content: []byte(cp.p.m.summary), RenderTOC: false, PageFmt: cp.p.m.markup,
+						converter.RenderContext{
-						Cfg:        p.Language(),
+							Src: []byte(cp.p.m.summary),
-						BaseFs:     p.s.BaseFs,
+						},
-						DocumentID: p.File().UniqueID(), DocumentName: p.File().Path(),
+					)
-						Config: cp.p.getRenderingConfig()})
+
-					html = cp.p.s.ContentSpec.TrimShortHTML(html)
+					if err != nil {
+						return err
+					}
+					html := cp.p.s.ContentSpec.TrimShortHTML(b.Bytes())
 					cp.summary = helpers.BytesToHTML(html)
 				}
 			}
@@ -311,13 +321,12 @@ func (p *pageContentOutput) setAutoSummary() error {
 
 }
 
-func (cp *pageContentOutput) renderContent(p page.Page, content []byte) []byte {
+func (cp *pageContentOutput) renderContent(content []byte) (converter.Result, error) {
-	return cp.p.s.ContentSpec.RenderBytes(&helpers.RenderingContext{
+	return cp.p.getContentConverter().Convert(
-		Content: content, RenderTOC: true, PageFmt: cp.p.m.markup,
+		converter.RenderContext{
-		Cfg:        p.Language(),
+			Src:       content,
-		BaseFs:     cp.p.s.BaseFs,
+			RenderTOC: true,
-		DocumentID: p.File().UniqueID(), DocumentName: p.File().Path(),
+		})
-		Config: cp.p.getRenderingConfig()})
 }
 
 func (p *pageContentOutput) setWordCounts(isCJKLanguage bool) {

hugolib/page_test.go

@@ -18,6 +18,10 @@ import (
 	"html/template"
 	"os"
 
+	"github.com/gohugoio/hugo/markup/rst"
+
+	"github.com/gohugoio/hugo/markup/asciidoc"
+
 	"github.com/gohugoio/hugo/config"
 
 	"github.com/gohugoio/hugo/common/loggers"
@@ -378,8 +382,8 @@ func testAllMarkdownEnginesForPages(t *testing.T,
 	}{
 		{"md", func() bool { return true }},
 		{"mmark", func() bool { return true }},
-		{"ad", func() bool { return helpers.HasAsciidoc() }},
+		{"ad", func() bool { return asciidoc.Supports() }},
-		{"rst", func() bool { return helpers.HasRst() }},
+		{"rst", func() bool { return rst.Supports() }},
 	}
 
 	for _, e := range engines {

hugolib/shortcode.go

@@ -21,6 +21,8 @@ import (
 	"html/template"
 	"path"
 
+	"github.com/gohugoio/hugo/markup/converter"
+
 	"github.com/gohugoio/hugo/common/herrors"
 	"github.com/pkg/errors"
 
@@ -43,7 +45,6 @@ import (
 	"github.com/gohugoio/hugo/output"
 
 	bp "github.com/gohugoio/hugo/bufferpool"
-	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/tpl"
 )
 
@@ -347,13 +348,19 @@ func renderShortcode(
 		// Pre Hugo 0.55 this was the behaviour even for the outer-most
 		// shortcode.
 		if sc.doMarkup && (level > 0 || sc.info.Config.Version == 1) {
-			newInner := s.ContentSpec.RenderBytes(&helpers.RenderingContext{
+			var err error
-				Content:      []byte(inner),
+
-				PageFmt:      p.m.markup,
+			b, err := p.getContentConverter().Convert(
-				Cfg:          p.Language(),
+				converter.RenderContext{
-				DocumentID:   p.File().UniqueID(),
+					Src: []byte(inner),
-				DocumentName: p.File().Path(),
+				},
-				Config:       p.getRenderingConfig()})
+			)
+
+			if err != nil {
+				return "", false, err
+			}
+
+			newInner := b.Bytes()
 
 			// If the type is “” (unknown) or “markdown”, we assume the markdown
 			// generation has been performed. Given the input: `a line`, markdown

hugolib/shortcode_test.go

@@ -18,6 +18,9 @@ import (
 	"path/filepath"
 	"reflect"
 
+	"github.com/gohugoio/hugo/markup/asciidoc"
+	"github.com/gohugoio/hugo/markup/rst"
+
 	"github.com/spf13/viper"
 
 	"github.com/gohugoio/hugo/parser/pageparser"
@@ -27,7 +30,6 @@ import (
 	"testing"
 
 	"github.com/gohugoio/hugo/deps"
-	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/tpl"
 	"github.com/spf13/cast"
 
@@ -538,6 +540,19 @@ title: "Foo"
 			"<h1>Hugo!</h1>"},
 	}
 
+	temp := tests[:0]
+	for _, test := range tests {
+		if strings.HasSuffix(test.contentPath, ".ad") && !asciidoc.Supports() {
+			t.Log("Skip Asciidoc test case as no Asciidoc present.")
+			continue
+		} else if strings.HasSuffix(test.contentPath, ".rst") && !rst.Supports() {
+			t.Log("Skip Rst test case as no rst2html present.")
+			continue
+		}
+		temp = append(temp, test)
+	}
+	tests = temp
+
 	sources := make([][2]string, len(tests))
 
 	for i, test := range tests {
@@ -578,11 +593,6 @@ title: "Foo"
 		test := test
 		t.Run(fmt.Sprintf("test=%d;contentPath=%s", i, test.contentPath), func(t *testing.T) {
 			t.Parallel()
-			if strings.HasSuffix(test.contentPath, ".ad") && !helpers.HasAsciidoc() {
-				t.Skip("Skip Asciidoc test case as no Asciidoc present.")
-			} else if strings.HasSuffix(test.contentPath, ".rst") && !helpers.HasRst() {
-				t.Skip("Skip Rst test case as no rst2html present.")
-			}
 
 			th := newTestHelper(s.Cfg, s.Fs, t)
 

hugolib/site.go

@@ -28,6 +28,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/gohugoio/hugo/markup/converter"
+
 	"github.com/gohugoio/hugo/hugofs/files"
 
 	"github.com/gohugoio/hugo/common/maps"
@@ -758,17 +760,23 @@ func (s *siteRefLinker) refLink(ref string, source interface{}, relative bool, o
 	}
 
 	if refURL.Fragment != "" {
+		_ = target
 		link = link + "#" + refURL.Fragment
 
-		if pctx, ok := target.(pageContext); ok && !target.File().IsZero() && !pctx.getRenderingConfig().PlainIDAnchors {
+		if pctx, ok := target.(pageContext); ok {
 			if refURL.Path != "" {
-				link = link + ":" + target.File().UniqueID()
+				if di, ok := pctx.getContentConverter().(converter.DocumentInfo); ok {
+					link = link + di.AnchorSuffix()
+				}
+			}
+		} else if pctx, ok := p.(pageContext); ok {
+			if di, ok := pctx.getContentConverter().(converter.DocumentInfo); ok {
+				link = link + di.AnchorSuffix()
 			}
-		} else if pctx, ok := p.(pageContext); ok && !p.File().IsZero() && !pctx.getRenderingConfig().PlainIDAnchors {
-			link = link + ":" + p.File().UniqueID()
 		}
 
 	}
+
 	return link, nil
 }
 

hugolib/site_test.go

@@ -1018,6 +1018,7 @@ func TestRefLinking(t *testing.T) {
 }
 
 func checkLinkCase(site *Site, link string, currentPage page.Page, relative bool, outputFormat string, expected string, t *testing.T, i int) {
+	t.Helper()
 	if out, err := site.refLink(link, currentPage, relative, outputFormat); err != nil || out != expected {
 		t.Fatalf("[%d] Expected %q from %q to resolve to %q, got %q - error: %s", i, link, currentPage.Path(), expected, out, err)
 	}

markup/asciidoc/convert.go

@@ -0,0 +1,97 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package asciidoc converts Asciidoc to HTML using Asciidoc or Asciidoctor
+// external binaries.
+package asciidoc
+
+import (
+	"os/exec"
+
+	"github.com/gohugoio/hugo/markup/internal"
+
+	"github.com/gohugoio/hugo/markup/converter"
+)
+
+// Provider is the package entry point.
+var Provider converter.NewProvider = provider{}
+
+type provider struct {
+}
+
+func (p provider) New(cfg converter.ProviderConfig) (converter.Provider, error) {
+	var n converter.NewConverter = func(ctx converter.DocumentContext) (converter.Converter, error) {
+		return &asciidocConverter{
+			ctx: ctx,
+			cfg: cfg,
+		}, nil
+	}
+	return n, nil
+}
+
+type asciidocConverter struct {
+	ctx converter.DocumentContext
+	cfg converter.ProviderConfig
+}
+
+func (a *asciidocConverter) Convert(ctx converter.RenderContext) (converter.Result, error) {
+	return converter.Bytes(a.getAsciidocContent(ctx.Src, a.ctx)), nil
+}
+
+// getAsciidocContent calls asciidoctor or asciidoc as an external helper
+// to convert AsciiDoc content to HTML.
+func (a *asciidocConverter) getAsciidocContent(src []byte, ctx converter.DocumentContext) []byte {
+	var isAsciidoctor bool
+	path := getAsciidoctorExecPath()
+	if path == "" {
+		path = getAsciidocExecPath()
+		if path == "" {
+			a.cfg.Logger.ERROR.Println("asciidoctor / asciidoc not found in $PATH: Please install.\n",
+				"                 Leaving AsciiDoc content unrendered.")
+			return src
+		}
+	} else {
+		isAsciidoctor = true
+	}
+
+	a.cfg.Logger.INFO.Println("Rendering", ctx.DocumentName, "with", path, "...")
+	args := []string{"--no-header-footer", "--safe"}
+	if isAsciidoctor {
+		// asciidoctor-specific arg to show stack traces on errors
+		args = append(args, "--trace")
+	}
+	args = append(args, "-")
+	return internal.ExternallyRenderContent(a.cfg, ctx, src, path, args)
+}
+
+func getAsciidocExecPath() string {
+	path, err := exec.LookPath("asciidoc")
+	if err != nil {
+		return ""
+	}
+	return path
+}
+
+func getAsciidoctorExecPath() string {
+	path, err := exec.LookPath("asciidoctor")
+	if err != nil {
+		return ""
+	}
+	return path
+}
+
+// Supports returns whether Asciidoc or Asciidoctor is installed on this computer.
+func Supports() bool {
+	return (getAsciidoctorExecPath() != "" ||
+		getAsciidocExecPath() != "")
+}

markup/asciidoc/convert_test.go

@@ -0,0 +1,38 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package asciidoc
+
+import (
+	"testing"
+
+	"github.com/gohugoio/hugo/common/loggers"
+
+	"github.com/gohugoio/hugo/markup/converter"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestConvert(t *testing.T) {
+	if !Supports() {
+		t.Skip("asciidoc/asciidoctor not installed")
+	}
+	c := qt.New(t)
+	p, err := Provider.New(converter.ProviderConfig{Logger: loggers.NewErrorLogger()})
+	c.Assert(err, qt.IsNil)
+	conv, err := p.New(converter.DocumentContext{})
+	c.Assert(err, qt.IsNil)
+	b, err := conv.Convert(converter.RenderContext{Src: []byte("testContent")})
+	c.Assert(err, qt.IsNil)
+	c.Assert(string(b.Bytes()), qt.Equals, "<div class=\"paragraph\">\n<p>testContent</p>\n</div>\n")
+}

markup/blackfriday/convert.go

@@ -0,0 +1,224 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package blackfriday converts Markdown to HTML using Blackfriday v1.
+package blackfriday
+
+import (
+	"github.com/gohugoio/hugo/markup/converter"
+	"github.com/gohugoio/hugo/markup/internal"
+	"github.com/russross/blackfriday"
+)
+
+// Provider is the package entry point.
+var Provider converter.NewProvider = provider{}
+
+type provider struct {
+}
+
+func (p provider) New(cfg converter.ProviderConfig) (converter.Provider, error) {
+	defaultBlackFriday, err := internal.NewBlackfriday(cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	defaultExtensions := getMarkdownExtensions(defaultBlackFriday)
+
+	pygmentsCodeFences := cfg.Cfg.GetBool("pygmentsCodeFences")
+	pygmentsCodeFencesGuessSyntax := cfg.Cfg.GetBool("pygmentsCodeFencesGuessSyntax")
+	pygmentsOptions := cfg.Cfg.GetString("pygmentsOptions")
+
+	var n converter.NewConverter = func(ctx converter.DocumentContext) (converter.Converter, error) {
+		b := defaultBlackFriday
+		extensions := defaultExtensions
+
+		if ctx.ConfigOverrides != nil {
+			var err error
+			b, err = internal.UpdateBlackFriday(b, ctx.ConfigOverrides)
+			if err != nil {
+				return nil, err
+			}
+			extensions = getMarkdownExtensions(b)
+		}
+
+		return &blackfridayConverter{
+			ctx:        ctx,
+			bf:         b,
+			extensions: extensions,
+			cfg:        cfg,
+
+			pygmentsCodeFences:            pygmentsCodeFences,
+			pygmentsCodeFencesGuessSyntax: pygmentsCodeFencesGuessSyntax,
+			pygmentsOptions:               pygmentsOptions,
+		}, nil
+	}
+
+	return n, nil
+
+}
+
+type blackfridayConverter struct {
+	ctx        converter.DocumentContext
+	bf         *internal.BlackFriday
+	extensions int
+
+	pygmentsCodeFences            bool
+	pygmentsCodeFencesGuessSyntax bool
+	pygmentsOptions               string
+
+	cfg converter.ProviderConfig
+}
+
+func (c *blackfridayConverter) AnchorSuffix() string {
+	if c.bf.PlainIDAnchors {
+		return ""
+	}
+	return ":" + c.ctx.DocumentID
+}
+
+func (c *blackfridayConverter) Convert(ctx converter.RenderContext) (converter.Result, error) {
+	r := c.getHTMLRenderer(ctx.RenderTOC)
+
+	return converter.Bytes(blackfriday.Markdown(ctx.Src, r, c.extensions)), nil
+
+}
+
+func (c *blackfridayConverter) getHTMLRenderer(renderTOC bool) blackfriday.Renderer {
+	flags := getFlags(renderTOC, c.bf)
+
+	documentID := c.ctx.DocumentID
+
+	renderParameters := blackfriday.HtmlRendererParameters{
+		FootnoteAnchorPrefix:       c.bf.FootnoteAnchorPrefix,
+		FootnoteReturnLinkContents: c.bf.FootnoteReturnLinkContents,
+	}
+
+	if documentID != "" && !c.bf.PlainIDAnchors {
+		renderParameters.FootnoteAnchorPrefix = documentID + ":" + renderParameters.FootnoteAnchorPrefix
+		renderParameters.HeaderIDSuffix = ":" + documentID
+	}
+
+	return &hugoHTMLRenderer{
+		c:        c,
+		Renderer: blackfriday.HtmlRendererWithParameters(flags, "", "", renderParameters),
+	}
+}
+
+func getFlags(renderTOC bool, cfg *internal.BlackFriday) int {
+
+	var flags int
+
+	if renderTOC {
+		flags = blackfriday.HTML_TOC
+	}
+
+	flags |= blackfriday.HTML_USE_XHTML
+	flags |= blackfriday.HTML_FOOTNOTE_RETURN_LINKS
+
+	if cfg.Smartypants {
+		flags |= blackfriday.HTML_USE_SMARTYPANTS
+	}
+
+	if cfg.SmartypantsQuotesNBSP {
+		flags |= blackfriday.HTML_SMARTYPANTS_QUOTES_NBSP
+	}
+
+	if cfg.AngledQuotes {
+		flags |= blackfriday.HTML_SMARTYPANTS_ANGLED_QUOTES
+	}
+
+	if cfg.Fractions {
+		flags |= blackfriday.HTML_SMARTYPANTS_FRACTIONS
+	}
+
+	if cfg.HrefTargetBlank {
+		flags |= blackfriday.HTML_HREF_TARGET_BLANK
+	}
+
+	if cfg.NofollowLinks {
+		flags |= blackfriday.HTML_NOFOLLOW_LINKS
+	}
+
+	if cfg.NoreferrerLinks {
+		flags |= blackfriday.HTML_NOREFERRER_LINKS
+	}
+
+	if cfg.SmartDashes {
+		flags |= blackfriday.HTML_SMARTYPANTS_DASHES
+	}
+
+	if cfg.LatexDashes {
+		flags |= blackfriday.HTML_SMARTYPANTS_LATEX_DASHES
+	}
+
+	if cfg.SkipHTML {
+		flags |= blackfriday.HTML_SKIP_HTML
+	}
+
+	return flags
+}
+
+func getMarkdownExtensions(cfg *internal.BlackFriday) int {
+	// Default Blackfriday common extensions
+	commonExtensions := 0 |
+		blackfriday.EXTENSION_NO_INTRA_EMPHASIS |
+		blackfriday.EXTENSION_TABLES |
+		blackfriday.EXTENSION_FENCED_CODE |
+		blackfriday.EXTENSION_AUTOLINK |
+		blackfriday.EXTENSION_STRIKETHROUGH |
+		blackfriday.EXTENSION_SPACE_HEADERS |
+		blackfriday.EXTENSION_HEADER_IDS |
+		blackfriday.EXTENSION_BACKSLASH_LINE_BREAK |
+		blackfriday.EXTENSION_DEFINITION_LISTS
+
+	// Extra Blackfriday extensions that Hugo enables by default
+	flags := commonExtensions |
+		blackfriday.EXTENSION_AUTO_HEADER_IDS |
+		blackfriday.EXTENSION_FOOTNOTES
+
+	for _, extension := range cfg.Extensions {
+		if flag, ok := blackfridayExtensionMap[extension]; ok {
+			flags |= flag
+		}
+	}
+	for _, extension := range cfg.ExtensionsMask {
+		if flag, ok := blackfridayExtensionMap[extension]; ok {
+			flags &= ^flag
+		}
+	}
+	return flags
+}
+
+var blackfridayExtensionMap = map[string]int{
+	"noIntraEmphasis":        blackfriday.EXTENSION_NO_INTRA_EMPHASIS,
+	"tables":                 blackfriday.EXTENSION_TABLES,
+	"fencedCode":             blackfriday.EXTENSION_FENCED_CODE,
+	"autolink":               blackfriday.EXTENSION_AUTOLINK,
+	"strikethrough":          blackfriday.EXTENSION_STRIKETHROUGH,
+	"laxHtmlBlocks":          blackfriday.EXTENSION_LAX_HTML_BLOCKS,
+	"spaceHeaders":           blackfriday.EXTENSION_SPACE_HEADERS,
+	"hardLineBreak":          blackfriday.EXTENSION_HARD_LINE_BREAK,
+	"tabSizeEight":           blackfriday.EXTENSION_TAB_SIZE_EIGHT,
+	"footnotes":              blackfriday.EXTENSION_FOOTNOTES,
+	"noEmptyLineBeforeBlock": blackfriday.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK,
+	"headerIds":              blackfriday.EXTENSION_HEADER_IDS,
+	"titleblock":             blackfriday.EXTENSION_TITLEBLOCK,
+	"autoHeaderIds":          blackfriday.EXTENSION_AUTO_HEADER_IDS,
+	"backslashLineBreak":     blackfriday.EXTENSION_BACKSLASH_LINE_BREAK,
+	"definitionLists":        blackfriday.EXTENSION_DEFINITION_LISTS,
+	"joinLines":              blackfriday.EXTENSION_JOIN_LINES,
+}
+
+var (
+	_ converter.DocumentInfo = (*blackfridayConverter)(nil)
+)

markup/blackfriday/convert_test.go

@@ -0,0 +1,194 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package blackfriday
+
+import (
+	"testing"
+
+	"github.com/spf13/viper"
+
+	"github.com/gohugoio/hugo/markup/internal"
+
+	"github.com/gohugoio/hugo/markup/converter"
+
+	qt "github.com/frankban/quicktest"
+	"github.com/russross/blackfriday"
+)
+
+func TestGetMarkdownExtensionsMasksAreRemovedFromExtensions(t *testing.T) {
+	c := qt.New(t)
+	b, err := internal.NewBlackfriday(converter.ProviderConfig{Cfg: viper.New()})
+	c.Assert(err, qt.IsNil)
+
+	b.Extensions = []string{"headerId"}
+	b.ExtensionsMask = []string{"noIntraEmphasis"}
+
+	actualFlags := getMarkdownExtensions(b)
+	if actualFlags&blackfriday.EXTENSION_NO_INTRA_EMPHASIS == blackfriday.EXTENSION_NO_INTRA_EMPHASIS {
+		t.Errorf("Masked out flag {%v} found amongst returned extensions.", blackfriday.EXTENSION_NO_INTRA_EMPHASIS)
+	}
+}
+
+func TestGetMarkdownExtensionsByDefaultAllExtensionsAreEnabled(t *testing.T) {
+	type data struct {
+		testFlag int
+	}
+
+	c := qt.New(t)
+	b, err := internal.NewBlackfriday(converter.ProviderConfig{Cfg: viper.New()})
+	c.Assert(err, qt.IsNil)
+
+	b.Extensions = []string{""}
+	b.ExtensionsMask = []string{""}
+	allExtensions := []data{
+		{blackfriday.EXTENSION_NO_INTRA_EMPHASIS},
+		{blackfriday.EXTENSION_TABLES},
+		{blackfriday.EXTENSION_FENCED_CODE},
+		{blackfriday.EXTENSION_AUTOLINK},
+		{blackfriday.EXTENSION_STRIKETHROUGH},
+		// {blackfriday.EXTENSION_LAX_HTML_BLOCKS},
+		{blackfriday.EXTENSION_SPACE_HEADERS},
+		// {blackfriday.EXTENSION_HARD_LINE_BREAK},
+		// {blackfriday.EXTENSION_TAB_SIZE_EIGHT},
+		{blackfriday.EXTENSION_FOOTNOTES},
+		// {blackfriday.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK},
+		{blackfriday.EXTENSION_HEADER_IDS},
+		// {blackfriday.EXTENSION_TITLEBLOCK},
+		{blackfriday.EXTENSION_AUTO_HEADER_IDS},
+		{blackfriday.EXTENSION_BACKSLASH_LINE_BREAK},
+		{blackfriday.EXTENSION_DEFINITION_LISTS},
+	}
+
+	actualFlags := getMarkdownExtensions(b)
+	for _, e := range allExtensions {
+		if actualFlags&e.testFlag != e.testFlag {
+			t.Errorf("Flag %v was not found in the list of extensions.", e)
+		}
+	}
+}
+
+func TestGetMarkdownExtensionsAddingFlagsThroughRenderingContext(t *testing.T) {
+	c := qt.New(t)
+	b, err := internal.NewBlackfriday(converter.ProviderConfig{Cfg: viper.New()})
+	c.Assert(err, qt.IsNil)
+
+	b.Extensions = []string{"definitionLists"}
+	b.ExtensionsMask = []string{""}
+
+	actualFlags := getMarkdownExtensions(b)
+	if actualFlags&blackfriday.EXTENSION_DEFINITION_LISTS != blackfriday.EXTENSION_DEFINITION_LISTS {
+		t.Errorf("Masked out flag {%v} found amongst returned extensions.", blackfriday.EXTENSION_DEFINITION_LISTS)
+	}
+}
+
+func TestGetFlags(t *testing.T) {
+	c := qt.New(t)
+	cfg := converter.ProviderConfig{Cfg: viper.New()}
+	b, err := internal.NewBlackfriday(cfg)
+	c.Assert(err, qt.IsNil)
+	flags := getFlags(false, b)
+	if flags&blackfriday.HTML_USE_XHTML != blackfriday.HTML_USE_XHTML {
+		t.Errorf("Test flag: %d was not found amongs set flags:%d; Result: %d", blackfriday.HTML_USE_XHTML, flags, flags&blackfriday.HTML_USE_XHTML)
+	}
+}
+
+func TestGetAllFlags(t *testing.T) {
+	c := qt.New(t)
+	cfg := converter.ProviderConfig{Cfg: viper.New()}
+	b, err := internal.NewBlackfriday(cfg)
+	c.Assert(err, qt.IsNil)
+
+	type data struct {
+		testFlag int
+	}
+
+	allFlags := []data{
+		{blackfriday.HTML_USE_XHTML},
+		{blackfriday.HTML_FOOTNOTE_RETURN_LINKS},
+		{blackfriday.HTML_USE_SMARTYPANTS},
+		{blackfriday.HTML_SMARTYPANTS_QUOTES_NBSP},
+		{blackfriday.HTML_SMARTYPANTS_ANGLED_QUOTES},
+		{blackfriday.HTML_SMARTYPANTS_FRACTIONS},
+		{blackfriday.HTML_HREF_TARGET_BLANK},
+		{blackfriday.HTML_NOFOLLOW_LINKS},
+		{blackfriday.HTML_NOREFERRER_LINKS},
+		{blackfriday.HTML_SMARTYPANTS_DASHES},
+		{blackfriday.HTML_SMARTYPANTS_LATEX_DASHES},
+	}
+
+	b.AngledQuotes = true
+	b.Fractions = true
+	b.HrefTargetBlank = true
+	b.NofollowLinks = true
+	b.NoreferrerLinks = true
+	b.LatexDashes = true
+	b.PlainIDAnchors = true
+	b.SmartDashes = true
+	b.Smartypants = true
+	b.SmartypantsQuotesNBSP = true
+
+	actualFlags := getFlags(false, b)
+
+	var expectedFlags int
+	//OR-ing flags together...
+	for _, d := range allFlags {
+		expectedFlags |= d.testFlag
+	}
+	if expectedFlags != actualFlags {
+		t.Errorf("Expected flags (%d) did not equal actual (%d) flags.", expectedFlags, actualFlags)
+	}
+}
+
+func TestConvert(t *testing.T) {
+	c := qt.New(t)
+	p, err := Provider.New(converter.ProviderConfig{
+		Cfg: viper.New(),
+	})
+	c.Assert(err, qt.IsNil)
+	conv, err := p.New(converter.DocumentContext{})
+	c.Assert(err, qt.IsNil)
+	b, err := conv.Convert(converter.RenderContext{Src: []byte("testContent")})
+	c.Assert(err, qt.IsNil)
+	c.Assert(string(b.Bytes()), qt.Equals, "<p>testContent</p>\n")
+}
+
+func TestGetHTMLRendererAnchors(t *testing.T) {
+	c := qt.New(t)
+	p, err := Provider.New(converter.ProviderConfig{
+		Cfg: viper.New(),
+	})
+	c.Assert(err, qt.IsNil)
+	conv, err := p.New(converter.DocumentContext{
+		DocumentID: "testid",
+		ConfigOverrides: map[string]interface{}{
+			"plainIDAnchors": false,
+			"footnotes":      true,
+		},
+	})
+	c.Assert(err, qt.IsNil)
+	b, err := conv.Convert(converter.RenderContext{Src: []byte(`# Header
+
+This is a footnote.[^1] And then some.
+
+
+[^1]: Footnote text.
+
+`)})
+
+	c.Assert(err, qt.IsNil)
+	s := string(b.Bytes())
+	c.Assert(s, qt.Contains, "<h1 id=\"header:testid\">Header</h1>")
+	c.Assert(s, qt.Contains, "This is a footnote.<sup class=\"footnote-ref\" id=\"fnref:testid:1\"><a href=\"#fn:testid:1\">1</a></sup>")
+	c.Assert(s, qt.Contains, "<a class=\"footnote-return\" href=\"#fnref:testid:1\"><sup>[return]</sup></a>")
+}

markup/blackfriday/renderer.go

@@ -1,4 +1,4 @@
-// Copyright 2016 The Hugo Authors. All rights reserved.
+// Copyright 2019 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,32 +11,29 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package helpers
+package blackfriday
 
 import (
 	"bytes"
 	"strings"
 
-	"github.com/gohugoio/hugo/config"
-	"github.com/miekg/mmark"
 	"github.com/russross/blackfriday"
 )
 
-// HugoHTMLRenderer wraps a blackfriday.Renderer, typically a blackfriday.Html
+// hugoHTMLRenderer wraps a blackfriday.Renderer, typically a blackfriday.Html
-// Enabling Hugo to customise the rendering experience
+// adding some custom behaviour.
-type HugoHTMLRenderer struct {
+type hugoHTMLRenderer struct {
-	cs *ContentSpec
+	c *blackfridayConverter
-	*RenderingContext
 	blackfriday.Renderer
 }
 
 // BlockCode renders a given text as a block of code.
 // Pygments is used if it is setup to handle code fences.
-func (r *HugoHTMLRenderer) BlockCode(out *bytes.Buffer, text []byte, lang string) {
+func (r *hugoHTMLRenderer) BlockCode(out *bytes.Buffer, text []byte, lang string) {
-	if r.Cfg.GetBool("pygmentsCodeFences") && (lang != "" || r.Cfg.GetBool("pygmentsCodeFencesGuessSyntax")) {
+	if r.c.pygmentsCodeFences && (lang != "" || r.c.pygmentsCodeFencesGuessSyntax) {
-		opts := r.Cfg.GetString("pygmentsOptions")
+		opts := r.c.pygmentsOptions
 		str := strings.Trim(string(text), "\n\r")
-		highlighted, _ := r.cs.Highlight(str, lang, opts)
+		highlighted, _ := r.c.cfg.Highlight(str, lang, opts)
 		out.WriteString(highlighted)
 	} else {
 		r.Renderer.BlockCode(out, text, lang)
@@ -44,8 +41,8 @@ func (r *HugoHTMLRenderer) BlockCode(out *bytes.Buffer, text []byte, lang string
 }
 
 // ListItem adds task list support to the Blackfriday renderer.
-func (r *HugoHTMLRenderer) ListItem(out *bytes.Buffer, text []byte, flags int) {
+func (r *hugoHTMLRenderer) ListItem(out *bytes.Buffer, text []byte, flags int) {
-	if !r.Config.TaskLists {
+	if !r.c.bf.TaskLists {
 		r.Renderer.ListItem(out, text, flags)
 		return
 	}
@@ -64,8 +61,8 @@ func (r *HugoHTMLRenderer) ListItem(out *bytes.Buffer, text []byte, flags int) {
 }
 
 // List adds task list support to the Blackfriday renderer.
-func (r *HugoHTMLRenderer) List(out *bytes.Buffer, text func() bool, flags int) {
+func (r *hugoHTMLRenderer) List(out *bytes.Buffer, text func() bool, flags int) {
-	if !r.Config.TaskLists {
+	if !r.c.bf.TaskLists {
 		r.Renderer.List(out, text, flags)
 		return
 	}
@@ -86,23 +83,3 @@ func (r *HugoHTMLRenderer) List(out *bytes.Buffer, text func() bool, flags int)
 		}
 	}
 }
-
-// HugoMmarkHTMLRenderer wraps a mmark.Renderer, typically a mmark.html,
-// enabling Hugo to customise the rendering experience.
-type HugoMmarkHTMLRenderer struct {
-	cs *ContentSpec
-	mmark.Renderer
-	Cfg config.Provider
-}
-
-// BlockCode renders a given text as a block of code.
-// Pygments is used if it is setup to handle code fences.
-func (r *HugoMmarkHTMLRenderer) BlockCode(out *bytes.Buffer, text []byte, lang string, caption []byte, subfigure bool, callouts bool) {
-	if r.Cfg.GetBool("pygmentsCodeFences") && (lang != "" || r.Cfg.GetBool("pygmentsCodeFencesGuessSyntax")) {
-		str := strings.Trim(string(text), "\n\r")
-		highlighted, _ := r.cs.Highlight(str, lang, "")
-		out.WriteString(highlighted)
-	} else {
-		r.Renderer.BlockCode(out, text, lang, caption, subfigure, callouts)
-	}
-}

markup/converter/converter.go

@@ -0,0 +1,83 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package converter
+
+import (
+	"github.com/gohugoio/hugo/common/loggers"
+	"github.com/gohugoio/hugo/config"
+	"github.com/spf13/afero"
+)
+
+// ProviderConfig configures a new Provider.
+type ProviderConfig struct {
+	Cfg       config.Provider // Site config
+	ContentFs afero.Fs
+	Logger    *loggers.Logger
+	Highlight func(code, lang, optsStr string) (string, error)
+}
+
+// NewProvider creates converter providers.
+type NewProvider interface {
+	New(cfg ProviderConfig) (Provider, error)
+}
+
+// Provider creates converters.
+type Provider interface {
+	New(ctx DocumentContext) (Converter, error)
+}
+
+// NewConverter is an adapter that can be used as a ConverterProvider.
+type NewConverter func(ctx DocumentContext) (Converter, error)
+
+// New creates a new Converter for the given ctx.
+func (n NewConverter) New(ctx DocumentContext) (Converter, error) {
+	return n(ctx)
+}
+
+// Converter wraps the Convert method that converts some markup into
+// another format, e.g. Markdown to HTML.
+type Converter interface {
+	Convert(ctx RenderContext) (Result, error)
+}
+
+// Result represents the minimum returned from Convert.
+type Result interface {
+	Bytes() []byte
+}
+
+// DocumentInfo holds additional information provided by some converters.
+type DocumentInfo interface {
+	AnchorSuffix() string
+}
+
+// Bytes holds a byte slice and implements the Result interface.
+type Bytes []byte
+
+// Bytes returns itself
+func (b Bytes) Bytes() []byte {
+	return b
+}
+
+// DocumentContext holds contextual information about the document to convert.
+type DocumentContext struct {
+	DocumentID      string
+	DocumentName    string
+	ConfigOverrides map[string]interface{}
+}
+
+// RenderContext holds contextual information about the content to render.
+type RenderContext struct {
+	Src       []byte
+	RenderTOC bool
+}

markup/internal/blackfriday.go

@@ -0,0 +1,108 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package helpers implements general utility functions that work with
+// and on content.  The helper functions defined here lay down the
+// foundation of how Hugo works with files and filepaths, and perform
+// string operations on content.
+
+package internal
+
+import (
+	"github.com/gohugoio/hugo/common/maps"
+	"github.com/gohugoio/hugo/markup/converter"
+	"github.com/mitchellh/mapstructure"
+	"github.com/pkg/errors"
+)
+
+// BlackFriday holds configuration values for BlackFriday rendering.
+// It is kept here because it's used in several packages.
+type BlackFriday struct {
+	Smartypants           bool
+	SmartypantsQuotesNBSP bool
+	AngledQuotes          bool
+	Fractions             bool
+	HrefTargetBlank       bool
+	NofollowLinks         bool
+	NoreferrerLinks       bool
+	SmartDashes           bool
+	LatexDashes           bool
+	TaskLists             bool
+	PlainIDAnchors        bool
+	Extensions            []string
+	ExtensionsMask        []string
+	SkipHTML              bool
+
+	FootnoteAnchorPrefix       string
+	FootnoteReturnLinkContents string
+}
+
+func UpdateBlackFriday(old *BlackFriday, m map[string]interface{}) (*BlackFriday, error) {
+	// Create a copy so we can modify it.
+	bf := *old
+	if err := mapstructure.Decode(m, &bf); err != nil {
+		return nil, errors.WithMessage(err, "failed to decode rendering config")
+	}
+	return &bf, nil
+}
+
+// NewBlackfriday creates a new Blackfriday filled with site config or some sane defaults.
+func NewBlackfriday(cfg converter.ProviderConfig) (*BlackFriday, error) {
+	var siteConfig map[string]interface{}
+	if cfg.Cfg != nil {
+		siteConfig = cfg.Cfg.GetStringMap("blackfriday")
+	}
+
+	defaultParam := map[string]interface{}{
+		"smartypants":           true,
+		"angledQuotes":          false,
+		"smartypantsQuotesNBSP": false,
+		"fractions":             true,
+		"hrefTargetBlank":       false,
+		"nofollowLinks":         false,
+		"noreferrerLinks":       false,
+		"smartDashes":           true,
+		"latexDashes":           true,
+		"plainIDAnchors":        true,
+		"taskLists":             true,
+		"skipHTML":              false,
+	}
+
+	maps.ToLower(defaultParam)
+
+	config := make(map[string]interface{})
+
+	for k, v := range defaultParam {
+		config[k] = v
+	}
+
+	for k, v := range siteConfig {
+		config[k] = v
+	}
+
+	combinedConfig := &BlackFriday{}
+	if err := mapstructure.Decode(config, combinedConfig); err != nil {
+		return nil, errors.Errorf("failed to decode Blackfriday config: %s", err)
+	}
+
+	// TODO(bep) update/consolidate docs
+	if combinedConfig.FootnoteAnchorPrefix == "" {
+		combinedConfig.FootnoteAnchorPrefix = cfg.Cfg.GetString("footnoteAnchorPrefix")
+	}
+
+	if combinedConfig.FootnoteReturnLinkContents == "" {
+		combinedConfig.FootnoteReturnLinkContents = cfg.Cfg.GetString("footnoteReturnLinkContents")
+	}
+
+	return combinedConfig, nil
+}

markup/internal/external.go

@@ -0,0 +1,52 @@
+package internal
+
+import (
+	"bytes"
+	"os/exec"
+	"strings"
+
+	"github.com/gohugoio/hugo/markup/converter"
+)
+
+func ExternallyRenderContent(
+	cfg converter.ProviderConfig,
+	ctx converter.DocumentContext,
+	content []byte, path string, args []string) []byte {
+
+	logger := cfg.Logger
+	cmd := exec.Command(path, args...)
+	cmd.Stdin = bytes.NewReader(content)
+	var out, cmderr bytes.Buffer
+	cmd.Stdout = &out
+	cmd.Stderr = &cmderr
+	err := cmd.Run()
+	// Most external helpers exit w/ non-zero exit code only if severe, i.e.
+	// halting errors occurred. -> log stderr output regardless of state of err
+	for _, item := range strings.Split(cmderr.String(), "\n") {
+		item := strings.TrimSpace(item)
+		if item != "" {
+			logger.ERROR.Printf("%s: %s", ctx.DocumentName, item)
+		}
+	}
+	if err != nil {
+		logger.ERROR.Printf("%s rendering %s: %v", path, ctx.DocumentName, err)
+	}
+
+	return normalizeExternalHelperLineFeeds(out.Bytes())
+}
+
+// Strips carriage returns from third-party / external processes (useful for Windows)
+func normalizeExternalHelperLineFeeds(content []byte) []byte {
+	return bytes.Replace(content, []byte("\r"), []byte(""), -1)
+}
+
+func GetPythonExecPath() string {
+	path, err := exec.LookPath("python")
+	if err != nil {
+		path, err = exec.LookPath("python.exe")
+		if err != nil {
+			return ""
+		}
+	}
+	return path
+}

markup/markup.go

@@ -0,0 +1,83 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package markup
+
+import (
+	"strings"
+
+	"github.com/gohugoio/hugo/markup/org"
+
+	"github.com/gohugoio/hugo/markup/asciidoc"
+	"github.com/gohugoio/hugo/markup/blackfriday"
+	"github.com/gohugoio/hugo/markup/converter"
+	"github.com/gohugoio/hugo/markup/mmark"
+	"github.com/gohugoio/hugo/markup/pandoc"
+	"github.com/gohugoio/hugo/markup/rst"
+)
+
+func NewConverterProvider(cfg converter.ProviderConfig) (ConverterProvider, error) {
+	converters := make(map[string]converter.Provider)
+
+	add := func(p converter.NewProvider, aliases ...string) error {
+		c, err := p.New(cfg)
+		if err != nil {
+			return err
+		}
+		addConverter(converters, c, aliases...)
+		return nil
+	}
+
+	if err := add(blackfriday.Provider, "md", "markdown", "blackfriday"); err != nil {
+		return nil, err
+	}
+	if err := add(mmark.Provider, "mmark"); err != nil {
+		return nil, err
+	}
+	if err := add(asciidoc.Provider, "asciidoc"); err != nil {
+		return nil, err
+	}
+	if err := add(rst.Provider, "rst"); err != nil {
+		return nil, err
+	}
+	if err := add(pandoc.Provider, "pandoc"); err != nil {
+		return nil, err
+	}
+	if err := add(org.Provider, "org"); err != nil {
+		return nil, err
+	}
+
+	return &converterRegistry{converters: converters}, nil
+}
+
+type ConverterProvider interface {
+	Get(name string) converter.Provider
+}
+
+type converterRegistry struct {
+	// Maps name (md, markdown, blackfriday etc.) to a converter provider.
+	// Note that this is also used for aliasing, so the same converter
+	// may be registered multiple times.
+	// All names are lower case.
+	converters map[string]converter.Provider
+}
+
+func (r *converterRegistry) Get(name string) converter.Provider {
+	return r.converters[strings.ToLower(name)]
+}
+
+func addConverter(m map[string]converter.Provider, c converter.Provider, aliases ...string) {
+	for _, alias := range aliases {
+		m[alias] = c
+	}
+}

markup/markup_test.go

@@ -0,0 +1,41 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package markup
+
+import (
+	"testing"
+
+	"github.com/spf13/viper"
+
+	"github.com/gohugoio/hugo/markup/converter"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestConverterRegistry(t *testing.T) {
+	c := qt.New(t)
+
+	r, err := NewConverterProvider(converter.ProviderConfig{Cfg: viper.New()})
+
+	c.Assert(err, qt.IsNil)
+
+	c.Assert(r.Get("foo"), qt.IsNil)
+	c.Assert(r.Get("markdown"), qt.Not(qt.IsNil))
+	c.Assert(r.Get("mmark"), qt.Not(qt.IsNil))
+	c.Assert(r.Get("asciidoc"), qt.Not(qt.IsNil))
+	c.Assert(r.Get("rst"), qt.Not(qt.IsNil))
+	c.Assert(r.Get("pandoc"), qt.Not(qt.IsNil))
+	c.Assert(r.Get("org"), qt.Not(qt.IsNil))
+
+}

markup/mmark/convert.go

@@ -0,0 +1,143 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package mmark converts Markdown to HTML using MMark v1.
+package mmark
+
+import (
+	"github.com/gohugoio/hugo/markup/internal"
+
+	"github.com/gohugoio/hugo/markup/converter"
+	"github.com/miekg/mmark"
+)
+
+// Provider is the package entry point.
+var Provider converter.NewProvider = provider{}
+
+type provider struct {
+}
+
+func (p provider) New(cfg converter.ProviderConfig) (converter.Provider, error) {
+	defaultBlackFriday, err := internal.NewBlackfriday(cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	defaultExtensions := getMmarkExtensions(defaultBlackFriday)
+
+	var n converter.NewConverter = func(ctx converter.DocumentContext) (converter.Converter, error) {
+		b := defaultBlackFriday
+		extensions := defaultExtensions
+
+		if ctx.ConfigOverrides != nil {
+			var err error
+			b, err = internal.UpdateBlackFriday(b, ctx.ConfigOverrides)
+			if err != nil {
+				return nil, err
+			}
+			extensions = getMmarkExtensions(b)
+		}
+
+		return &mmarkConverter{
+			ctx:        ctx,
+			b:          b,
+			extensions: extensions,
+			cfg:        cfg,
+		}, nil
+	}
+
+	return n, nil
+
+}
+
+type mmarkConverter struct {
+	ctx        converter.DocumentContext
+	extensions int
+	b          *internal.BlackFriday
+	cfg        converter.ProviderConfig
+}
+
+func (c *mmarkConverter) Convert(ctx converter.RenderContext) (converter.Result, error) {
+	r := getHTMLRenderer(c.ctx, c.b, c.cfg)
+	return mmark.Parse(ctx.Src, r, c.extensions), nil
+}
+
+func getHTMLRenderer(
+	ctx converter.DocumentContext,
+	cfg *internal.BlackFriday,
+	pcfg converter.ProviderConfig) mmark.Renderer {
+
+	var (
+		flags      int
+		documentID string
+	)
+
+	documentID = ctx.DocumentID
+
+	renderParameters := mmark.HtmlRendererParameters{
+		FootnoteAnchorPrefix:       cfg.FootnoteAnchorPrefix,
+		FootnoteReturnLinkContents: cfg.FootnoteReturnLinkContents,
+	}
+
+	if documentID != "" && !cfg.PlainIDAnchors {
+		renderParameters.FootnoteAnchorPrefix = documentID + ":" + renderParameters.FootnoteAnchorPrefix
+	}
+
+	htmlFlags := flags
+	htmlFlags |= mmark.HTML_FOOTNOTE_RETURN_LINKS
+
+	return &mmarkRenderer{
+		Config:    cfg,
+		Cfg:       pcfg.Cfg,
+		highlight: pcfg.Highlight,
+		Renderer:  mmark.HtmlRendererWithParameters(htmlFlags, "", "", renderParameters),
+	}
+
+}
+
+func getMmarkExtensions(cfg *internal.BlackFriday) int {
+	flags := 0
+	flags |= mmark.EXTENSION_TABLES
+	flags |= mmark.EXTENSION_FENCED_CODE
+	flags |= mmark.EXTENSION_AUTOLINK
+	flags |= mmark.EXTENSION_SPACE_HEADERS
+	flags |= mmark.EXTENSION_CITATION
+	flags |= mmark.EXTENSION_TITLEBLOCK_TOML
+	flags |= mmark.EXTENSION_HEADER_IDS
+	flags |= mmark.EXTENSION_AUTO_HEADER_IDS
+	flags |= mmark.EXTENSION_UNIQUE_HEADER_IDS
+	flags |= mmark.EXTENSION_FOOTNOTES
+	flags |= mmark.EXTENSION_SHORT_REF
+	flags |= mmark.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK
+	flags |= mmark.EXTENSION_INCLUDE
+
+	for _, extension := range cfg.Extensions {
+		if flag, ok := mmarkExtensionMap[extension]; ok {
+			flags |= flag
+		}
+	}
+	return flags
+}
+
+var mmarkExtensionMap = map[string]int{
+	"tables":                 mmark.EXTENSION_TABLES,
+	"fencedCode":             mmark.EXTENSION_FENCED_CODE,
+	"autolink":               mmark.EXTENSION_AUTOLINK,
+	"laxHtmlBlocks":          mmark.EXTENSION_LAX_HTML_BLOCKS,
+	"spaceHeaders":           mmark.EXTENSION_SPACE_HEADERS,
+	"hardLineBreak":          mmark.EXTENSION_HARD_LINE_BREAK,
+	"footnotes":              mmark.EXTENSION_FOOTNOTES,
+	"noEmptyLineBeforeBlock": mmark.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK,
+	"headerIds":              mmark.EXTENSION_HEADER_IDS,
+	"autoHeaderIds":          mmark.EXTENSION_AUTO_HEADER_IDS,
+}

markup/mmark/convert_test.go

@@ -0,0 +1,77 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mmark
+
+import (
+	"testing"
+
+	"github.com/spf13/viper"
+
+	"github.com/gohugoio/hugo/common/loggers"
+
+	"github.com/miekg/mmark"
+
+	"github.com/gohugoio/hugo/markup/internal"
+
+	"github.com/gohugoio/hugo/markup/converter"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestGetMmarkExtensions(t *testing.T) {
+	c := qt.New(t)
+	b, err := internal.NewBlackfriday(converter.ProviderConfig{Cfg: viper.New()})
+	c.Assert(err, qt.IsNil)
+
+	//TODO: This is doing the same just with different marks...
+	type data struct {
+		testFlag int
+	}
+
+	b.Extensions = []string{"tables"}
+	b.ExtensionsMask = []string{""}
+	allExtensions := []data{
+		{mmark.EXTENSION_TABLES},
+		{mmark.EXTENSION_FENCED_CODE},
+		{mmark.EXTENSION_AUTOLINK},
+		{mmark.EXTENSION_SPACE_HEADERS},
+		{mmark.EXTENSION_CITATION},
+		{mmark.EXTENSION_TITLEBLOCK_TOML},
+		{mmark.EXTENSION_HEADER_IDS},
+		{mmark.EXTENSION_AUTO_HEADER_IDS},
+		{mmark.EXTENSION_UNIQUE_HEADER_IDS},
+		{mmark.EXTENSION_FOOTNOTES},
+		{mmark.EXTENSION_SHORT_REF},
+		{mmark.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK},
+		{mmark.EXTENSION_INCLUDE},
+	}
+
+	actualFlags := getMmarkExtensions(b)
+	for _, e := range allExtensions {
+		if actualFlags&e.testFlag != e.testFlag {
+			t.Errorf("Flag %v was not found in the list of extensions.", e)
+		}
+	}
+}
+
+func TestConvert(t *testing.T) {
+	c := qt.New(t)
+	p, err := Provider.New(converter.ProviderConfig{Cfg: viper.New(), Logger: loggers.NewErrorLogger()})
+	c.Assert(err, qt.IsNil)
+	conv, err := p.New(converter.DocumentContext{})
+	c.Assert(err, qt.IsNil)
+	b, err := conv.Convert(converter.RenderContext{Src: []byte("testContent")})
+	c.Assert(err, qt.IsNil)
+	c.Assert(string(b.Bytes()), qt.Equals, "<p>testContent</p>\n")
+}

markup/mmark/renderer.go

@@ -0,0 +1,44 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mmark
+
+import (
+	"bytes"
+	"strings"
+
+	"github.com/miekg/mmark"
+
+	"github.com/gohugoio/hugo/config"
+	"github.com/gohugoio/hugo/markup/internal"
+)
+
+// hugoHTMLRenderer wraps a blackfriday.Renderer, typically a blackfriday.Html
+// adding some custom behaviour.
+type mmarkRenderer struct {
+	Cfg       config.Provider
+	Config    *internal.BlackFriday
+	highlight func(code, lang, optsStr string) (string, error)
+	mmark.Renderer
+}
+
+// BlockCode renders a given text as a block of code.
+func (r *mmarkRenderer) BlockCode(out *bytes.Buffer, text []byte, lang string, caption []byte, subfigure bool, callouts bool) {
+	if r.Cfg.GetBool("pygmentsCodeFences") && (lang != "" || r.Cfg.GetBool("pygmentsCodeFencesGuessSyntax")) {
+		str := strings.Trim(string(text), "\n\r")
+		highlighted, _ := r.highlight(str, lang, "")
+		out.WriteString(highlighted)
+	} else {
+		r.Renderer.BlockCode(out, text, lang, caption, subfigure, callouts)
+	}
+}

markup/org/convert.go

@@ -0,0 +1,69 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package org converts Emacs Org-Mode to HTML.
+package org
+
+import (
+	"bytes"
+
+	"github.com/gohugoio/hugo/markup/converter"
+	"github.com/niklasfasching/go-org/org"
+	"github.com/spf13/afero"
+)
+
+// Provider is the package entry point.
+var Provider converter.NewProvider = provide{}
+
+type provide struct {
+}
+
+func (p provide) New(cfg converter.ProviderConfig) (converter.Provider, error) {
+	var n converter.NewConverter = func(ctx converter.DocumentContext) (converter.Converter, error) {
+		return &orgConverter{
+			ctx: ctx,
+			cfg: cfg,
+		}, nil
+	}
+	return n, nil
+}
+
+type orgConverter struct {
+	ctx converter.DocumentContext
+	cfg converter.ProviderConfig
+}
+
+func (c *orgConverter) Convert(ctx converter.RenderContext) (converter.Result, error) {
+	logger := c.cfg.Logger
+	config := org.New()
+	config.Log = logger.WARN
+	config.ReadFile = func(filename string) ([]byte, error) {
+		return afero.ReadFile(c.cfg.ContentFs, filename)
+	}
+	writer := org.NewHTMLWriter()
+	writer.HighlightCodeBlock = func(source, lang string) string {
+		highlightedSource, err := c.cfg.Highlight(source, lang, "")
+		if err != nil {
+			logger.ERROR.Printf("Could not highlight source as lang %s. Using raw source.", lang)
+			return source
+		}
+		return highlightedSource
+	}
+
+	html, err := config.Parse(bytes.NewReader(ctx.Src), c.ctx.DocumentName).Write(writer)
+	if err != nil {
+		logger.ERROR.Printf("Could not render org: %s. Using unrendered content.", err)
+		return converter.Bytes(ctx.Src), nil
+	}
+	return converter.Bytes([]byte(html)), nil
+}

markup/org/convert_test.go

@@ -0,0 +1,35 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org
+
+import (
+	"testing"
+
+	"github.com/gohugoio/hugo/common/loggers"
+
+	"github.com/gohugoio/hugo/markup/converter"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestConvert(t *testing.T) {
+	c := qt.New(t)
+	p, err := Provider.New(converter.ProviderConfig{Logger: loggers.NewErrorLogger()})
+	c.Assert(err, qt.IsNil)
+	conv, err := p.New(converter.DocumentContext{})
+	c.Assert(err, qt.IsNil)
+	b, err := conv.Convert(converter.RenderContext{Src: []byte("testContent")})
+	c.Assert(err, qt.IsNil)
+	c.Assert(string(b.Bytes()), qt.Equals, "<p>\ntestContent\n</p>\n")
+}

markup/pandoc/convert.go

@@ -0,0 +1,76 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package pandoc converts content to HTML using Pandoc as an external helper.
+package pandoc
+
+import (
+	"os/exec"
+
+	"github.com/gohugoio/hugo/markup/internal"
+
+	"github.com/gohugoio/hugo/markup/converter"
+)
+
+// Provider is the package entry point.
+var Provider converter.NewProvider = provider{}
+
+type provider struct {
+}
+
+func (p provider) New(cfg converter.ProviderConfig) (converter.Provider, error) {
+	var n converter.NewConverter = func(ctx converter.DocumentContext) (converter.Converter, error) {
+		return &pandocConverter{
+			ctx: ctx,
+			cfg: cfg,
+		}, nil
+	}
+	return n, nil
+
+}
+
+type pandocConverter struct {
+	ctx converter.DocumentContext
+	cfg converter.ProviderConfig
+}
+
+func (c *pandocConverter) Convert(ctx converter.RenderContext) (converter.Result, error) {
+	return converter.Bytes(c.getPandocContent(ctx.Src, c.ctx)), nil
+}
+
+// getPandocContent calls pandoc as an external helper to convert pandoc markdown to HTML.
+func (c *pandocConverter) getPandocContent(src []byte, ctx converter.DocumentContext) []byte {
+	logger := c.cfg.Logger
+	path := getPandocExecPath()
+	if path == "" {
+		logger.ERROR.Println("pandoc not found in $PATH: Please install.\n",
+			"                 Leaving pandoc content unrendered.")
+		return src
+	}
+	args := []string{"--mathjax"}
+	return internal.ExternallyRenderContent(c.cfg, ctx, src, path, args)
+}
+
+func getPandocExecPath() string {
+	path, err := exec.LookPath("pandoc")
+	if err != nil {
+		return ""
+	}
+
+	return path
+}
+
+// Supports returns whether Pandoc is installed on this computer.
+func Supports() bool {
+	return getPandocExecPath() != ""
+}

markup/pandoc/convert_test.go

@@ -0,0 +1,38 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pandoc
+
+import (
+	"testing"
+
+	"github.com/gohugoio/hugo/common/loggers"
+
+	"github.com/gohugoio/hugo/markup/converter"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestConvert(t *testing.T) {
+	if !Supports() {
+		t.Skip("pandoc not installed")
+	}
+	c := qt.New(t)
+	p, err := Provider.New(converter.ProviderConfig{Logger: loggers.NewErrorLogger()})
+	c.Assert(err, qt.IsNil)
+	conv, err := p.New(converter.DocumentContext{})
+	c.Assert(err, qt.IsNil)
+	b, err := conv.Convert(converter.RenderContext{Src: []byte("testContent")})
+	c.Assert(err, qt.IsNil)
+	c.Assert(string(b.Bytes()), qt.Equals, "<p>testContent</p>\n")
+}

markup/rst/convert.go

@@ -0,0 +1,109 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package rst converts content to HTML using the RST external helper.
+package rst
+
+import (
+	"bytes"
+	"os/exec"
+	"runtime"
+
+	"github.com/gohugoio/hugo/markup/internal"
+
+	"github.com/gohugoio/hugo/markup/converter"
+)
+
+// Provider is the package entry point.
+var Provider converter.NewProvider = provider{}
+
+type provider struct {
+}
+
+func (p provider) New(cfg converter.ProviderConfig) (converter.Provider, error) {
+	var n converter.NewConverter = func(ctx converter.DocumentContext) (converter.Converter, error) {
+		return &rstConverter{
+			ctx: ctx,
+			cfg: cfg,
+		}, nil
+	}
+	return n, nil
+
+}
+
+type rstConverter struct {
+	ctx converter.DocumentContext
+	cfg converter.ProviderConfig
+}
+
+func (c *rstConverter) Convert(ctx converter.RenderContext) (converter.Result, error) {
+	return converter.Bytes(c.getRstContent(ctx.Src, c.ctx)), nil
+}
+
+// getRstContent calls the Python script rst2html as an external helper
+// to convert reStructuredText content to HTML.
+func (c *rstConverter) getRstContent(src []byte, ctx converter.DocumentContext) []byte {
+	logger := c.cfg.Logger
+	path := getRstExecPath()
+
+	if path == "" {
+		logger.ERROR.Println("rst2html / rst2html.py not found in $PATH: Please install.\n",
+			"                 Leaving reStructuredText content unrendered.")
+		return src
+	}
+	logger.INFO.Println("Rendering", ctx.DocumentName, "with", path, "...")
+	var result []byte
+	// certain *nix based OSs wrap executables in scripted launchers
+	// invoking binaries on these OSs via python interpreter causes SyntaxError
+	// invoke directly so that shebangs work as expected
+	// handle Windows manually because it doesn't do shebangs
+	if runtime.GOOS == "windows" {
+		python := internal.GetPythonExecPath()
+		args := []string{path, "--leave-comments", "--initial-header-level=2"}
+		result = internal.ExternallyRenderContent(c.cfg, ctx, src, python, args)
+	} else {
+		args := []string{"--leave-comments", "--initial-header-level=2"}
+		result = internal.ExternallyRenderContent(c.cfg, ctx, src, path, args)
+	}
+	// TODO(bep) check if rst2html has a body only option.
+	bodyStart := bytes.Index(result, []byte("<body>\n"))
+	if bodyStart < 0 {
+		bodyStart = -7 //compensate for length
+	}
+
+	bodyEnd := bytes.Index(result, []byte("\n</body>"))
+	if bodyEnd < 0 || bodyEnd >= len(result) {
+		bodyEnd = len(result) - 1
+		if bodyEnd < 0 {
+			bodyEnd = 0
+		}
+	}
+
+	return result[bodyStart+7 : bodyEnd]
+}
+
+func getRstExecPath() string {
+	path, err := exec.LookPath("rst2html")
+	if err != nil {
+		path, err = exec.LookPath("rst2html.py")
+		if err != nil {
+			return ""
+		}
+	}
+	return path
+}
+
+// Supports returns whether rst is installed on this computer.
+func Supports() bool {
+	return getRstExecPath() != ""
+}

markup/rst/convert_test.go

@@ -0,0 +1,38 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rst
+
+import (
+	"testing"
+
+	"github.com/gohugoio/hugo/common/loggers"
+
+	"github.com/gohugoio/hugo/markup/converter"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestConvert(t *testing.T) {
+	if !Supports() {
+		t.Skip("rst not installed")
+	}
+	c := qt.New(t)
+	p, err := Provider.New(converter.ProviderConfig{Logger: loggers.NewErrorLogger()})
+	c.Assert(err, qt.IsNil)
+	conv, err := p.New(converter.DocumentContext{})
+	c.Assert(err, qt.IsNil)
+	b, err := conv.Convert(converter.RenderContext{Src: []byte("testContent")})
+	c.Assert(err, qt.IsNil)
+	c.Assert(string(b.Bytes()), qt.Equals, "<div class=\"document\">\n\n\n<p>testContent</p>\n</div>")
+}

tpl/collections/collections_test.go

@@ -29,6 +29,7 @@ import (
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/hugofs"
 	"github.com/gohugoio/hugo/langs"
+	"github.com/spf13/afero"
 	"github.com/spf13/viper"
 )
 
@@ -894,7 +895,7 @@ func ToTstXIs(slice interface{}) []TstXI {
 func newDeps(cfg config.Provider) *deps.Deps {
 	l := langs.NewLanguage("en", cfg)
 	l.Set("i18nDir", "i18n")
-	cs, err := helpers.NewContentSpec(l)
+	cs, err := helpers.NewContentSpec(l, loggers.NewErrorLogger(), afero.NewMemMapFs())
 	if err != nil {
 		panic(err)
 	}

tpl/data/resources_test.go

@@ -195,7 +195,7 @@ func newDeps(cfg config.Provider) *deps.Deps {
 	}
 	cfg.Set("allModules", modules.Modules{mod})
 
-	cs, err := helpers.NewContentSpec(cfg)
+	cs, err := helpers.NewContentSpec(cfg, loggers.NewErrorLogger(), afero.NewMemMapFs())
 	if err != nil {
 		panic(err)
 	}

tpl/transform/transform.go

@@ -97,19 +97,16 @@ func (ns *Namespace) Markdownify(s interface{}) (template.HTML, error) {
 		return "", err
 	}
 
-	m := ns.deps.ContentSpec.RenderBytes(
+	b, err := ns.deps.ContentSpec.RenderMarkdown([]byte(ss))
-		&helpers.RenderingContext{
+
-			Cfg:     ns.deps.Cfg,
+	if err != nil {
-			Content: []byte(ss),
+		return "", err
-			PageFmt: "markdown",
+	}
-			Config:  ns.deps.ContentSpec.BlackFriday,
-		},
-	)
 
 	// Strip if this is a short inline type of text.
-	m = ns.deps.ContentSpec.TrimShortHTML(m)
+	b = ns.deps.ContentSpec.TrimShortHTML(b)
 
-	return helpers.BytesToHTML(m), nil
+	return helpers.BytesToHTML(b), nil
 }
 
 // Plainify returns a copy of s with all HTML tags removed.

tpl/transform/transform_test.go

@@ -17,6 +17,9 @@ import (
 	"html/template"
 	"testing"
 
+	"github.com/gohugoio/hugo/common/loggers"
+	"github.com/spf13/afero"
+
 	qt "github.com/frankban/quicktest"
 	"github.com/gohugoio/hugo/config"
 	"github.com/gohugoio/hugo/deps"
@@ -239,7 +242,7 @@ func newDeps(cfg config.Provider) *deps.Deps {
 
 	l := langs.NewLanguage("en", cfg)
 
-	cs, err := helpers.NewContentSpec(l)
+	cs, err := helpers.NewContentSpec(l, loggers.NewErrorLogger(), afero.NewMemMapFs())
 	if err != nil {
 		panic(err)
 	}