mirror of
https://github.com/gohugoio/hugo.git
synced 2024-11-21 20:46:30 -05:00
Add embeded template for robots.txt
This commit is contained in:
parent
7c5a1fd16b
commit
9a6dc6c791
9 changed files with 143 additions and 2 deletions
|
@ -112,7 +112,7 @@ Complete documentation is available at http://gohugo.io/.`,
|
||||||
var hugoCmdV *cobra.Command
|
var hugoCmdV *cobra.Command
|
||||||
|
|
||||||
// Flags that are to be added to commands.
|
// Flags that are to be added to commands.
|
||||||
var BuildWatch, IgnoreCache, Draft, Future, UglyURLs, CanonifyURLs, Verbose, Logging, VerboseLog, DisableRSS, DisableSitemap, PluralizeListTitles, PreserveTaxonomyNames, NoTimes, ForceSync bool
|
var BuildWatch, IgnoreCache, Draft, Future, UglyURLs, CanonifyURLs, Verbose, Logging, VerboseLog, DisableRSS, DisableSitemap, DisableRobotsTXT, PluralizeListTitles, PreserveTaxonomyNames, NoTimes, ForceSync bool
|
||||||
var Source, CacheDir, Destination, Theme, BaseURL, CfgFile, LogFile, Editor string
|
var Source, CacheDir, Destination, Theme, BaseURL, CfgFile, LogFile, Editor string
|
||||||
|
|
||||||
// Execute adds all child commands to the root command HugoCmd and sets flags appropriately.
|
// Execute adds all child commands to the root command HugoCmd and sets flags appropriately.
|
||||||
|
@ -159,6 +159,7 @@ func initCoreCommonFlags(cmd *cobra.Command) {
|
||||||
cmd.Flags().BoolVarP(&Future, "buildFuture", "F", false, "include content with publishdate in the future")
|
cmd.Flags().BoolVarP(&Future, "buildFuture", "F", false, "include content with publishdate in the future")
|
||||||
cmd.Flags().BoolVar(&DisableRSS, "disableRSS", false, "Do not build RSS files")
|
cmd.Flags().BoolVar(&DisableRSS, "disableRSS", false, "Do not build RSS files")
|
||||||
cmd.Flags().BoolVar(&DisableSitemap, "disableSitemap", false, "Do not build Sitemap file")
|
cmd.Flags().BoolVar(&DisableSitemap, "disableSitemap", false, "Do not build Sitemap file")
|
||||||
|
cmd.Flags().BoolVar(&DisableRobotsTXT, "disableRobotsTXT", false, "Do not build Robots TXT file")
|
||||||
cmd.Flags().StringVarP(&Source, "source", "s", "", "filesystem path to read files relative from")
|
cmd.Flags().StringVarP(&Source, "source", "s", "", "filesystem path to read files relative from")
|
||||||
cmd.Flags().StringVarP(&CacheDir, "cacheDir", "", "", "filesystem path to cache directory. Defaults: $TMPDIR/hugo_cache/")
|
cmd.Flags().StringVarP(&CacheDir, "cacheDir", "", "", "filesystem path to cache directory. Defaults: $TMPDIR/hugo_cache/")
|
||||||
cmd.Flags().BoolVarP(&IgnoreCache, "ignoreCache", "", false, "Ignores the cache directory for reading but still writes to it")
|
cmd.Flags().BoolVarP(&IgnoreCache, "ignoreCache", "", false, "Ignores the cache directory for reading but still writes to it")
|
||||||
|
@ -204,6 +205,7 @@ func LoadDefaultSettings() {
|
||||||
viper.SetDefault("MetaDataFormat", "toml")
|
viper.SetDefault("MetaDataFormat", "toml")
|
||||||
viper.SetDefault("DisableRSS", false)
|
viper.SetDefault("DisableRSS", false)
|
||||||
viper.SetDefault("DisableSitemap", false)
|
viper.SetDefault("DisableSitemap", false)
|
||||||
|
viper.SetDefault("DisableRobotsTXT", false)
|
||||||
viper.SetDefault("ContentDir", "content")
|
viper.SetDefault("ContentDir", "content")
|
||||||
viper.SetDefault("LayoutDir", "layouts")
|
viper.SetDefault("LayoutDir", "layouts")
|
||||||
viper.SetDefault("StaticDir", "static")
|
viper.SetDefault("StaticDir", "static")
|
||||||
|
@ -294,6 +296,9 @@ func InitializeConfig(subCmdVs ...*cobra.Command) error {
|
||||||
if cmdV.Flags().Lookup("disableSitemap").Changed {
|
if cmdV.Flags().Lookup("disableSitemap").Changed {
|
||||||
viper.Set("DisableSitemap", DisableSitemap)
|
viper.Set("DisableSitemap", DisableSitemap)
|
||||||
}
|
}
|
||||||
|
if cmdV.Flags().Lookup("disableRobotsTXT").Changed {
|
||||||
|
viper.Set("DisableRobotsTXT", DisableRobotsTXT)
|
||||||
|
}
|
||||||
if cmdV.Flags().Lookup("pluralizeListTitles").Changed {
|
if cmdV.Flags().Lookup("pluralizeListTitles").Changed {
|
||||||
viper.Set("PluralizeListTitles", PluralizeListTitles)
|
viper.Set("PluralizeListTitles", PluralizeListTitles)
|
||||||
}
|
}
|
||||||
|
|
|
@ -34,6 +34,7 @@ hugo
|
||||||
-d, --destination="": filesystem path to write files to
|
-d, --destination="": filesystem path to write files to
|
||||||
--disableRSS[=false]: Do not build RSS files
|
--disableRSS[=false]: Do not build RSS files
|
||||||
--disableSitemap[=false]: Do not build Sitemap file
|
--disableSitemap[=false]: Do not build Sitemap file
|
||||||
|
--disableRobotsTXT[=false]: Do not build robots TXT file
|
||||||
--editor="": edit new content with this editor, if provided
|
--editor="": edit new content with this editor, if provided
|
||||||
--ignoreCache[=false]: Ignores the cache directory for reading but still writes to it
|
--ignoreCache[=false]: Ignores the cache directory for reading but still writes to it
|
||||||
--log[=false]: Enable Logging
|
--log[=false]: Enable Logging
|
||||||
|
|
34
docs/content/extras/robots-txt.md
Normal file
34
docs/content/extras/robots-txt.md
Normal file
|
@ -0,0 +1,34 @@
|
||||||
|
---
|
||||||
|
date: 2013-07-09
|
||||||
|
menu:
|
||||||
|
main:
|
||||||
|
parent: extras
|
||||||
|
next: /community/mailing-list
|
||||||
|
prev: /extras/urls
|
||||||
|
title: Table of Contents
|
||||||
|
weight: 120
|
||||||
|
---
|
||||||
|
|
||||||
|
Hugo can generated customized [robots.txt](http://www.robotstxt.org/) in the
|
||||||
|
[same way than any other template]({{< ref "templates/go-templates.md" >}}).
|
||||||
|
|
||||||
|
By default it generates a robots.txt which allows everything, it looks exactly
|
||||||
|
|
||||||
|
User-agent: *
|
||||||
|
|
||||||
|
To disable it just set `disableRobotsTXT` option to false in the [command line]({{< ref "commands/hugo.md" >}}) or [configuration file]({{< ref "overview/configuration.md" >}}).
|
||||||
|
|
||||||
|
Hugo will use the template `robots.txt` following the list starting with the one with more priority
|
||||||
|
|
||||||
|
* /layouts/robots.txt
|
||||||
|
* /themes/`THEME`/layout/robots.txt
|
||||||
|
|
||||||
|
An example of a robots.txt layout is:
|
||||||
|
|
||||||
|
User-agent: *
|
||||||
|
|
||||||
|
{{range .Data.Pages}}
|
||||||
|
Disallow: {{.RelPermalink}}{{end}}
|
||||||
|
|
||||||
|
|
||||||
|
This template disallows and all the pages of the site creating one `Disallow` entry for each one.
|
|
@ -5,7 +5,7 @@ date: 2014-01-03
|
||||||
menu:
|
menu:
|
||||||
main:
|
main:
|
||||||
parent: extras
|
parent: extras
|
||||||
next: /community/mailing-list
|
next: /extras/robots-txt
|
||||||
notoc: true
|
notoc: true
|
||||||
prev: /extras/toc
|
prev: /extras/toc
|
||||||
title: URLs
|
title: URLs
|
||||||
|
|
|
@ -96,6 +96,8 @@ Following is a list of Hugo-defined variables that you can configure and their c
|
||||||
disableRSS: false
|
disableRSS: false
|
||||||
# Do not build Sitemap file
|
# Do not build Sitemap file
|
||||||
disableSitemap: false
|
disableSitemap: false
|
||||||
|
# Do not build robots.txt file
|
||||||
|
disableRobotsTXT: false
|
||||||
# edit new content with this editor, if provided
|
# edit new content with this editor, if provided
|
||||||
editor: ""
|
editor: ""
|
||||||
footnoteAnchorPrefix: ""
|
footnoteAnchorPrefix: ""
|
||||||
|
|
67
hugolib/robotstxt_test.go
Normal file
67
hugolib/robotstxt_test.go
Normal file
|
@ -0,0 +1,67 @@
|
||||||
|
package hugolib
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/spf13/afero"
|
||||||
|
"github.com/spf13/hugo/helpers"
|
||||||
|
"github.com/spf13/hugo/hugofs"
|
||||||
|
"github.com/spf13/hugo/source"
|
||||||
|
"github.com/spf13/viper"
|
||||||
|
)
|
||||||
|
|
||||||
|
const ROBOTSTXT_TEMPLATE = `User-agent: Googlebot
|
||||||
|
{{ range .Data.Pages }}
|
||||||
|
Disallow: {{.RelPermalink}}
|
||||||
|
{{ end }}
|
||||||
|
`
|
||||||
|
|
||||||
|
func TestRobotsTXTOutput(t *testing.T) {
|
||||||
|
viper.Reset()
|
||||||
|
defer viper.Reset()
|
||||||
|
|
||||||
|
hugofs.DestinationFS = new(afero.MemMapFs)
|
||||||
|
|
||||||
|
viper.Set("baseurl", "http://auth/bub/")
|
||||||
|
|
||||||
|
s := &Site{
|
||||||
|
Source: &source.InMemorySource{ByteSource: WEIGHTED_SOURCES},
|
||||||
|
}
|
||||||
|
|
||||||
|
s.initializeSiteInfo()
|
||||||
|
|
||||||
|
s.prepTemplates()
|
||||||
|
s.addTemplate("robots.txt", ROBOTSTXT_TEMPLATE)
|
||||||
|
|
||||||
|
if err := s.CreatePages(); err != nil {
|
||||||
|
t.Fatalf("Unable to create pages: %s", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := s.BuildSiteMeta(); err != nil {
|
||||||
|
t.Fatalf("Unable to build site metadata: %s", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := s.RenderHomePage(); err != nil {
|
||||||
|
t.Fatalf("Unable to RenderHomePage: %s", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := s.RenderSitemap(); err != nil {
|
||||||
|
t.Fatalf("Unable to RenderSitemap: %s", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := s.RenderRobotsTXT(); err != nil {
|
||||||
|
t.Fatalf("Unable to RenderRobotsTXT :%s", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
robotsFile, err := hugofs.DestinationFS.Open("robots.txt")
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Unable to locate: robots.txt")
|
||||||
|
}
|
||||||
|
|
||||||
|
robots := helpers.ReaderToBytes(robotsFile)
|
||||||
|
if !bytes.HasPrefix(robots, []byte("User-agent: Googlebot")) {
|
||||||
|
t.Errorf("Robots file should start with 'User-agentL Googlebot'. %s", robots)
|
||||||
|
}
|
||||||
|
}
|
|
@ -583,6 +583,12 @@ func (s *Site) Render() (err error) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
s.timerStep("render and write Sitemap")
|
s.timerStep("render and write Sitemap")
|
||||||
|
|
||||||
|
if err = s.RenderRobotsTXT(); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.timerStep("render and write robots.txt")
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1561,6 +1567,26 @@ func (s *Site) RenderSitemap() error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *Site) RenderRobotsTXT() error {
|
||||||
|
if viper.GetBool("DisableRobotsTXT") {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
n := s.NewNode()
|
||||||
|
n.Data["Pages"] = s.Pages
|
||||||
|
|
||||||
|
rLayouts := []string{"robots.txt", "_default/robots.txt", "_internal/_default/robots.txt"}
|
||||||
|
outBuffer := bp.GetBuffer()
|
||||||
|
defer bp.PutBuffer(outBuffer)
|
||||||
|
err := s.render("robots", n, outBuffer, s.appendThemeTemplates(rLayouts)...)
|
||||||
|
|
||||||
|
if err == nil {
|
||||||
|
err = s.WriteDestFile("robots.txt", outBuffer)
|
||||||
|
}
|
||||||
|
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
func (s *Site) Stats() {
|
func (s *Site) Stats() {
|
||||||
jww.FEEDBACK.Println(s.draftStats())
|
jww.FEEDBACK.Println(s.draftStats())
|
||||||
jww.FEEDBACK.Println(s.futureStats())
|
jww.FEEDBACK.Println(s.futureStats())
|
||||||
|
|
|
@ -68,6 +68,10 @@ func TestSitemapOutput(t *testing.T) {
|
||||||
t.Fatalf("Unable to RenderSitemap: %s", err)
|
t.Fatalf("Unable to RenderSitemap: %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if err := s.RenderRobotsTXT(); err != nil {
|
||||||
|
t.Fatalf("Unable to RenderRobotsTXT :%s", err)
|
||||||
|
}
|
||||||
|
|
||||||
sitemapFile, err := hugofs.DestinationFS.Open("sitemap.xml")
|
sitemapFile, err := hugofs.DestinationFS.Open("sitemap.xml")
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
@ -242,4 +242,6 @@ ga('send', 'pageview');
|
||||||
</script>
|
</script>
|
||||||
<script async src='//www.google-analytics.com/analytics.js'></script>
|
<script async src='//www.google-analytics.com/analytics.js'></script>
|
||||||
{{ end }}`)
|
{{ end }}`)
|
||||||
|
|
||||||
|
t.AddInternalTemplate("_default", "robots.txt", "User-agent: *")
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue