hugo/commands/convert.go
Bjørn Erik Pedersen 3cdf19e9b7
Implement Page bundling and image handling
This commit is not the smallest in Hugo's history.

Some hightlights include:

* Page bundles (for complete articles, keeping images and content together etc.).
* Bundled images can be processed in as many versions/sizes as you need with the three methods `Resize`, `Fill` and `Fit`.
* Processed images are cached inside `resources/_gen/images` (default) in your project.
* Symbolic links (both files and dirs) are now allowed anywhere inside /content
* A new table based build summary
* The "Total in nn ms" now reports the total including the handling of the files inside /static. So if it now reports more than you're used to, it is just **more real** and probably faster than before (see below).

A site building  benchmark run compared to `v0.31.1` shows that this should be slightly faster and use less memory:

```bash
▶ ./benchSite.sh "TOML,num_langs=.*,num_root_sections=5,num_pages=(500|1000),tags_per_page=5,shortcodes,render"

benchmark                                                                                                         old ns/op     new ns/op     delta
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4      101785785     78067944      -23.30%
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4     185481057     149159919     -19.58%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4      103149918     85679409      -16.94%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4     203515478     169208775     -16.86%

benchmark                                                                                                         old allocs     new allocs     delta
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4      532464         391539         -26.47%
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4     1056549        772702         -26.87%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4      555974         406630         -26.86%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4     1086545        789922         -27.30%

benchmark                                                                                                         old bytes     new bytes     delta
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4      53243246      43598155      -18.12%
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4     105811617     86087116      -18.64%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4      54558852      44545097      -18.35%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4     106903858     86978413      -18.64%
```

Fixes #3651
Closes #3158
Fixes #1014
Closes #2021
Fixes #1240
Updates #3757
2017-12-27 18:44:47 +01:00

183 lines
5.1 KiB
Go

// Copyright 2015 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package commands
import (
"fmt"
"time"
src "github.com/gohugoio/hugo/source"
"github.com/gohugoio/hugo/hugolib"
"path/filepath"
"github.com/gohugoio/hugo/parser"
"github.com/spf13/cast"
"github.com/spf13/cobra"
)
var outputDir string
var unsafe bool
var convertCmd = &cobra.Command{
Use: "convert",
Short: "Convert your content to different formats",
Long: `Convert your content (e.g. front matter) to different formats.
See convert's subcommands toJSON, toTOML and toYAML for more information.`,
RunE: nil,
}
var toJSONCmd = &cobra.Command{
Use: "toJSON",
Short: "Convert front matter to JSON",
Long: `toJSON converts all front matter in the content directory
to use JSON for the front matter.`,
RunE: func(cmd *cobra.Command, args []string) error {
return convertContents(rune([]byte(parser.JSONLead)[0]))
},
}
var toTOMLCmd = &cobra.Command{
Use: "toTOML",
Short: "Convert front matter to TOML",
Long: `toTOML converts all front matter in the content directory
to use TOML for the front matter.`,
RunE: func(cmd *cobra.Command, args []string) error {
return convertContents(rune([]byte(parser.TOMLLead)[0]))
},
}
var toYAMLCmd = &cobra.Command{
Use: "toYAML",
Short: "Convert front matter to YAML",
Long: `toYAML converts all front matter in the content directory
to use YAML for the front matter.`,
RunE: func(cmd *cobra.Command, args []string) error {
return convertContents(rune([]byte(parser.YAMLLead)[0]))
},
}
func init() {
convertCmd.AddCommand(toJSONCmd)
convertCmd.AddCommand(toTOMLCmd)
convertCmd.AddCommand(toYAMLCmd)
convertCmd.PersistentFlags().StringVarP(&outputDir, "output", "o", "", "filesystem path to write files to")
convertCmd.PersistentFlags().StringVarP(&source, "source", "s", "", "filesystem path to read files relative from")
convertCmd.PersistentFlags().BoolVar(&unsafe, "unsafe", false, "enable less safe operations, please backup first")
convertCmd.PersistentFlags().SetAnnotation("source", cobra.BashCompSubdirsInDir, []string{})
}
func convertContents(mark rune) error {
if outputDir == "" && !unsafe {
return newUserError("Unsafe operation not allowed, use --unsafe or set a different output path")
}
c, err := InitializeConfig(false, nil)
if err != nil {
return err
}
h, err := hugolib.NewHugoSites(*c.DepsCfg)
if err != nil {
return err
}
if err := h.Build(hugolib.BuildCfg{SkipRender: true}); err != nil {
return err
}
site := h.Sites[0]
site.Log.FEEDBACK.Println("processing", len(site.AllPages), "content files")
for _, p := range site.AllPages {
if err := convertAndSavePage(p, site, mark); err != nil {
return err
}
}
return nil
}
func convertAndSavePage(p *hugolib.Page, site *hugolib.Site, mark rune) error {
// The resources are not in .Site.AllPages.
for _, r := range p.Resources.ByType("page") {
if err := convertAndSavePage(r.(*hugolib.Page), site, mark); err != nil {
return err
}
}
if p.Filename() == "" {
// No content file.
return nil
}
site.Log.INFO.Println("Attempting to convert", p.LogicalName())
newPage, err := site.NewPage(p.LogicalName())
if err != nil {
return err
}
f, _ := p.File.(src.ReadableFile)
file, err := f.Open()
if err != nil {
site.Log.ERROR.Println("Error reading file:", p.Path())
file.Close()
return nil
}
psr, err := parser.ReadFrom(file)
if err != nil {
site.Log.ERROR.Println("Error processing file:", p.Path())
file.Close()
return err
}
file.Close()
metadata, err := psr.Metadata()
if err != nil {
site.Log.ERROR.Println("Error processing file:", p.Path())
return err
}
// better handling of dates in formats that don't have support for them
if mark == parser.FormatToLeadRune("json") || mark == parser.FormatToLeadRune("yaml") || mark == parser.FormatToLeadRune("toml") {
newMetadata := cast.ToStringMap(metadata)
for k, v := range newMetadata {
switch vv := v.(type) {
case time.Time:
newMetadata[k] = vv.Format(time.RFC3339)
}
}
metadata = newMetadata
}
newPage.SetSourceContent(psr.Content())
if err = newPage.SetSourceMetaData(metadata, mark); err != nil {
site.Log.ERROR.Printf("Failed to set source metadata for file %q: %s. For more info see For more info see https://github.com/gohugoio/hugo/issues/2458", newPage.FullFilePath(), err)
return nil
}
newFilename := p.Filename()
if outputDir != "" {
newFilename = filepath.Join(outputDir, p.Dir(), newPage.LogicalName())
}
if err = newPage.SaveSourceAs(newFilename); err != nil {
return fmt.Errorf("Failed to save file %q: %s", newFilename, err)
}
return nil
}