hugo/hugolib/page_bundler.go
Bjørn Erik Pedersen 3cdf19e9b7
Implement Page bundling and image handling
This commit is not the smallest in Hugo's history.

Some hightlights include:

* Page bundles (for complete articles, keeping images and content together etc.).
* Bundled images can be processed in as many versions/sizes as you need with the three methods `Resize`, `Fill` and `Fit`.
* Processed images are cached inside `resources/_gen/images` (default) in your project.
* Symbolic links (both files and dirs) are now allowed anywhere inside /content
* A new table based build summary
* The "Total in nn ms" now reports the total including the handling of the files inside /static. So if it now reports more than you're used to, it is just **more real** and probably faster than before (see below).

A site building  benchmark run compared to `v0.31.1` shows that this should be slightly faster and use less memory:

```bash
▶ ./benchSite.sh "TOML,num_langs=.*,num_root_sections=5,num_pages=(500|1000),tags_per_page=5,shortcodes,render"

benchmark                                                                                                         old ns/op     new ns/op     delta
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4      101785785     78067944      -23.30%
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4     185481057     149159919     -19.58%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4      103149918     85679409      -16.94%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4     203515478     169208775     -16.86%

benchmark                                                                                                         old allocs     new allocs     delta
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4      532464         391539         -26.47%
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4     1056549        772702         -26.87%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4      555974         406630         -26.86%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4     1086545        789922         -27.30%

benchmark                                                                                                         old bytes     new bytes     delta
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4      53243246      43598155      -18.12%
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4     105811617     86087116      -18.64%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4      54558852      44545097      -18.35%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4     106903858     86978413      -18.64%
```

Fixes #3651
Closes #3158
Fixes #1014
Closes #2021
Fixes #1240
Updates #3757
2017-12-27 18:44:47 +01:00

188 lines
4.2 KiB
Go

// Copyright 2017-present The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package hugolib
import (
"fmt"
"math"
"runtime"
"strings"
// Use this until errgroup gets ported to context
// See https://github.com/golang/go/issues/19781
"golang.org/x/net/context"
"golang.org/x/sync/errgroup"
)
type siteContentProcessor struct {
baseDir string
site *Site
handleContent contentHandler
// The input file bundles.
fileBundlesChan chan *bundleDir
// The input file singles.
fileSinglesChan chan *fileInfo
// These assets should be just copied to destination.
fileAssetsChan chan []string
numWorkers int
// The output Pages
pagesChan chan *Page
// Used for partial rebuilds (aka. live reload)
// Will signal replacement of pages in the site collection.
partialBuild bool
}
func newSiteContentProcessor(baseDir string, partialBuild bool, s *Site) *siteContentProcessor {
numWorkers := 12
if n := runtime.NumCPU() * 3; n > numWorkers {
numWorkers = n
}
numWorkers = int(math.Ceil(float64(numWorkers) / float64(len(s.owner.Sites))))
return &siteContentProcessor{
partialBuild: partialBuild,
baseDir: baseDir,
site: s,
handleContent: newHandlerChain(s),
fileBundlesChan: make(chan *bundleDir, numWorkers),
fileSinglesChan: make(chan *fileInfo, numWorkers),
fileAssetsChan: make(chan []string, numWorkers),
numWorkers: numWorkers,
pagesChan: make(chan *Page, numWorkers),
}
}
func (s *siteContentProcessor) closeInput() {
close(s.fileSinglesChan)
close(s.fileBundlesChan)
close(s.fileAssetsChan)
}
func (s *siteContentProcessor) process(ctx context.Context) error {
g1, ctx := errgroup.WithContext(ctx)
g2, _ := errgroup.WithContext(ctx)
// There can be only one of these per site.
g1.Go(func() error {
for p := range s.pagesChan {
if p.s != s.site {
panic(fmt.Sprintf("invalid page site: %v vs %v", p.s, s))
}
if s.partialBuild {
s.site.replacePage(p)
} else {
s.site.addPage(p)
}
}
return nil
})
for i := 0; i < s.numWorkers; i++ {
g2.Go(func() error {
for {
select {
case f, ok := <-s.fileSinglesChan:
if !ok {
return nil
}
err := s.readAndConvertContentFile(f)
if err != nil {
return err
}
case <-ctx.Done():
return ctx.Err()
}
}
})
g2.Go(func() error {
for {
select {
case filenames, ok := <-s.fileAssetsChan:
if !ok {
return nil
}
for _, filename := range filenames {
name := strings.TrimPrefix(filename, s.baseDir)
f, err := s.site.Fs.Source.Open(filename)
if err != nil {
return err
}
err = s.site.publish(&s.site.PathSpec.ProcessingStats.Files, name, f)
f.Close()
if err != nil {
return err
}
}
case <-ctx.Done():
return ctx.Err()
}
}
})
g2.Go(func() error {
for {
select {
case bundle, ok := <-s.fileBundlesChan:
if !ok {
return nil
}
err := s.readAndConvertContentBundle(bundle)
if err != nil {
return err
}
case <-ctx.Done():
return ctx.Err()
}
}
})
}
if err := g2.Wait(); err != nil {
return err
}
close(s.pagesChan)
if err := g1.Wait(); err != nil {
return err
}
s.site.rawAllPages.Sort()
return nil
}
func (s *siteContentProcessor) readAndConvertContentFile(file *fileInfo) error {
ctx := &handlerContext{source: file, baseDir: s.baseDir, pages: s.pagesChan}
return s.handleContent(ctx).err
}
func (s *siteContentProcessor) readAndConvertContentBundle(bundle *bundleDir) error {
ctx := &handlerContext{bundle: bundle, baseDir: s.baseDir, pages: s.pagesChan}
return s.handleContent(ctx).err
}