hugo/commands/static_syncer.go
Bjørn Erik Pedersen 3cdf19e9b7
Implement Page bundling and image handling
This commit is not the smallest in Hugo's history.

Some hightlights include:

* Page bundles (for complete articles, keeping images and content together etc.).
* Bundled images can be processed in as many versions/sizes as you need with the three methods `Resize`, `Fill` and `Fit`.
* Processed images are cached inside `resources/_gen/images` (default) in your project.
* Symbolic links (both files and dirs) are now allowed anywhere inside /content
* A new table based build summary
* The "Total in nn ms" now reports the total including the handling of the files inside /static. So if it now reports more than you're used to, it is just **more real** and probably faster than before (see below).

A site building  benchmark run compared to `v0.31.1` shows that this should be slightly faster and use less memory:

```bash
▶ ./benchSite.sh "TOML,num_langs=.*,num_root_sections=5,num_pages=(500|1000),tags_per_page=5,shortcodes,render"

benchmark                                                                                                         old ns/op     new ns/op     delta
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4      101785785     78067944      -23.30%
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4     185481057     149159919     -19.58%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4      103149918     85679409      -16.94%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4     203515478     169208775     -16.86%

benchmark                                                                                                         old allocs     new allocs     delta
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4      532464         391539         -26.47%
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4     1056549        772702         -26.87%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4      555974         406630         -26.86%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4     1086545        789922         -27.30%

benchmark                                                                                                         old bytes     new bytes     delta
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4      53243246      43598155      -18.12%
BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4     105811617     86087116      -18.64%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4      54558852      44545097      -18.35%
BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4     106903858     86978413      -18.64%
```

Fixes #3651
Closes #3158
Fixes #1014
Closes #2021
Fixes #1240
Updates #3757
2017-12-27 18:44:47 +01:00

141 lines
4.6 KiB
Go

// Copyright 2017 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package commands
import (
"os"
"path/filepath"
"github.com/fsnotify/fsnotify"
"github.com/gohugoio/hugo/helpers"
src "github.com/gohugoio/hugo/source"
"github.com/spf13/fsync"
)
type staticSyncer struct {
c *commandeer
d *src.Dirs
}
func newStaticSyncer(c *commandeer) (*staticSyncer, error) {
dirs, err := src.NewDirs(c.Fs, c.Cfg, c.DepsCfg.Logger)
if err != nil {
return nil, err
}
return &staticSyncer{c: c, d: dirs}, nil
}
func (s *staticSyncer) isStatic(path string) bool {
return s.d.IsStatic(path)
}
func (s *staticSyncer) syncsStaticEvents(staticEvents []fsnotify.Event) error {
c := s.c
syncFn := func(dirs *src.Dirs, publishDir string) (uint64, error) {
staticSourceFs, err := dirs.CreateStaticFs()
if err != nil {
return 0, err
}
if dirs.Language != nil {
// Multihost setup
publishDir = filepath.Join(publishDir, dirs.Language.Lang)
}
if staticSourceFs == nil {
c.Logger.WARN.Println("No static directories found to sync")
return 0, nil
}
syncer := fsync.NewSyncer()
syncer.NoTimes = c.Cfg.GetBool("noTimes")
syncer.NoChmod = c.Cfg.GetBool("noChmod")
syncer.SrcFs = staticSourceFs
syncer.DestFs = c.Fs.Destination
// prevent spamming the log on changes
logger := helpers.NewDistinctFeedbackLogger()
for _, ev := range staticEvents {
// Due to our approach of layering both directories and the content's rendered output
// into one we can't accurately remove a file not in one of the source directories.
// If a file is in the local static dir and also in the theme static dir and we remove
// it from one of those locations we expect it to still exist in the destination
//
// If Hugo generates a file (from the content dir) over a static file
// the content generated file should take precedence.
//
// Because we are now watching and handling individual events it is possible that a static
// event that occupies the same path as a content generated file will take precedence
// until a regeneration of the content takes places.
//
// Hugo assumes that these cases are very rare and will permit this bad behavior
// The alternative is to track every single file and which pipeline rendered it
// and then to handle conflict resolution on every event.
fromPath := ev.Name
// If we are here we already know the event took place in a static dir
relPath := dirs.MakeStaticPathRelative(fromPath)
if relPath == "" {
// Not member of this virtual host.
continue
}
// Remove || rename is harder and will require an assumption.
// Hugo takes the following approach:
// If the static file exists in any of the static source directories after this event
// Hugo will re-sync it.
// If it does not exist in all of the static directories Hugo will remove it.
//
// This assumes that Hugo has not generated content on top of a static file and then removed
// the source of that static file. In this case Hugo will incorrectly remove that file
// from the published directory.
if ev.Op&fsnotify.Rename == fsnotify.Rename || ev.Op&fsnotify.Remove == fsnotify.Remove {
if _, err := staticSourceFs.Stat(relPath); os.IsNotExist(err) {
// If file doesn't exist in any static dir, remove it
toRemove := filepath.Join(publishDir, relPath)
logger.Println("File no longer exists in static dir, removing", toRemove)
_ = c.Fs.Destination.RemoveAll(toRemove)
} else if err == nil {
// If file still exists, sync it
logger.Println("Syncing", relPath, "to", publishDir)
if err := syncer.Sync(filepath.Join(publishDir, relPath), relPath); err != nil {
c.Logger.ERROR.Println(err)
}
} else {
c.Logger.ERROR.Println(err)
}
continue
}
// For all other event operations Hugo will sync static.
logger.Println("Syncing", relPath, "to", publishDir)
if err := syncer.Sync(filepath.Join(publishDir, relPath), relPath); err != nil {
c.Logger.ERROR.Println(err)
}
}
return 0, nil
}
_, err := c.doWithPublishDirs(syncFn)
return err
}