hugo/source/filesystem.go
Tatsushi Demachi 0c98d8e9ed Use LazyFileReader for reading file contents
Simple ioutil.ReadFile is used for reading file contents but it reads
all of the file contents and copies them into the memory and is run in a
single goroutine. It causes much memory consumption at copying media
files in content directory to publish directory and it is not good at
performance.

This improves the both issue by replacing ReadFile with LazyFileReader.

It postpones reading the file contents until it is really needed. As the
result, actual file read is run in parallelized goroutine. It improves
performance especially in a really big site.

In addition, if this reader is called from io.Copy, it does not copy the
file contents into the memory but just copies them into destination
file. It improves much memory consumption issue when the site has many
media files.

Fix #1181
2015-08-22 19:43:15 +02:00

162 lines
3.4 KiB
Go

// Copyright © 2014 Steve Francia <spf@spf13.com>.
//
// Licensed under the Simple Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://opensource.org/licenses/Simple-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package source
import (
"github.com/spf13/viper"
"io"
"os"
"path/filepath"
"regexp"
"strings"
"github.com/spf13/hugo/helpers"
jww "github.com/spf13/jwalterweatherman"
)
type Input interface {
Files() []*File
}
type Filesystem struct {
files []*File
Base string
AvoidPaths []string
}
func (f *Filesystem) FilesByExts(exts ...string) []*File {
var newFiles []*File
if len(exts) == 0 {
return f.Files()
}
for _, x := range f.Files() {
for _, e := range exts {
if x.Ext() == strings.TrimPrefix(e, ".") {
newFiles = append(newFiles, x)
}
}
}
return newFiles
}
func (f *Filesystem) Files() []*File {
if len(f.files) < 1 {
f.captureFiles()
}
return f.files
}
func (f *Filesystem) add(name string, reader io.Reader) (err error) {
var file *File
//if f.Base == "" {
//file = NewFileWithContents(name, reader)
//} else {
file, err = NewFileFromAbs(f.Base, name, reader)
//}
if err == nil {
f.files = append(f.files, file)
}
return err
}
func (f *Filesystem) getRelativePath(name string) (final string, err error) {
return helpers.GetRelativePath(name, f.Base)
}
func (f *Filesystem) captureFiles() {
walker := func(filePath string, fi os.FileInfo, err error) error {
if err != nil {
return nil
}
if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
link, err := filepath.EvalSymlinks(filePath)
if err != nil {
jww.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", filePath, err)
return nil
}
linkfi, err := os.Stat(link)
if err != nil {
jww.ERROR.Printf("Cannot stat '%s', error was: %s", link, err)
return nil
}
if !linkfi.Mode().IsRegular() {
jww.ERROR.Printf("Symbolic links for directories not supported, skipping '%s'", filePath)
}
return nil
}
if fi.IsDir() {
if f.avoid(filePath) || isNonProcessablePath(filePath) {
return filepath.SkipDir
}
return nil
}
if isNonProcessablePath(filePath) {
return nil
}
rd, err := NewLazyFileReader(filePath)
if err != nil {
return err
}
f.add(filePath, rd)
return nil
}
filepath.Walk(f.Base, walker)
}
func (f *Filesystem) avoid(filePath string) bool {
for _, avoid := range f.AvoidPaths {
if avoid == filePath {
return true
}
}
return false
}
func isNonProcessablePath(filePath string) bool {
base := filepath.Base(filePath)
if base[0] == '.' {
return true
}
if base[0] == '#' {
return true
}
if base[len(base)-1] == '~' {
return true
}
ignoreFiles := viper.GetStringSlice("IgnoreFiles")
if len(ignoreFiles) > 0 {
for _, ignorePattern := range ignoreFiles {
match, err := regexp.MatchString(ignorePattern, filePath)
if err != nil {
helpers.DistinctErrorLog.Printf("Invalid regexp '%s' in ignoreFiles: %s", ignorePattern, err)
return false
} else if match {
return true
}
}
}
return false
}