mirror of
https://github.com/gohugoio/hugo.git
synced 2024-11-07 20:30:36 -05:00
Add LazyFileReader type to source library
LazyFileReader is an io.Reader implementation to postpone reading the file contents until it is really needed. It is introduced for improving performance and memory consumption at reading media files in content directory.
This commit is contained in:
Tatsushi Demachi
Bjørn Erik Pedersen
parent
3982854eeb
commit
97eb55da89
2 changed files with 378 additions and 0 deletions
160
source/lazy_file_reader.go
Normal file
160
source/lazy_file_reader.go
Normal file
|
|
@ -0,0 +1,160 @@
|
|||
// Copyright © 2014 Steve Francia <spf@spf13.com>.
|
||||
//
|
||||
// Licensed under the Simple Public License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://opensource.org/licenses/Simple-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package source
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
)
|
||||
|
||||
// LazyFileReader is an io.Reader implementation to postpone reading the file
|
||||
// contents until it is really needed. It keeps filename and file contents once
|
||||
// it is read.
|
||||
type LazyFileReader struct {
|
||||
filename string
|
||||
contents *bytes.Reader
|
||||
pos int64
|
||||
}
|
||||
|
||||
// NewLazyFileReader creates and initializes a new LazyFileReader of filename.
|
||||
// It checks whether the file can be opened. If it fails, it returns nil and an
|
||||
// error.
|
||||
func NewLazyFileReader(filename string) (*LazyFileReader, error) {
|
||||
f, err := os.Open(filename)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
return &LazyFileReader{filename: filename, contents: nil, pos: 0}, nil
|
||||
}
|
||||
|
||||
// Filename returns a file name which LazyFileReader keeps
|
||||
func (l *LazyFileReader) Filename() string {
|
||||
return l.filename
|
||||
}
|
||||
|
||||
// Read reads up to len(p) bytes from the LazyFileReader's file and copies them
|
||||
// into p. It returns the number of bytes read and any error encountered. If
|
||||
// the file is once read, it returns its contents from cache, doesn't re-read
|
||||
// the file.
|
||||
func (l *LazyFileReader) Read(p []byte) (n int, err error) {
|
||||
if l.contents == nil {
|
||||
b, err := ioutil.ReadFile(l.filename)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to read content from %s: %s", l.filename, err.Error())
|
||||
}
|
||||
l.contents = bytes.NewReader(b)
|
||||
}
|
||||
l.contents.Seek(l.pos, 0)
|
||||
if err != nil {
|
||||
return 0, errors.New("failed to set read position: " + err.Error())
|
||||
}
|
||||
n, err = l.contents.Read(p)
|
||||
l.pos += int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Seek implements the io.Seeker interface. Once reader contents is consumed by
|
||||
// Read, WriteTo etc, to read it again, it must be rewinded by this function
|
||||
func (l *LazyFileReader) Seek(offset int64, whence int) (pos int64, err error) {
|
||||
if l.contents == nil {
|
||||
switch whence {
|
||||
case 0:
|
||||
pos = offset
|
||||
case 1:
|
||||
pos = l.pos + offset
|
||||
case 2:
|
||||
fi, err := os.Stat(l.filename)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to get %q info: %s", l.filename, err.Error())
|
||||
}
|
||||
pos = fi.Size() + offset
|
||||
default:
|
||||
return 0, errors.New("invalid whence")
|
||||
}
|
||||
if pos < 0 {
|
||||
return 0, errors.New("negative position")
|
||||
}
|
||||
} else {
|
||||
pos, err = l.contents.Seek(offset, whence)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
l.pos = pos
|
||||
return pos, nil
|
||||
}
|
||||
|
||||
// WriteTo writes data to w until all the LazyFileReader's file contents is
|
||||
// drained or an error occurs. If the file is once read, it just writes its
|
||||
// read cache to w, doesn't re-read the file but this method itself doesn't try
|
||||
// to keep the contents in cache.
|
||||
func (l *LazyFileReader) WriteTo(w io.Writer) (n int64, err error) {
|
||||
if l.contents != nil {
|
||||
l.contents.Seek(l.pos, 0)
|
||||
if err != nil {
|
||||
return 0, errors.New("failed to set read position: " + err.Error())
|
||||
}
|
||||
n, err = l.contents.WriteTo(w)
|
||||
l.pos += n
|
||||
return n, err
|
||||
}
|
||||
f, err := os.Open(l.filename)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to open %s to read content: %s", l.filename, err.Error())
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
fi, err := f.Stat()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to get %q info: %s", l.filename, err.Error())
|
||||
}
|
||||
|
||||
if l.pos >= fi.Size() {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// following code is taken from io.Copy in 'io/io.go'
|
||||
buf := make([]byte, 32*1024)
|
||||
for {
|
||||
nr, er := f.Read(buf)
|
||||
if nr > 0 {
|
||||
nw, ew := w.Write(buf[0:nr])
|
||||
if nw > 0 {
|
||||
l.pos += int64(nw)
|
||||
n += int64(nw)
|
||||
}
|
||||
if ew != nil {
|
||||
err = ew
|
||||
break
|
||||
}
|
||||
if nr != nw {
|
||||
err = io.ErrShortWrite
|
||||
break
|
||||
}
|
||||
}
|
||||
if er == io.EOF {
|
||||
break
|
||||
}
|
||||
if er != nil {
|
||||
err = er
|
||||
break
|
||||
}
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
218
source/lazy_file_reader_test.go
Normal file
218
source/lazy_file_reader_test.go
Normal file
|
|
@ -0,0 +1,218 @@
|
|||
package source
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNewLazyFileReader(t *testing.T) {
|
||||
filename := "itdoesnotexistfile"
|
||||
_, err := NewLazyFileReader(filename)
|
||||
if err == nil {
|
||||
t.Errorf("NewLazyFileReader %s: error expected but no error is returned", filename)
|
||||
}
|
||||
|
||||
filename = "lazy_file_reader_test.go"
|
||||
_, err = NewLazyFileReader(filename)
|
||||
if err != nil {
|
||||
t.Errorf("NewLazyFileReader %s: %v", filename, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilename(t *testing.T) {
|
||||
filename := "lazy_file_reader_test.go"
|
||||
rd, err := NewLazyFileReader(filename)
|
||||
if err != nil {
|
||||
t.Fatalf("NewLazyFileReader %s: %v", filename, err)
|
||||
}
|
||||
if rd.Filename() != filename {
|
||||
t.Errorf("Filename: expected filename %q, got %q", filename, rd.Filename())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRead(t *testing.T) {
|
||||
filename := "lazy_file_reader_test.go"
|
||||
fi, err := os.Stat(filename)
|
||||
if err != nil {
|
||||
t.Fatalf("os.Stat: %v", err)
|
||||
}
|
||||
|
||||
b, err := ioutil.ReadFile(filename)
|
||||
if err != nil {
|
||||
t.Fatalf("ioutil.ReadFile: %v", err)
|
||||
}
|
||||
|
||||
rd, err := NewLazyFileReader(filename)
|
||||
if err != nil {
|
||||
t.Fatalf("NewLazyFileReader %s: %v", filename, err)
|
||||
}
|
||||
|
||||
tst := func(testcase string) {
|
||||
p := make([]byte, fi.Size())
|
||||
n, err := rd.Read(p)
|
||||
if err != nil {
|
||||
t.Fatalf("Read %s case: %v", testcase, err)
|
||||
}
|
||||
if int64(n) != fi.Size() {
|
||||
t.Errorf("Read %s case: read bytes length expected %d, got %d", testcase, fi.Size(), n)
|
||||
}
|
||||
if !bytes.Equal(b, p) {
|
||||
t.Errorf("Read %s case: read bytes are different from expected", testcase)
|
||||
}
|
||||
}
|
||||
tst("No cache")
|
||||
_, err = rd.Seek(0, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("Seek: %v", err)
|
||||
}
|
||||
tst("Cache")
|
||||
}
|
||||
|
||||
func TestSeek(t *testing.T) {
|
||||
type testcase struct {
|
||||
seek int
|
||||
offset int64
|
||||
length int
|
||||
moveto int64
|
||||
expected []byte
|
||||
}
|
||||
|
||||
filename := "lazy_file_reader_test.go"
|
||||
b, err := ioutil.ReadFile(filename)
|
||||
if err != nil {
|
||||
t.Fatalf("ioutil.ReadFile: %v", err)
|
||||
}
|
||||
|
||||
// no cache case
|
||||
for i, this := range []testcase{
|
||||
{seek: os.SEEK_SET, offset: 0, length: 10, moveto: 0, expected: b[:10]},
|
||||
{seek: os.SEEK_SET, offset: 5, length: 10, moveto: 5, expected: b[5:15]},
|
||||
{seek: os.SEEK_CUR, offset: 5, length: 10, moveto: 5, expected: b[5:15]}, // current pos = 0
|
||||
{seek: os.SEEK_END, offset: -1, length: 1, moveto: int64(len(b) - 1), expected: b[len(b)-1:]},
|
||||
{seek: 3, expected: nil},
|
||||
{seek: os.SEEK_SET, offset: -1, expected: nil},
|
||||
} {
|
||||
rd, err := NewLazyFileReader(filename)
|
||||
if err != nil {
|
||||
t.Errorf("[%d] NewLazyFileReader %s: %v", i, filename, err)
|
||||
continue
|
||||
}
|
||||
|
||||
pos, err := rd.Seek(this.offset, this.seek)
|
||||
if this.expected == nil {
|
||||
if err == nil {
|
||||
t.Errorf("[%d] Seek didn't return an expected error", i)
|
||||
}
|
||||
} else {
|
||||
if err != nil {
|
||||
t.Errorf("[%d] Seek failed unexpectedly: %v", i, err)
|
||||
continue
|
||||
}
|
||||
if pos != this.moveto {
|
||||
t.Errorf("[%d] Seek failed to move the pointer: got %d, expected: %d", i, pos, this.moveto)
|
||||
}
|
||||
|
||||
buf := make([]byte, this.length)
|
||||
n, err := rd.Read(buf)
|
||||
if err != nil {
|
||||
t.Errorf("[%d] Read failed unexpectedly: %v", i, err)
|
||||
}
|
||||
if !bytes.Equal(this.expected, buf[:n]) {
|
||||
t.Errorf("[%d] Seek and Read got %q but expected %q", i, buf[:n], this.expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// cache case
|
||||
rd, err := NewLazyFileReader(filename)
|
||||
if err != nil {
|
||||
t.Fatalf("NewLazyFileReader %s: %v", filename, err)
|
||||
}
|
||||
dummy := make([]byte, len(b))
|
||||
_, err = rd.Read(dummy)
|
||||
if err != nil {
|
||||
t.Fatalf("Read failed unexpectedly: %v", err)
|
||||
}
|
||||
|
||||
for i, this := range []testcase{
|
||||
{seek: os.SEEK_SET, offset: 0, length: 10, moveto: 0, expected: b[:10]},
|
||||
{seek: os.SEEK_SET, offset: 5, length: 10, moveto: 5, expected: b[5:15]},
|
||||
{seek: os.SEEK_CUR, offset: 1, length: 10, moveto: 16, expected: b[16:26]}, // current pos = 15
|
||||
{seek: os.SEEK_END, offset: -1, length: 1, moveto: int64(len(b) - 1), expected: b[len(b)-1:]},
|
||||
{seek: 3, expected: nil},
|
||||
{seek: os.SEEK_SET, offset: -1, expected: nil},
|
||||
} {
|
||||
pos, err := rd.Seek(this.offset, this.seek)
|
||||
if this.expected == nil {
|
||||
if err == nil {
|
||||
t.Errorf("[%d] Seek didn't return an expected error", i)
|
||||
}
|
||||
} else {
|
||||
if err != nil {
|
||||
t.Errorf("[%d] Seek failed unexpectedly: %v", i, err)
|
||||
continue
|
||||
}
|
||||
if pos != this.moveto {
|
||||
t.Errorf("[%d] Seek failed to move the pointer: got %d, expected: %d", i, pos, this.moveto)
|
||||
}
|
||||
|
||||
buf := make([]byte, this.length)
|
||||
n, err := rd.Read(buf)
|
||||
if err != nil {
|
||||
t.Errorf("[%d] Read failed unexpectedly: %v", i, err)
|
||||
}
|
||||
if !bytes.Equal(this.expected, buf[:n]) {
|
||||
t.Errorf("[%d] Seek and Read got %q but expected %q", i, buf[:n], this.expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteTo(t *testing.T) {
|
||||
filename := "lazy_file_reader_test.go"
|
||||
fi, err := os.Stat(filename)
|
||||
if err != nil {
|
||||
t.Fatalf("os.Stat: %v", err)
|
||||
}
|
||||
|
||||
b, err := ioutil.ReadFile(filename)
|
||||
if err != nil {
|
||||
t.Fatalf("ioutil.ReadFile: %v", err)
|
||||
}
|
||||
|
||||
rd, err := NewLazyFileReader(filename)
|
||||
if err != nil {
|
||||
t.Fatalf("NewLazyFileReader %s: %v", filename, err)
|
||||
}
|
||||
|
||||
tst := func(testcase string, expectedSize int64, checkEqual bool) {
|
||||
buf := bytes.NewBuffer(make([]byte, 0, bytes.MinRead))
|
||||
n, err := rd.WriteTo(buf)
|
||||
if err != nil {
|
||||
t.Fatalf("WriteTo %s case: %v", testcase, err)
|
||||
}
|
||||
if n != expectedSize {
|
||||
t.Errorf("WriteTo %s case: written bytes length expected %d, got %d", testcase, expectedSize, n)
|
||||
}
|
||||
if checkEqual && !bytes.Equal(b, buf.Bytes()) {
|
||||
t.Errorf("WriteTo %s case: written bytes are different from expected", testcase)
|
||||
}
|
||||
}
|
||||
tst("No cache", fi.Size(), true)
|
||||
tst("No cache 2nd", 0, false)
|
||||
|
||||
p := make([]byte, fi.Size())
|
||||
_, err = rd.Read(p)
|
||||
if err != nil && err != io.EOF {
|
||||
t.Fatalf("Read: %v", err)
|
||||
}
|
||||
_, err = rd.Seek(0, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("Seek: %v", err)
|
||||
}
|
||||
|
||||
tst("Cache", fi.Size(), true)
|
||||
}
|
||||
Loading…
Reference in a new issue