diff --git a/source/lazy_file_reader.go b/source/lazy_file_reader.go new file mode 100644 index 000000000..0da053405 --- /dev/null +++ b/source/lazy_file_reader.go @@ -0,0 +1,160 @@ +// Copyright © 2014 Steve Francia . +// +// Licensed under the Simple Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://opensource.org/licenses/Simple-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package source + +import ( + "bytes" + "errors" + "fmt" + "io" + "io/ioutil" + "os" +) + +// LazyFileReader is an io.Reader implementation to postpone reading the file +// contents until it is really needed. It keeps filename and file contents once +// it is read. +type LazyFileReader struct { + filename string + contents *bytes.Reader + pos int64 +} + +// NewLazyFileReader creates and initializes a new LazyFileReader of filename. +// It checks whether the file can be opened. If it fails, it returns nil and an +// error. +func NewLazyFileReader(filename string) (*LazyFileReader, error) { + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + return &LazyFileReader{filename: filename, contents: nil, pos: 0}, nil +} + +// Filename returns a file name which LazyFileReader keeps +func (l *LazyFileReader) Filename() string { + return l.filename +} + +// Read reads up to len(p) bytes from the LazyFileReader's file and copies them +// into p. It returns the number of bytes read and any error encountered. If +// the file is once read, it returns its contents from cache, doesn't re-read +// the file. +func (l *LazyFileReader) Read(p []byte) (n int, err error) { + if l.contents == nil { + b, err := ioutil.ReadFile(l.filename) + if err != nil { + return 0, fmt.Errorf("failed to read content from %s: %s", l.filename, err.Error()) + } + l.contents = bytes.NewReader(b) + } + l.contents.Seek(l.pos, 0) + if err != nil { + return 0, errors.New("failed to set read position: " + err.Error()) + } + n, err = l.contents.Read(p) + l.pos += int64(n) + return n, err +} + +// Seek implements the io.Seeker interface. Once reader contents is consumed by +// Read, WriteTo etc, to read it again, it must be rewinded by this function +func (l *LazyFileReader) Seek(offset int64, whence int) (pos int64, err error) { + if l.contents == nil { + switch whence { + case 0: + pos = offset + case 1: + pos = l.pos + offset + case 2: + fi, err := os.Stat(l.filename) + if err != nil { + return 0, fmt.Errorf("failed to get %q info: %s", l.filename, err.Error()) + } + pos = fi.Size() + offset + default: + return 0, errors.New("invalid whence") + } + if pos < 0 { + return 0, errors.New("negative position") + } + } else { + pos, err = l.contents.Seek(offset, whence) + if err != nil { + return 0, err + } + } + l.pos = pos + return pos, nil +} + +// WriteTo writes data to w until all the LazyFileReader's file contents is +// drained or an error occurs. If the file is once read, it just writes its +// read cache to w, doesn't re-read the file but this method itself doesn't try +// to keep the contents in cache. +func (l *LazyFileReader) WriteTo(w io.Writer) (n int64, err error) { + if l.contents != nil { + l.contents.Seek(l.pos, 0) + if err != nil { + return 0, errors.New("failed to set read position: " + err.Error()) + } + n, err = l.contents.WriteTo(w) + l.pos += n + return n, err + } + f, err := os.Open(l.filename) + if err != nil { + return 0, fmt.Errorf("failed to open %s to read content: %s", l.filename, err.Error()) + } + defer f.Close() + + fi, err := f.Stat() + if err != nil { + return 0, fmt.Errorf("failed to get %q info: %s", l.filename, err.Error()) + } + + if l.pos >= fi.Size() { + return 0, nil + } + + // following code is taken from io.Copy in 'io/io.go' + buf := make([]byte, 32*1024) + for { + nr, er := f.Read(buf) + if nr > 0 { + nw, ew := w.Write(buf[0:nr]) + if nw > 0 { + l.pos += int64(nw) + n += int64(nw) + } + if ew != nil { + err = ew + break + } + if nr != nw { + err = io.ErrShortWrite + break + } + } + if er == io.EOF { + break + } + if er != nil { + err = er + break + } + } + return n, err +} diff --git a/source/lazy_file_reader_test.go b/source/lazy_file_reader_test.go new file mode 100644 index 000000000..60d6f49e3 --- /dev/null +++ b/source/lazy_file_reader_test.go @@ -0,0 +1,218 @@ +package source + +import ( + "bytes" + "io" + "io/ioutil" + "os" + "testing" +) + +func TestNewLazyFileReader(t *testing.T) { + filename := "itdoesnotexistfile" + _, err := NewLazyFileReader(filename) + if err == nil { + t.Errorf("NewLazyFileReader %s: error expected but no error is returned", filename) + } + + filename = "lazy_file_reader_test.go" + _, err = NewLazyFileReader(filename) + if err != nil { + t.Errorf("NewLazyFileReader %s: %v", filename, err) + } +} + +func TestFilename(t *testing.T) { + filename := "lazy_file_reader_test.go" + rd, err := NewLazyFileReader(filename) + if err != nil { + t.Fatalf("NewLazyFileReader %s: %v", filename, err) + } + if rd.Filename() != filename { + t.Errorf("Filename: expected filename %q, got %q", filename, rd.Filename()) + } +} + +func TestRead(t *testing.T) { + filename := "lazy_file_reader_test.go" + fi, err := os.Stat(filename) + if err != nil { + t.Fatalf("os.Stat: %v", err) + } + + b, err := ioutil.ReadFile(filename) + if err != nil { + t.Fatalf("ioutil.ReadFile: %v", err) + } + + rd, err := NewLazyFileReader(filename) + if err != nil { + t.Fatalf("NewLazyFileReader %s: %v", filename, err) + } + + tst := func(testcase string) { + p := make([]byte, fi.Size()) + n, err := rd.Read(p) + if err != nil { + t.Fatalf("Read %s case: %v", testcase, err) + } + if int64(n) != fi.Size() { + t.Errorf("Read %s case: read bytes length expected %d, got %d", testcase, fi.Size(), n) + } + if !bytes.Equal(b, p) { + t.Errorf("Read %s case: read bytes are different from expected", testcase) + } + } + tst("No cache") + _, err = rd.Seek(0, 0) + if err != nil { + t.Fatalf("Seek: %v", err) + } + tst("Cache") +} + +func TestSeek(t *testing.T) { + type testcase struct { + seek int + offset int64 + length int + moveto int64 + expected []byte + } + + filename := "lazy_file_reader_test.go" + b, err := ioutil.ReadFile(filename) + if err != nil { + t.Fatalf("ioutil.ReadFile: %v", err) + } + + // no cache case + for i, this := range []testcase{ + {seek: os.SEEK_SET, offset: 0, length: 10, moveto: 0, expected: b[:10]}, + {seek: os.SEEK_SET, offset: 5, length: 10, moveto: 5, expected: b[5:15]}, + {seek: os.SEEK_CUR, offset: 5, length: 10, moveto: 5, expected: b[5:15]}, // current pos = 0 + {seek: os.SEEK_END, offset: -1, length: 1, moveto: int64(len(b) - 1), expected: b[len(b)-1:]}, + {seek: 3, expected: nil}, + {seek: os.SEEK_SET, offset: -1, expected: nil}, + } { + rd, err := NewLazyFileReader(filename) + if err != nil { + t.Errorf("[%d] NewLazyFileReader %s: %v", i, filename, err) + continue + } + + pos, err := rd.Seek(this.offset, this.seek) + if this.expected == nil { + if err == nil { + t.Errorf("[%d] Seek didn't return an expected error", i) + } + } else { + if err != nil { + t.Errorf("[%d] Seek failed unexpectedly: %v", i, err) + continue + } + if pos != this.moveto { + t.Errorf("[%d] Seek failed to move the pointer: got %d, expected: %d", i, pos, this.moveto) + } + + buf := make([]byte, this.length) + n, err := rd.Read(buf) + if err != nil { + t.Errorf("[%d] Read failed unexpectedly: %v", i, err) + } + if !bytes.Equal(this.expected, buf[:n]) { + t.Errorf("[%d] Seek and Read got %q but expected %q", i, buf[:n], this.expected) + } + } + } + + // cache case + rd, err := NewLazyFileReader(filename) + if err != nil { + t.Fatalf("NewLazyFileReader %s: %v", filename, err) + } + dummy := make([]byte, len(b)) + _, err = rd.Read(dummy) + if err != nil { + t.Fatalf("Read failed unexpectedly: %v", err) + } + + for i, this := range []testcase{ + {seek: os.SEEK_SET, offset: 0, length: 10, moveto: 0, expected: b[:10]}, + {seek: os.SEEK_SET, offset: 5, length: 10, moveto: 5, expected: b[5:15]}, + {seek: os.SEEK_CUR, offset: 1, length: 10, moveto: 16, expected: b[16:26]}, // current pos = 15 + {seek: os.SEEK_END, offset: -1, length: 1, moveto: int64(len(b) - 1), expected: b[len(b)-1:]}, + {seek: 3, expected: nil}, + {seek: os.SEEK_SET, offset: -1, expected: nil}, + } { + pos, err := rd.Seek(this.offset, this.seek) + if this.expected == nil { + if err == nil { + t.Errorf("[%d] Seek didn't return an expected error", i) + } + } else { + if err != nil { + t.Errorf("[%d] Seek failed unexpectedly: %v", i, err) + continue + } + if pos != this.moveto { + t.Errorf("[%d] Seek failed to move the pointer: got %d, expected: %d", i, pos, this.moveto) + } + + buf := make([]byte, this.length) + n, err := rd.Read(buf) + if err != nil { + t.Errorf("[%d] Read failed unexpectedly: %v", i, err) + } + if !bytes.Equal(this.expected, buf[:n]) { + t.Errorf("[%d] Seek and Read got %q but expected %q", i, buf[:n], this.expected) + } + } + } +} + +func TestWriteTo(t *testing.T) { + filename := "lazy_file_reader_test.go" + fi, err := os.Stat(filename) + if err != nil { + t.Fatalf("os.Stat: %v", err) + } + + b, err := ioutil.ReadFile(filename) + if err != nil { + t.Fatalf("ioutil.ReadFile: %v", err) + } + + rd, err := NewLazyFileReader(filename) + if err != nil { + t.Fatalf("NewLazyFileReader %s: %v", filename, err) + } + + tst := func(testcase string, expectedSize int64, checkEqual bool) { + buf := bytes.NewBuffer(make([]byte, 0, bytes.MinRead)) + n, err := rd.WriteTo(buf) + if err != nil { + t.Fatalf("WriteTo %s case: %v", testcase, err) + } + if n != expectedSize { + t.Errorf("WriteTo %s case: written bytes length expected %d, got %d", testcase, expectedSize, n) + } + if checkEqual && !bytes.Equal(b, buf.Bytes()) { + t.Errorf("WriteTo %s case: written bytes are different from expected", testcase) + } + } + tst("No cache", fi.Size(), true) + tst("No cache 2nd", 0, false) + + p := make([]byte, fi.Size()) + _, err = rd.Read(p) + if err != nil && err != io.EOF { + t.Fatalf("Read: %v", err) + } + _, err = rd.Seek(0, 0) + if err != nil { + t.Fatalf("Seek: %v", err) + } + + tst("Cache", fi.Size(), true) +}