backend: Remove GetReader

backend/generic.go

@@ -3,7 +3,6 @@ package backend
 import (
 	"crypto/sha256"
 	"errors"
-	"io"
 )
 
 const (
@@ -82,39 +81,3 @@ outer:
 
 	return IDSize, nil
 }
-
-// wrap around io.LimitedReader that implements io.ReadCloser
-type blobReader struct {
-	cl     io.Closer
-	rd     io.Reader
-	closed bool
-}
-
-func (l *blobReader) Read(p []byte) (int, error) {
-	n, err := l.rd.Read(p)
-	if err == io.EOF {
-		l.Close()
-	}
-
-	return n, err
-}
-
-func (l *blobReader) Close() error {
-	if l == nil {
-		return nil
-	}
-
-	if !l.closed {
-		err := l.cl.Close()
-		l.closed = true
-		return err
-	}
-
-	return nil
-}
-
-// LimitReadCloser returns a new reader wraps r in an io.LimitReader, but also
-// implements the Close() method.
-func LimitReadCloser(r io.ReadCloser, n int64) *blobReader {
-	return &blobReader{cl: r, rd: io.LimitReader(r, n)}
-}

backend/interface.go

@@ -25,10 +25,6 @@ type Backend interface {
 	// has been called on the returned Blob.
 	Create() (Blob, error)
 
-	// GetReader returns an io.ReadCloser for the Blob with the given name of
-	// type t at offset and length.
-	GetReader(t Type, name string, offset, length uint) (io.ReadCloser, error)
-
 	// Test a boolean value whether a Blob with the name and type exists.
 	Test(t Type, name string) (bool, error)
 

backend/local/backend_test.go

@@ -44,13 +44,6 @@ func TestLocalBackendConfig(t *testing.T) {
 	test.TestConfig(t)
 }
 
-func TestLocalBackendGetReader(t *testing.T) {
-	if SkipMessage != "" {
-		t.Skip(SkipMessage)
-	}
-	test.TestGetReader(t)
-}
-
 func TestLocalBackendLoad(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

backend/local/local.go

@@ -196,33 +196,6 @@ func dirname(base string, t backend.Type, name string) string {
 	return filepath.Join(base, n)
 }
 
-// GetReader returns an io.ReadCloser for the Blob with the given name of
-// type t at offset and length. If length is 0, the reader reads until EOF.
-func (b *Local) GetReader(t backend.Type, name string, offset, length uint) (io.ReadCloser, error) {
-	f, err := os.Open(filename(b.p, t, name))
-	if err != nil {
-		return nil, err
-	}
-
-	b.mu.Lock()
-	open, _ := b.open[filename(b.p, t, name)]
-	b.open[filename(b.p, t, name)] = append(open, f)
-	b.mu.Unlock()
-
-	if offset > 0 {
-		_, err = f.Seek(int64(offset), 0)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	if length == 0 {
-		return f, nil
-	}
-
-	return backend.LimitReadCloser(f, int64(length)), nil
-}
-
 // Load returns the data stored in the backend for h at the given offset
 // and saves it in p. Load has the same semantics as io.ReaderAt.
 func (b *Local) Load(h backend.Handle, p []byte, off int64) (n int, err error) {

backend/mem/backend_test.go

@@ -44,13 +44,6 @@ func TestMemBackendConfig(t *testing.T) {
 	test.TestConfig(t)
 }
 
-func TestMemBackendGetReader(t *testing.T) {
-	if SkipMessage != "" {
-		t.Skip(SkipMessage)
-	}
-	test.TestGetReader(t)
-}
-
 func TestMemBackendLoad(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

backend/mem/mem_backend.go

@@ -41,10 +41,6 @@ func New() *MemoryBackend {
 		return memCreate(be)
 	}
 
-	be.MockBackend.GetReaderFn = func(t backend.Type, name string, offset, length uint) (io.ReadCloser, error) {
-		return memGetReader(be, t, name, offset, length)
-	}
-
 	be.MockBackend.LoadFn = func(h backend.Handle, p []byte, off int64) (int, error) {
 		return memLoad(be, h, p, off)
 	}
@@ -133,38 +129,6 @@ func memCreate(be *MemoryBackend) (backend.Blob, error) {
 	return blob, nil
 }
 
-func memGetReader(be *MemoryBackend, t backend.Type, name string, offset, length uint) (io.ReadCloser, error) {
-	be.m.Lock()
-	defer be.m.Unlock()
-
-	if t == backend.Config {
-		name = ""
-	}
-
-	debug.Log("MemoryBackend.GetReader", "get %v %v offset %v len %v", t, name, offset, length)
-
-	if _, ok := be.data[entry{t, name}]; !ok {
-		return nil, errors.New("no such data")
-	}
-
-	buf := be.data[entry{t, name}]
-	if offset > uint(len(buf)) {
-		return nil, errors.New("offset beyond end of file")
-	}
-
-	buf = buf[offset:]
-
-	if length > 0 {
-		if length > uint(len(buf)) {
-			length = uint(len(buf))
-		}
-
-		buf = buf[:length]
-	}
-
-	return backend.ReadCloser(bytes.NewReader(buf)), nil
-}
-
 func memLoad(be *MemoryBackend, h backend.Handle, p []byte, off int64) (int, error) {
 	be.m.Lock()
 	defer be.m.Unlock()

backend/mock_backend.go

@@ -1,23 +1,19 @@
 package backend
 
-import (
+import "errors"
-	"errors"
-	"io"
-)
 
 // MockBackend implements a backend whose functions can be specified. This
 // should only be used for tests.
 type MockBackend struct {
-	CloseFn     func() error
+	CloseFn    func() error
-	CreateFn    func() (Blob, error)
+	CreateFn   func() (Blob, error)
-	LoadFn      func(h Handle, p []byte, off int64) (int, error)
+	LoadFn     func(h Handle, p []byte, off int64) (int, error)
-	StatFn      func(h Handle) (BlobInfo, error)
+	StatFn     func(h Handle) (BlobInfo, error)
-	GetReaderFn func(Type, string, uint, uint) (io.ReadCloser, error)
+	ListFn     func(Type, <-chan struct{}) <-chan string
-	ListFn      func(Type, <-chan struct{}) <-chan string
+	RemoveFn   func(Type, string) error
-	RemoveFn    func(Type, string) error
+	TestFn     func(Type, string) (bool, error)
-	TestFn      func(Type, string) (bool, error)
+	DeleteFn   func() error
-	DeleteFn    func() error
+	LocationFn func() string
-	LocationFn  func() string
 }
 
 func (m *MockBackend) Close() error {
@@ -60,14 +56,6 @@ func (m *MockBackend) Stat(h Handle) (BlobInfo, error) {
 	return m.StatFn(h)
 }
 
-func (m *MockBackend) GetReader(t Type, name string, offset, len uint) (io.ReadCloser, error) {
-	if m.GetReaderFn == nil {
-		return nil, errors.New("not implemented")
-	}
-
-	return m.GetReaderFn(t, name, offset, len)
-}
-
 func (m *MockBackend) List(t Type, done <-chan struct{}) <-chan string {
 	if m.ListFn == nil {
 		ch := make(chan string)

backend/s3/backend_test.go

@@ -44,13 +44,6 @@ func TestS3BackendConfig(t *testing.T) {
 	test.TestConfig(t)
 }
 
-func TestS3BackendGetReader(t *testing.T) {
-	if SkipMessage != "" {
-		t.Skip(SkipMessage)
-	}
-	test.TestGetReader(t)
-}
-
 func TestS3BackendLoad(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

backend/s3/s3.go

@@ -145,31 +145,6 @@ func (be *S3Backend) Create() (backend.Blob, error) {
 	return &blob, nil
 }
 
-// GetReader returns an io.ReadCloser for the Blob with the given name of
-// type t at offset and length. If length is 0, the reader reads until EOF.
-func (be *S3Backend) GetReader(t backend.Type, name string, offset, length uint) (io.ReadCloser, error) {
-	debug.Log("s3.GetReader", "%v %v, offset %v len %v", t, name, offset, length)
-	path := s3path(t, name)
-	obj, err := be.client.GetObject(be.bucketname, path)
-	if err != nil {
-		debug.Log("s3.GetReader", "  err %v", err)
-		return nil, err
-	}
-
-	if offset > 0 {
-		_, err = obj.Seek(int64(offset), 0)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	if length == 0 {
-		return obj, nil
-	}
-
-	return backend.LimitReadCloser(obj, int64(length)), nil
-}
-
 // Load returns the data stored in the backend for h at the given offset
 // and saves it in p. Load has the same semantics as io.ReaderAt.
 func (be S3Backend) Load(h backend.Handle, p []byte, off int64) (int, error) {

backend/sftp/backend_test.go

@@ -44,13 +44,6 @@ func TestSftpBackendConfig(t *testing.T) {
 	test.TestConfig(t)
 }
 
-func TestSftpBackendGetReader(t *testing.T) {
-	if SkipMessage != "" {
-		t.Skip(SkipMessage)
-	}
-	test.TestGetReader(t)
-}
-
 func TestSftpBackendLoad(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

backend/sftp/sftp.go

@@ -344,28 +344,6 @@ func (r *SFTP) dirname(t backend.Type, name string) string {
 	return Join(r.p, n)
 }
 
-// GetReader returns an io.ReadCloser for the Blob with the given name of
-// type t at offset and length. If length is 0, the reader reads until EOF.
-func (r *SFTP) GetReader(t backend.Type, name string, offset, length uint) (io.ReadCloser, error) {
-	f, err := r.c.Open(r.filename(t, name))
-	if err != nil {
-		return nil, err
-	}
-
-	if offset > 0 {
-		_, err = f.Seek(int64(offset), 0)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	if length == 0 {
-		return f, nil
-	}
-
-	return backend.LimitReadCloser(f, int64(length)), nil
-}
-
 // Load returns the data stored in the backend for h at the given offset
 // and saves it in p. Load has the same semantics as io.ReaderAt.
 func (r *SFTP) Load(h backend.Handle, p []byte, off int64) (n int, err error) {

backend/test/backend_test.go

@@ -44,13 +44,6 @@ func TestTestBackendConfig(t *testing.T) {
 	test.TestConfig(t)
 }
 
-func TestTestBackendGetReader(t *testing.T) {
-	if SkipMessage != "" {
-		t.Skip(SkipMessage)
-	}
-	test.TestGetReader(t)
-}
-
 func TestTestBackendLoad(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

backend/test/tests.go

@@ -152,7 +152,7 @@ func TestConfig(t testing.TB) {
 	var testString = "Config"
 
 	// create config and read it back
-	_, err := b.GetReader(backend.Config, "", 0, 0)
+	_, err := backend.LoadAll(b, backend.Handle{Type: backend.Config}, nil)
 	if err == nil {
 		t.Fatalf("did not get expected error for non-existing config")
 	}
@@ -175,76 +175,17 @@ func TestConfig(t testing.TB) {
 	// try accessing the config with different names, should all return the
 	// same config
 	for _, name := range []string{"", "foo", "bar", "0000000000000000000000000000000000000000000000000000000000000000"} {
-		rd, err := b.GetReader(backend.Config, name, 0, 0)
+		buf, err := backend.LoadAll(b, backend.Handle{Type: backend.Config}, nil)
 		if err != nil {
 			t.Fatalf("unable to read config with name %q: %v", name, err)
 		}
 
-		buf, err := ioutil.ReadAll(rd)
-		if err != nil {
-			t.Fatalf("read config error: %v", err)
-		}
-
-		err = rd.Close()
-		if err != nil {
-			t.Fatalf("close error: %v", err)
-		}
-
 		if string(buf) != testString {
 			t.Fatalf("wrong data returned, want %q, got %q", testString, string(buf))
 		}
 	}
 }
 
-// TestGetReader tests various ways the GetReader function can be called.
-func TestGetReader(t testing.TB) {
-	b := open(t)
-	defer close(t)
-
-	length := rand.Intn(1<<24) + 2000
-
-	data := make([]byte, length)
-	_, err := io.ReadFull(crand.Reader, data)
-	OK(t, err)
-
-	blob, err := b.Create()
-	OK(t, err)
-
-	id := backend.Hash(data)
-
-	_, err = blob.Write([]byte(data))
-	OK(t, err)
-	OK(t, blob.Finalize(backend.Data, id.String()))
-
-	for i := 0; i < 500; i++ {
-		l := rand.Intn(length + 2000)
-		o := rand.Intn(length + 2000)
-
-		d := data
-		if o < len(d) {
-			d = d[o:]
-		} else {
-			o = len(d)
-			d = d[:0]
-		}
-
-		if l > 0 && l < len(d) {
-			d = d[:l]
-		}
-
-		rd, err := b.GetReader(backend.Data, id.String(), uint(o), uint(l))
-		OK(t, err)
-		buf, err := ioutil.ReadAll(rd)
-		OK(t, err)
-
-		if !bytes.Equal(buf, d) {
-			t.Fatalf("data not equal")
-		}
-	}
-
-	OK(t, b.Remove(backend.Data, id.String()))
-}
-
 // TestLoad tests the backend's Load function.
 func TestLoad(t testing.TB) {
 	b := open(t)
@@ -360,12 +301,8 @@ func TestWrite(t testing.TB) {
 		name := fmt.Sprintf("%s-%d", id, i)
 		OK(t, blob.Finalize(backend.Data, name))
 
-		rd, err := b.GetReader(backend.Data, name, 0, 0)
+		buf, err := backend.LoadAll(b, backend.Handle{Type: backend.Data, Name: name}, nil)
 		OK(t, err)
-
-		buf, err := ioutil.ReadAll(rd)
-		OK(t, err)
-
 		if len(buf) != len(data) {
 			t.Fatalf("number of bytes does not match, want %v, got %v", len(data), len(buf))
 		}
@@ -436,12 +373,13 @@ func TestBackend(t testing.TB) {
 			OK(t, err)
 			Assert(t, !ret, "blob was found to exist before creating")
 
-			// try to open not existing blob
+			// try to stat a not existing blob
-			_, err = b.GetReader(tpe, id.String(), 0, 0)
+			h := backend.Handle{Type: tpe, Name: id.String()}
+			_, err = b.Stat(h)
 			Assert(t, err != nil, "blob data could be extracted before creation")
 
 			// try to read not existing blob
-			_, err = b.GetReader(tpe, id.String(), 0, 1)
+			_, err = b.Load(h, nil, 0)
 			Assert(t, err != nil, "blob reader could be obtained before creation")
 
 			// try to get string out, should fail
@@ -454,24 +392,22 @@ func TestBackend(t testing.TB) {
 		for _, test := range testStrings {
 			store(t, b, tpe, []byte(test.data))
 
-			// test GetReader()
+			// test Load()
-			rd, err := b.GetReader(tpe, test.id, 0, uint(len(test.data)))
+			h := backend.Handle{Type: tpe, Name: test.id}
+			buf, err := backend.LoadAll(b, h, nil)
 			OK(t, err)
-			Assert(t, rd != nil, "GetReader() returned nil")
+			Equals(t, test.data, string(buf))
-
-			read(t, rd, []byte(test.data))
-			OK(t, rd.Close())
 
 			// try to read it out with an offset and a length
 			start := 1
 			end := len(test.data) - 2
 			length := end - start
-			rd, err = b.GetReader(tpe, test.id, uint(start), uint(length))
-			OK(t, err)
-			Assert(t, rd != nil, "GetReader() returned nil")
 
-			read(t, rd, []byte(test.data[start:end]))
+			buf2 := make([]byte, length)
-			OK(t, rd.Close())
+			n, err := b.Load(h, buf2, int64(start))
+			OK(t, err)
+			Equals(t, length, n)
+			Equals(t, test.data[start:end], string(buf2))
 		}
 
 		// test adding the first file again