Add backend.Get()

src/restic/backend.go

@@ -26,6 +26,11 @@ type Backend interface {
 	// Save stores the data in the backend under the given handle.
 	Save(h Handle, rd io.Reader) error
 
+	// Get returns a reader that yields the contents of the file at h at the
+	// given offset. If length is nonzero, only a portion of the file is
+	// returned. rd must be closed after use.
+	Get(h Handle, length int, offset int64) (io.ReadCloser, error)
+
 	// Stat returns information about the File identified by h.
 	Stat(h Handle) (FileInfo, error)
 

src/restic/backend/local/backend_test.go

@@ -58,6 +58,13 @@ func TestLocalBackendLoadNegativeOffset(t *testing.T) {
 	test.TestLoadNegativeOffset(t)
 }
 
+func TestLocalBackendGet(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestGet(t)
+}
+
 func TestLocalBackendSave(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

src/restic/backend/local/local.go

@@ -206,6 +206,39 @@ func (b *Local) Save(h restic.Handle, rd io.Reader) (err error) {
 	return setNewFileMode(filename, fi)
 }
 
+// Get returns a reader that yields the contents of the file at h at the
+// given offset. If length is nonzero, only a portion of the file is
+// returned. rd must be closed after use.
+func (b *Local) Get(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	debug.Log("Get %v, length %v, offset %v", h, length, offset)
+	if err := h.Valid(); err != nil {
+		return nil, err
+	}
+
+	if offset < 0 {
+		return nil, errors.New("offset is negative")
+	}
+
+	f, err := os.Open(filename(b.p, h.Type, h.Name))
+	if err != nil {
+		return nil, err
+	}
+
+	if offset > 0 {
+		_, err = f.Seek(offset, 0)
+		if err != nil {
+			f.Close()
+			return nil, err
+		}
+	}
+
+	if length > 0 {
+		return backend.LimitReadCloser(f, int64(length)), nil
+	}
+
+	return f, nil
+}
+
 // Stat returns information about a blob.
 func (b *Local) Stat(h restic.Handle) (restic.FileInfo, error) {
 	debug.Log("Stat %v", h)

src/restic/backend/mem/backend_test.go

@@ -58,6 +58,13 @@ func TestMemBackendLoadNegativeOffset(t *testing.T) {
 	test.TestLoadNegativeOffset(t)
 }
 
+func TestMemBackendGet(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestGet(t)
+}
+
 func TestMemBackendSave(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

src/restic/backend/mem/mem_backend.go

@@ -1,11 +1,13 @@
 package mem
 
 import (
+	"bytes"
 	"io"
 	"io/ioutil"
 	"restic"
 	"sync"
 
+	"restic/backend"
 	"restic/errors"
 
 	"restic/debug"
@@ -121,6 +123,44 @@ func (be *MemoryBackend) Save(h restic.Handle, rd io.Reader) error {
 	return nil
 }
 
+// Get returns a reader that yields the contents of the file at h at the
+// given offset. If length is nonzero, only a portion of the file is
+// returned. rd must be closed after use.
+func (be *MemoryBackend) Get(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	if err := h.Valid(); err != nil {
+		return nil, err
+	}
+
+	be.m.Lock()
+	defer be.m.Unlock()
+
+	if h.Type == restic.ConfigFile {
+		h.Name = ""
+	}
+
+	debug.Log("Get %v offset %v len %v", h, offset, length)
+
+	if offset < 0 {
+		return nil, errors.New("offset is negative")
+	}
+
+	if _, ok := be.data[entry{h.Type, h.Name}]; !ok {
+		return nil, errors.New("no such data")
+	}
+
+	buf := be.data[entry{h.Type, h.Name}]
+	if offset > int64(len(buf)) {
+		return nil, errors.New("offset beyond end of file")
+	}
+
+	buf = buf[offset:]
+	if length > 0 && len(buf) > length {
+		buf = buf[:length]
+	}
+
+	return backend.Closer{bytes.NewReader(buf)}, nil
+}
+
 // Stat returns information about a file in the backend.
 func (be *MemoryBackend) Stat(h restic.Handle) (restic.FileInfo, error) {
 	be.m.Lock()

src/restic/backend/rest/backend_test.go

@@ -58,6 +58,13 @@ func TestRestBackendLoadNegativeOffset(t *testing.T) {
 	test.TestLoadNegativeOffset(t)
 }
 
+func TestRestBackendGet(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestGet(t)
+}
+
 func TestRestBackendSave(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

src/restic/backend/rest/rest.go

@@ -11,6 +11,7 @@ import (
 	"restic"
 	"strings"
 
+	"restic/debug"
 	"restic/errors"
 
 	"restic/backend"
@@ -76,10 +77,15 @@ func (b *restBackend) Location() string {
 // Load returns the data stored in the backend for h at the given offset
 // and saves it in p. Load has the same semantics as io.ReaderAt.
 func (b *restBackend) Load(h restic.Handle, p []byte, off int64) (n int, err error) {
+	debug.Log("Load(%v, length %v, offset %v)", h, len(p), off)
 	if err := h.Valid(); err != nil {
 		return 0, err
 	}
 
+	if len(p) == 0 {
+		return 0, errors.New("buffer length is zero")
+	}
+
 	// invert offset
 	if off < 0 {
 		info, err := b.Stat(h)
@@ -98,6 +104,7 @@ func (b *restBackend) Load(h restic.Handle, p []byte, off int64) (n int, err err
 	if err != nil {
 		return 0, errors.Wrap(err, "http.NewRequest")
 	}
+	debug.Log("Load(%v) send range %d-%d", h, off, off+int64(len(p)-1))
 	req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", off, off+int64(len(p))))
 	<-b.connChan
 	resp, err := b.client.Do(req)
@@ -156,6 +163,56 @@ func (b *restBackend) Save(h restic.Handle, rd io.Reader) (err error) {
 	return nil
 }
 
+// Get returns a reader that yields the contents of the file at h at the
+// given offset. If length is nonzero, only a portion of the file is
+// returned. rd must be closed after use.
+func (b *restBackend) Get(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	debug.Log("Get %v, length %v, offset %v", h, length, offset)
+	if err := h.Valid(); err != nil {
+		return nil, err
+	}
+
+	if offset < 0 {
+		return nil, errors.New("offset is negative")
+	}
+
+	if length < 0 {
+		return nil, errors.Errorf("invalid length %d", length)
+	}
+
+	req, err := http.NewRequest("GET", restPath(b.url, h), nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "http.NewRequest")
+	}
+
+	byteRange := fmt.Sprintf("bytes=%d-", offset)
+	if length > 0 {
+		byteRange = fmt.Sprintf("bytes=%d-%d", offset, offset+int64(length)-1)
+	}
+	req.Header.Add("Range", byteRange)
+	debug.Log("Get(%v) send range %v", h, byteRange)
+
+	<-b.connChan
+	resp, err := b.client.Do(req)
+	b.connChan <- struct{}{}
+
+	if err != nil {
+		if resp != nil {
+			io.Copy(ioutil.Discard, resp.Body)
+			resp.Body.Close()
+		}
+		return nil, errors.Wrap(err, "client.Do")
+	}
+
+	if resp.StatusCode != 200 && resp.StatusCode != 206 {
+		io.Copy(ioutil.Discard, resp.Body)
+		resp.Body.Close()
+		return nil, errors.Errorf("unexpected HTTP response code %v", resp.StatusCode)
+	}
+
+	return resp.Body, nil
+}
+
 // Stat returns information about a blob.
 func (b *restBackend) Stat(h restic.Handle) (restic.FileInfo, error) {
 	if err := h.Valid(); err != nil {

src/restic/backend/s3/backend_test.go

@@ -58,6 +58,13 @@ func TestS3BackendLoadNegativeOffset(t *testing.T) {
 	test.TestLoadNegativeOffset(t)
 }
 
+func TestS3BackendGet(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestGet(t)
+}
+
 func TestS3BackendSave(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

src/restic/backend/s3/s3.go

@@ -1,11 +1,13 @@
 package s3
 
 import (
+	"bytes"
 	"io"
 	"path"
 	"restic"
 	"strings"
 
+	"restic/backend"
 	"restic/errors"
 
 	"github.com/minio/minio-go"
@@ -74,7 +76,7 @@ func (be *s3) Location() string {
 
 // Load returns the data stored in the backend for h at the given offset
 // and saves it in p. Load has the same semantics as io.ReaderAt.
-func (be s3) Load(h restic.Handle, p []byte, off int64) (n int, err error) {
+func (be *s3) Load(h restic.Handle, p []byte, off int64) (n int, err error) {
 	var obj *minio.Object
 
 	debug.Log("%v, offset %v, len %v", h, off, len(p))
@@ -146,7 +148,7 @@ func (be s3) Load(h restic.Handle, p []byte, off int64) (n int, err error) {
 }
 
 // Save stores data in the backend at the handle.
-func (be s3) Save(h restic.Handle, rd io.Reader) (err error) {
+func (be *s3) Save(h restic.Handle, rd io.Reader) (err error) {
 	if err := h.Valid(); err != nil {
 		return err
 	}
@@ -175,8 +177,82 @@ func (be s3) Save(h restic.Handle, rd io.Reader) (err error) {
 	return errors.Wrap(err, "client.PutObject")
 }
 
+// Get returns a reader that yields the contents of the file at h at the
+// given offset. If length is nonzero, only a portion of the file is
+// returned. rd must be closed after use.
+func (be *s3) Get(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	debug.Log("Get %v, length %v, offset %v", h, length, offset)
+	if err := h.Valid(); err != nil {
+		return nil, err
+	}
+
+	if offset < 0 {
+		return nil, errors.New("offset is negative")
+	}
+
+	if length < 0 {
+		return nil, errors.Errorf("invalid length %d", length)
+	}
+
+	var obj *minio.Object
+
+	objName := be.s3path(h.Type, h.Name)
+
+	<-be.connChan
+	defer func() {
+		be.connChan <- struct{}{}
+	}()
+
+	obj, err := be.client.GetObject(be.bucketname, objName)
+	if err != nil {
+		debug.Log("  err %v", err)
+		return nil, errors.Wrap(err, "client.GetObject")
+	}
+
+	// if we're going to read the whole object, just pass it on.
+	if length == 0 {
+		debug.Log("Get %v: pass on object", h)
+		_, err = obj.Seek(offset, 0)
+		if err != nil {
+			return nil, errors.Wrap(err, "obj.Seek")
+		}
+
+		return obj, nil
+	}
+
+	// otherwise use a buffer with ReadAt
+	info, err := obj.Stat()
+	if err != nil {
+		return nil, errors.Wrap(err, "obj.Stat")
+	}
+
+	if offset > info.Size {
+		return nil, errors.Errorf("offset larger than file size")
+	}
+
+	l := int64(length)
+	if offset+l > info.Size {
+		l = info.Size - offset
+	}
+
+	buf := make([]byte, l)
+	n, err := obj.ReadAt(buf, offset)
+	debug.Log("Get %v: use buffer with ReadAt: %v, %v", h, n, err)
+	if err == io.EOF {
+		debug.Log("Get %v: shorten buffer %v -> %v", h, len(buf), n)
+		buf = buf[:n]
+		err = nil
+	}
+
+	if err != nil {
+		return nil, errors.Wrap(err, "obj.ReadAt")
+	}
+
+	return backend.Closer{Reader: bytes.NewReader(buf)}, nil
+}
+
 // Stat returns information about a blob.
-func (be s3) Stat(h restic.Handle) (bi restic.FileInfo, err error) {
+func (be *s3) Stat(h restic.Handle) (bi restic.FileInfo, err error) {
 	debug.Log("%v", h)
 
 	objName := be.s3path(h.Type, h.Name)

src/restic/backend/sftp/backend_test.go

@@ -58,6 +58,13 @@ func TestSftpBackendLoadNegativeOffset(t *testing.T) {
 	test.TestLoadNegativeOffset(t)
 }
 
+func TestSftpBackendGet(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestGet(t)
+}
+
 func TestSftpBackendSave(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

src/restic/backend/sftp/sftp.go

@@ -398,6 +398,39 @@ func (r *SFTP) Save(h restic.Handle, rd io.Reader) (err error) {
 	return err
 }
 
+// Get returns a reader that yields the contents of the file at h at the
+// given offset. If length is nonzero, only a portion of the file is
+// returned. rd must be closed after use.
+func (r *SFTP) Get(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	debug.Log("Get %v, length %v, offset %v", h, length, offset)
+	if err := h.Valid(); err != nil {
+		return nil, err
+	}
+
+	if offset < 0 {
+		return nil, errors.New("offset is negative")
+	}
+
+	f, err := r.c.Open(r.filename(h.Type, h.Name))
+	if err != nil {
+		return nil, err
+	}
+
+	if offset > 0 {
+		_, err = f.Seek(offset, 0)
+		if err != nil {
+			f.Close()
+			return nil, err
+		}
+	}
+
+	if length > 0 {
+		return backend.LimitReadCloser(f, int64(length)), nil
+	}
+
+	return f, nil
+}
+
 // Stat returns information about a blob.
 func (r *SFTP) Stat(h restic.Handle) (restic.FileInfo, error) {
 	debug.Log("stat %v", h)

src/restic/backend/test/backend_test.go

@@ -58,6 +58,13 @@ func TestTestBackendLoadNegativeOffset(t *testing.T) {
 	test.TestLoadNegativeOffset(t)
 }
 
+func TestTestBackendGet(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestGet(t)
+}
+
 func TestTestBackendSave(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

src/restic/backend/test/tests.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"math/rand"
 	"reflect"
 	"restic"
@@ -369,6 +370,99 @@ func TestLoadNegativeOffset(t testing.TB) {
 	test.OK(t, b.Remove(restic.DataFile, id.String()))
 }
 
+// TestGet tests the backend's Get function.
+func TestGet(t testing.TB) {
+	b := open(t)
+	defer close(t)
+
+	_, err := b.Get(restic.Handle{}, 0, 0)
+	if err == nil {
+		t.Fatalf("Get() did not return an error for invalid handle")
+	}
+
+	_, err = b.Get(restic.Handle{Type: restic.DataFile, Name: "foobar"}, 0, 0)
+	if err == nil {
+		t.Fatalf("Get() did not return an error for non-existing blob")
+	}
+
+	length := rand.Intn(1<<24) + 2000
+
+	data := test.Random(23, length)
+	id := restic.Hash(data)
+
+	handle := restic.Handle{Type: restic.DataFile, Name: id.String()}
+	err = b.Save(handle, bytes.NewReader(data))
+	if err != nil {
+		t.Fatalf("Save() error: %v", err)
+	}
+
+	rd, err := b.Get(handle, 100, -1)
+	if err == nil {
+		t.Fatalf("Get() returned no error for negative offset!")
+	}
+
+	if rd != nil {
+		t.Fatalf("Get() returned a non-nil reader for negative offset!")
+	}
+
+	for i := 0; i < 50; i++ {
+		l := rand.Intn(length + 2000)
+		o := rand.Intn(length + 2000)
+
+		d := data
+		if o < len(d) {
+			d = d[o:]
+		} else {
+			o = len(d)
+			d = d[:0]
+		}
+
+		getlen := l
+		if l >= len(d) && rand.Float32() >= 0.5 {
+			getlen = 0
+		}
+
+		if l > 0 && l < len(d) {
+			d = d[:l]
+		}
+
+		rd, err := b.Get(handle, getlen, int64(o))
+		if err != nil {
+			t.Errorf("Get(%d, %d) returned unexpected error: %v", l, o, err)
+			continue
+		}
+
+		buf, err := ioutil.ReadAll(rd)
+		if err != nil {
+			t.Errorf("Get(%d, %d) ReadAll() returned unexpected error: %v", l, o, err)
+			continue
+		}
+
+		if l <= len(d) && len(buf) != l {
+			t.Errorf("Get(%d, %d) wrong number of bytes read: want %d, got %d", l, o, l, len(buf))
+			continue
+		}
+
+		if l > len(d) && len(buf) != len(d) {
+			t.Errorf("Get(%d, %d) wrong number of bytes read for overlong read: want %d, got %d", l, o, l, len(buf))
+			continue
+		}
+
+		if !bytes.Equal(buf, d) {
+			t.Errorf("Get(%d, %d) returned wrong bytes", l, o)
+			continue
+		}
+
+		err = rd.Close()
+		if err != nil {
+			t.Errorf("Get(%d, %d) rd.Close() returned unexpected error: %v", l, o, err)
+			continue
+		}
+	}
+
+	test.OK(t, b.Remove(restic.DataFile, id.String()))
+}
+
 // TestSave tests saving data in the backend.
 func TestSave(t testing.TB) {
 	b := open(t)

src/restic/backend/utils.go

@@ -28,3 +28,30 @@ func LoadAll(be restic.Backend, h restic.Handle, buf []byte) ([]byte, error) {
 	buf = buf[:n]
 	return buf, err
 }
+
+// Closer wraps an io.Reader and adds a Close() method that does nothing.
+type Closer struct {
+	io.Reader
+}
+
+// Close is a no-op.
+func (c Closer) Close() error {
+	return nil
+}
+
+// LimitedReadCloser wraps io.LimitedReader and exposes the Close() method.
+type LimitedReadCloser struct {
+	io.ReadCloser
+	io.Reader
+}
+
+// Read reads data from the limited reader.
+func (l *LimitedReadCloser) Read(p []byte) (int, error) {
+	return l.Reader.Read(p)
+}
+
+// LimitReadCloser returns a new reader wraps r in an io.LimitReader, but also
+// exposes the Close() method.
+func LimitReadCloser(r io.ReadCloser, n int64) *LimitedReadCloser {
+	return &LimitedReadCloser{ReadCloser: r, Reader: io.LimitReader(r, n)}
+}

src/restic/mock/backend.go

@@ -12,6 +12,7 @@ type Backend struct {
 	CloseFn    func() error
 	LoadFn     func(h restic.Handle, p []byte, off int64) (int, error)
 	SaveFn     func(h restic.Handle, rd io.Reader) error
+	GetFn      func(h restic.Handle, length int, offset int64) (io.ReadCloser, error)
 	StatFn     func(h restic.Handle) (restic.FileInfo, error)
 	ListFn     func(restic.FileType, <-chan struct{}) <-chan string
 	RemoveFn   func(restic.FileType, string) error
@@ -56,6 +57,15 @@ func (m *Backend) Save(h restic.Handle, rd io.Reader) error {
 	return m.SaveFn(h, rd)
 }
 
+// Get loads data from the backend.
+func (m *Backend) Get(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	if m.GetFn == nil {
+		return nil, errors.New("not implemented")
+	}
+
+	return m.GetFn(h, length, offset)
+}
+
 // Stat an object in the backend.
 func (m *Backend) Stat(h restic.Handle) (restic.FileInfo, error) {
 	if m.StatFn == nil {