2
2
mirror of https://github.com/octoleo/restic.git synced 2024-12-23 11:28:54 +00:00

Add backend.Get()

This commit is contained in:
Alexander Neumann 2017-01-22 22:01:12 +01:00
parent a36c01372d
commit 05afedd950
15 changed files with 420 additions and 3 deletions

View File

@ -26,6 +26,11 @@ type Backend interface {
// Save stores the data in the backend under the given handle. // Save stores the data in the backend under the given handle.
Save(h Handle, rd io.Reader) error Save(h Handle, rd io.Reader) error
// Get returns a reader that yields the contents of the file at h at the
// given offset. If length is nonzero, only a portion of the file is
// returned. rd must be closed after use.
Get(h Handle, length int, offset int64) (io.ReadCloser, error)
// Stat returns information about the File identified by h. // Stat returns information about the File identified by h.
Stat(h Handle) (FileInfo, error) Stat(h Handle) (FileInfo, error)

View File

@ -58,6 +58,13 @@ func TestLocalBackendLoadNegativeOffset(t *testing.T) {
test.TestLoadNegativeOffset(t) test.TestLoadNegativeOffset(t)
} }
func TestLocalBackendGet(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestGet(t)
}
func TestLocalBackendSave(t *testing.T) { func TestLocalBackendSave(t *testing.T) {
if SkipMessage != "" { if SkipMessage != "" {
t.Skip(SkipMessage) t.Skip(SkipMessage)

View File

@ -206,6 +206,39 @@ func (b *Local) Save(h restic.Handle, rd io.Reader) (err error) {
return setNewFileMode(filename, fi) return setNewFileMode(filename, fi)
} }
// Get returns a reader that yields the contents of the file at h at the
// given offset. If length is nonzero, only a portion of the file is
// returned. rd must be closed after use.
func (b *Local) Get(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
debug.Log("Get %v, length %v, offset %v", h, length, offset)
if err := h.Valid(); err != nil {
return nil, err
}
if offset < 0 {
return nil, errors.New("offset is negative")
}
f, err := os.Open(filename(b.p, h.Type, h.Name))
if err != nil {
return nil, err
}
if offset > 0 {
_, err = f.Seek(offset, 0)
if err != nil {
f.Close()
return nil, err
}
}
if length > 0 {
return backend.LimitReadCloser(f, int64(length)), nil
}
return f, nil
}
// Stat returns information about a blob. // Stat returns information about a blob.
func (b *Local) Stat(h restic.Handle) (restic.FileInfo, error) { func (b *Local) Stat(h restic.Handle) (restic.FileInfo, error) {
debug.Log("Stat %v", h) debug.Log("Stat %v", h)

View File

@ -58,6 +58,13 @@ func TestMemBackendLoadNegativeOffset(t *testing.T) {
test.TestLoadNegativeOffset(t) test.TestLoadNegativeOffset(t)
} }
func TestMemBackendGet(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestGet(t)
}
func TestMemBackendSave(t *testing.T) { func TestMemBackendSave(t *testing.T) {
if SkipMessage != "" { if SkipMessage != "" {
t.Skip(SkipMessage) t.Skip(SkipMessage)

View File

@ -1,11 +1,13 @@
package mem package mem
import ( import (
"bytes"
"io" "io"
"io/ioutil" "io/ioutil"
"restic" "restic"
"sync" "sync"
"restic/backend"
"restic/errors" "restic/errors"
"restic/debug" "restic/debug"
@ -121,6 +123,44 @@ func (be *MemoryBackend) Save(h restic.Handle, rd io.Reader) error {
return nil return nil
} }
// Get returns a reader that yields the contents of the file at h at the
// given offset. If length is nonzero, only a portion of the file is
// returned. rd must be closed after use.
func (be *MemoryBackend) Get(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
if err := h.Valid(); err != nil {
return nil, err
}
be.m.Lock()
defer be.m.Unlock()
if h.Type == restic.ConfigFile {
h.Name = ""
}
debug.Log("Get %v offset %v len %v", h, offset, length)
if offset < 0 {
return nil, errors.New("offset is negative")
}
if _, ok := be.data[entry{h.Type, h.Name}]; !ok {
return nil, errors.New("no such data")
}
buf := be.data[entry{h.Type, h.Name}]
if offset > int64(len(buf)) {
return nil, errors.New("offset beyond end of file")
}
buf = buf[offset:]
if length > 0 && len(buf) > length {
buf = buf[:length]
}
return backend.Closer{bytes.NewReader(buf)}, nil
}
// Stat returns information about a file in the backend. // Stat returns information about a file in the backend.
func (be *MemoryBackend) Stat(h restic.Handle) (restic.FileInfo, error) { func (be *MemoryBackend) Stat(h restic.Handle) (restic.FileInfo, error) {
be.m.Lock() be.m.Lock()

View File

@ -58,6 +58,13 @@ func TestRestBackendLoadNegativeOffset(t *testing.T) {
test.TestLoadNegativeOffset(t) test.TestLoadNegativeOffset(t)
} }
func TestRestBackendGet(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestGet(t)
}
func TestRestBackendSave(t *testing.T) { func TestRestBackendSave(t *testing.T) {
if SkipMessage != "" { if SkipMessage != "" {
t.Skip(SkipMessage) t.Skip(SkipMessage)

View File

@ -11,6 +11,7 @@ import (
"restic" "restic"
"strings" "strings"
"restic/debug"
"restic/errors" "restic/errors"
"restic/backend" "restic/backend"
@ -76,10 +77,15 @@ func (b *restBackend) Location() string {
// Load returns the data stored in the backend for h at the given offset // Load returns the data stored in the backend for h at the given offset
// and saves it in p. Load has the same semantics as io.ReaderAt. // and saves it in p. Load has the same semantics as io.ReaderAt.
func (b *restBackend) Load(h restic.Handle, p []byte, off int64) (n int, err error) { func (b *restBackend) Load(h restic.Handle, p []byte, off int64) (n int, err error) {
debug.Log("Load(%v, length %v, offset %v)", h, len(p), off)
if err := h.Valid(); err != nil { if err := h.Valid(); err != nil {
return 0, err return 0, err
} }
if len(p) == 0 {
return 0, errors.New("buffer length is zero")
}
// invert offset // invert offset
if off < 0 { if off < 0 {
info, err := b.Stat(h) info, err := b.Stat(h)
@ -98,6 +104,7 @@ func (b *restBackend) Load(h restic.Handle, p []byte, off int64) (n int, err err
if err != nil { if err != nil {
return 0, errors.Wrap(err, "http.NewRequest") return 0, errors.Wrap(err, "http.NewRequest")
} }
debug.Log("Load(%v) send range %d-%d", h, off, off+int64(len(p)-1))
req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", off, off+int64(len(p)))) req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", off, off+int64(len(p))))
<-b.connChan <-b.connChan
resp, err := b.client.Do(req) resp, err := b.client.Do(req)
@ -156,6 +163,56 @@ func (b *restBackend) Save(h restic.Handle, rd io.Reader) (err error) {
return nil return nil
} }
// Get returns a reader that yields the contents of the file at h at the
// given offset. If length is nonzero, only a portion of the file is
// returned. rd must be closed after use.
func (b *restBackend) Get(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
debug.Log("Get %v, length %v, offset %v", h, length, offset)
if err := h.Valid(); err != nil {
return nil, err
}
if offset < 0 {
return nil, errors.New("offset is negative")
}
if length < 0 {
return nil, errors.Errorf("invalid length %d", length)
}
req, err := http.NewRequest("GET", restPath(b.url, h), nil)
if err != nil {
return nil, errors.Wrap(err, "http.NewRequest")
}
byteRange := fmt.Sprintf("bytes=%d-", offset)
if length > 0 {
byteRange = fmt.Sprintf("bytes=%d-%d", offset, offset+int64(length)-1)
}
req.Header.Add("Range", byteRange)
debug.Log("Get(%v) send range %v", h, byteRange)
<-b.connChan
resp, err := b.client.Do(req)
b.connChan <- struct{}{}
if err != nil {
if resp != nil {
io.Copy(ioutil.Discard, resp.Body)
resp.Body.Close()
}
return nil, errors.Wrap(err, "client.Do")
}
if resp.StatusCode != 200 && resp.StatusCode != 206 {
io.Copy(ioutil.Discard, resp.Body)
resp.Body.Close()
return nil, errors.Errorf("unexpected HTTP response code %v", resp.StatusCode)
}
return resp.Body, nil
}
// Stat returns information about a blob. // Stat returns information about a blob.
func (b *restBackend) Stat(h restic.Handle) (restic.FileInfo, error) { func (b *restBackend) Stat(h restic.Handle) (restic.FileInfo, error) {
if err := h.Valid(); err != nil { if err := h.Valid(); err != nil {

View File

@ -58,6 +58,13 @@ func TestS3BackendLoadNegativeOffset(t *testing.T) {
test.TestLoadNegativeOffset(t) test.TestLoadNegativeOffset(t)
} }
func TestS3BackendGet(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestGet(t)
}
func TestS3BackendSave(t *testing.T) { func TestS3BackendSave(t *testing.T) {
if SkipMessage != "" { if SkipMessage != "" {
t.Skip(SkipMessage) t.Skip(SkipMessage)

View File

@ -1,11 +1,13 @@
package s3 package s3
import ( import (
"bytes"
"io" "io"
"path" "path"
"restic" "restic"
"strings" "strings"
"restic/backend"
"restic/errors" "restic/errors"
"github.com/minio/minio-go" "github.com/minio/minio-go"
@ -74,7 +76,7 @@ func (be *s3) Location() string {
// Load returns the data stored in the backend for h at the given offset // Load returns the data stored in the backend for h at the given offset
// and saves it in p. Load has the same semantics as io.ReaderAt. // and saves it in p. Load has the same semantics as io.ReaderAt.
func (be s3) Load(h restic.Handle, p []byte, off int64) (n int, err error) { func (be *s3) Load(h restic.Handle, p []byte, off int64) (n int, err error) {
var obj *minio.Object var obj *minio.Object
debug.Log("%v, offset %v, len %v", h, off, len(p)) debug.Log("%v, offset %v, len %v", h, off, len(p))
@ -146,7 +148,7 @@ func (be s3) Load(h restic.Handle, p []byte, off int64) (n int, err error) {
} }
// Save stores data in the backend at the handle. // Save stores data in the backend at the handle.
func (be s3) Save(h restic.Handle, rd io.Reader) (err error) { func (be *s3) Save(h restic.Handle, rd io.Reader) (err error) {
if err := h.Valid(); err != nil { if err := h.Valid(); err != nil {
return err return err
} }
@ -175,8 +177,82 @@ func (be s3) Save(h restic.Handle, rd io.Reader) (err error) {
return errors.Wrap(err, "client.PutObject") return errors.Wrap(err, "client.PutObject")
} }
// Get returns a reader that yields the contents of the file at h at the
// given offset. If length is nonzero, only a portion of the file is
// returned. rd must be closed after use.
func (be *s3) Get(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
debug.Log("Get %v, length %v, offset %v", h, length, offset)
if err := h.Valid(); err != nil {
return nil, err
}
if offset < 0 {
return nil, errors.New("offset is negative")
}
if length < 0 {
return nil, errors.Errorf("invalid length %d", length)
}
var obj *minio.Object
objName := be.s3path(h.Type, h.Name)
<-be.connChan
defer func() {
be.connChan <- struct{}{}
}()
obj, err := be.client.GetObject(be.bucketname, objName)
if err != nil {
debug.Log(" err %v", err)
return nil, errors.Wrap(err, "client.GetObject")
}
// if we're going to read the whole object, just pass it on.
if length == 0 {
debug.Log("Get %v: pass on object", h)
_, err = obj.Seek(offset, 0)
if err != nil {
return nil, errors.Wrap(err, "obj.Seek")
}
return obj, nil
}
// otherwise use a buffer with ReadAt
info, err := obj.Stat()
if err != nil {
return nil, errors.Wrap(err, "obj.Stat")
}
if offset > info.Size {
return nil, errors.Errorf("offset larger than file size")
}
l := int64(length)
if offset+l > info.Size {
l = info.Size - offset
}
buf := make([]byte, l)
n, err := obj.ReadAt(buf, offset)
debug.Log("Get %v: use buffer with ReadAt: %v, %v", h, n, err)
if err == io.EOF {
debug.Log("Get %v: shorten buffer %v -> %v", h, len(buf), n)
buf = buf[:n]
err = nil
}
if err != nil {
return nil, errors.Wrap(err, "obj.ReadAt")
}
return backend.Closer{Reader: bytes.NewReader(buf)}, nil
}
// Stat returns information about a blob. // Stat returns information about a blob.
func (be s3) Stat(h restic.Handle) (bi restic.FileInfo, err error) { func (be *s3) Stat(h restic.Handle) (bi restic.FileInfo, err error) {
debug.Log("%v", h) debug.Log("%v", h)
objName := be.s3path(h.Type, h.Name) objName := be.s3path(h.Type, h.Name)

View File

@ -58,6 +58,13 @@ func TestSftpBackendLoadNegativeOffset(t *testing.T) {
test.TestLoadNegativeOffset(t) test.TestLoadNegativeOffset(t)
} }
func TestSftpBackendGet(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestGet(t)
}
func TestSftpBackendSave(t *testing.T) { func TestSftpBackendSave(t *testing.T) {
if SkipMessage != "" { if SkipMessage != "" {
t.Skip(SkipMessage) t.Skip(SkipMessage)

View File

@ -398,6 +398,39 @@ func (r *SFTP) Save(h restic.Handle, rd io.Reader) (err error) {
return err return err
} }
// Get returns a reader that yields the contents of the file at h at the
// given offset. If length is nonzero, only a portion of the file is
// returned. rd must be closed after use.
func (r *SFTP) Get(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
debug.Log("Get %v, length %v, offset %v", h, length, offset)
if err := h.Valid(); err != nil {
return nil, err
}
if offset < 0 {
return nil, errors.New("offset is negative")
}
f, err := r.c.Open(r.filename(h.Type, h.Name))
if err != nil {
return nil, err
}
if offset > 0 {
_, err = f.Seek(offset, 0)
if err != nil {
f.Close()
return nil, err
}
}
if length > 0 {
return backend.LimitReadCloser(f, int64(length)), nil
}
return f, nil
}
// Stat returns information about a blob. // Stat returns information about a blob.
func (r *SFTP) Stat(h restic.Handle) (restic.FileInfo, error) { func (r *SFTP) Stat(h restic.Handle) (restic.FileInfo, error) {
debug.Log("stat %v", h) debug.Log("stat %v", h)

View File

@ -58,6 +58,13 @@ func TestTestBackendLoadNegativeOffset(t *testing.T) {
test.TestLoadNegativeOffset(t) test.TestLoadNegativeOffset(t)
} }
func TestTestBackendGet(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestGet(t)
}
func TestTestBackendSave(t *testing.T) { func TestTestBackendSave(t *testing.T) {
if SkipMessage != "" { if SkipMessage != "" {
t.Skip(SkipMessage) t.Skip(SkipMessage)

View File

@ -4,6 +4,7 @@ import (
"bytes" "bytes"
"fmt" "fmt"
"io" "io"
"io/ioutil"
"math/rand" "math/rand"
"reflect" "reflect"
"restic" "restic"
@ -369,6 +370,99 @@ func TestLoadNegativeOffset(t testing.TB) {
test.OK(t, b.Remove(restic.DataFile, id.String())) test.OK(t, b.Remove(restic.DataFile, id.String()))
} }
// TestGet tests the backend's Get function.
func TestGet(t testing.TB) {
b := open(t)
defer close(t)
_, err := b.Get(restic.Handle{}, 0, 0)
if err == nil {
t.Fatalf("Get() did not return an error for invalid handle")
}
_, err = b.Get(restic.Handle{Type: restic.DataFile, Name: "foobar"}, 0, 0)
if err == nil {
t.Fatalf("Get() did not return an error for non-existing blob")
}
length := rand.Intn(1<<24) + 2000
data := test.Random(23, length)
id := restic.Hash(data)
handle := restic.Handle{Type: restic.DataFile, Name: id.String()}
err = b.Save(handle, bytes.NewReader(data))
if err != nil {
t.Fatalf("Save() error: %v", err)
}
rd, err := b.Get(handle, 100, -1)
if err == nil {
t.Fatalf("Get() returned no error for negative offset!")
}
if rd != nil {
t.Fatalf("Get() returned a non-nil reader for negative offset!")
}
for i := 0; i < 50; i++ {
l := rand.Intn(length + 2000)
o := rand.Intn(length + 2000)
d := data
if o < len(d) {
d = d[o:]
} else {
o = len(d)
d = d[:0]
}
getlen := l
if l >= len(d) && rand.Float32() >= 0.5 {
getlen = 0
}
if l > 0 && l < len(d) {
d = d[:l]
}
rd, err := b.Get(handle, getlen, int64(o))
if err != nil {
t.Errorf("Get(%d, %d) returned unexpected error: %v", l, o, err)
continue
}
buf, err := ioutil.ReadAll(rd)
if err != nil {
t.Errorf("Get(%d, %d) ReadAll() returned unexpected error: %v", l, o, err)
continue
}
if l <= len(d) && len(buf) != l {
t.Errorf("Get(%d, %d) wrong number of bytes read: want %d, got %d", l, o, l, len(buf))
continue
}
if l > len(d) && len(buf) != len(d) {
t.Errorf("Get(%d, %d) wrong number of bytes read for overlong read: want %d, got %d", l, o, l, len(buf))
continue
}
if !bytes.Equal(buf, d) {
t.Errorf("Get(%d, %d) returned wrong bytes", l, o)
continue
}
err = rd.Close()
if err != nil {
t.Errorf("Get(%d, %d) rd.Close() returned unexpected error: %v", l, o, err)
continue
}
}
test.OK(t, b.Remove(restic.DataFile, id.String()))
}
// TestSave tests saving data in the backend. // TestSave tests saving data in the backend.
func TestSave(t testing.TB) { func TestSave(t testing.TB) {
b := open(t) b := open(t)

View File

@ -28,3 +28,30 @@ func LoadAll(be restic.Backend, h restic.Handle, buf []byte) ([]byte, error) {
buf = buf[:n] buf = buf[:n]
return buf, err return buf, err
} }
// Closer wraps an io.Reader and adds a Close() method that does nothing.
type Closer struct {
io.Reader
}
// Close is a no-op.
func (c Closer) Close() error {
return nil
}
// LimitedReadCloser wraps io.LimitedReader and exposes the Close() method.
type LimitedReadCloser struct {
io.ReadCloser
io.Reader
}
// Read reads data from the limited reader.
func (l *LimitedReadCloser) Read(p []byte) (int, error) {
return l.Reader.Read(p)
}
// LimitReadCloser returns a new reader wraps r in an io.LimitReader, but also
// exposes the Close() method.
func LimitReadCloser(r io.ReadCloser, n int64) *LimitedReadCloser {
return &LimitedReadCloser{ReadCloser: r, Reader: io.LimitReader(r, n)}
}

View File

@ -12,6 +12,7 @@ type Backend struct {
CloseFn func() error CloseFn func() error
LoadFn func(h restic.Handle, p []byte, off int64) (int, error) LoadFn func(h restic.Handle, p []byte, off int64) (int, error)
SaveFn func(h restic.Handle, rd io.Reader) error SaveFn func(h restic.Handle, rd io.Reader) error
GetFn func(h restic.Handle, length int, offset int64) (io.ReadCloser, error)
StatFn func(h restic.Handle) (restic.FileInfo, error) StatFn func(h restic.Handle) (restic.FileInfo, error)
ListFn func(restic.FileType, <-chan struct{}) <-chan string ListFn func(restic.FileType, <-chan struct{}) <-chan string
RemoveFn func(restic.FileType, string) error RemoveFn func(restic.FileType, string) error
@ -56,6 +57,15 @@ func (m *Backend) Save(h restic.Handle, rd io.Reader) error {
return m.SaveFn(h, rd) return m.SaveFn(h, rd)
} }
// Get loads data from the backend.
func (m *Backend) Get(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
if m.GetFn == nil {
return nil, errors.New("not implemented")
}
return m.GetFn(h, length, offset)
}
// Stat an object in the backend. // Stat an object in the backend.
func (m *Backend) Stat(h restic.Handle) (restic.FileInfo, error) { func (m *Backend) Stat(h restic.Handle) (restic.FileInfo, error) {
if m.StatFn == nil { if m.StatFn == nil {