octoleo
/
restic
mirror of https://github.com/octoleo/restic.git
2
2
Fork 0
Code Issues Releases Wiki Activity

Add backend.Get()

This commit is contained in:
Alexander Neumann 2017-01-22 22:01:12 +01:00
parent a36c01372d
commit 05afedd950
15 changed files with 420 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/restic/backend.go
View File

@ -26,6 +26,11 @@ type Backend interface {
// Save stores the data in the backend under the given handle.
Save(h Handle, rd io.Reader) error
// Get returns a reader that yields the contents of the file at h at the
// given offset. If length is nonzero, only a portion of the file is
// returned. rd must be closed after use.
Get(h Handle, length int, offset int64) (io.ReadCloser, error)
// Stat returns information about the File identified by h.
Stat(h Handle) (FileInfo, error)

7
src/restic/backend/local/backend_test.go
View File

@ -58,6 +58,13 @@ func TestLocalBackendLoadNegativeOffset(t *testing.T) {
test.TestLoadNegativeOffset(t)
}
func TestLocalBackendGet(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestGet(t)
}
func TestLocalBackendSave(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)

33
src/restic/backend/local/local.go
View File

@ -206,6 +206,39 @@ func (b *Local) Save(h restic.Handle, rd io.Reader) (err error) {
return setNewFileMode(filename, fi)
}
// Get returns a reader that yields the contents of the file at h at the
// given offset. If length is nonzero, only a portion of the file is
// returned. rd must be closed after use.
func (b *Local) Get(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
debug.Log("Get %v, length %v, offset %v", h, length, offset)
if err := h.Valid(); err != nil {
return nil, err
}
if offset < 0 {
return nil, errors.New("offset is negative")
}
f, err := os.Open(filename(b.p, h.Type, h.Name))
if err != nil {
return nil, err
}
if offset > 0 {
_, err = f.Seek(offset, 0)
if err != nil {
f.Close()
return nil, err
}
}
if length > 0 {
return backend.LimitReadCloser(f, int64(length)), nil
}
return f, nil
}
// Stat returns information about a blob.
func (b *Local) Stat(h restic.Handle) (restic.FileInfo, error) {
debug.Log("Stat %v", h)

7
src/restic/backend/mem/backend_test.go
View File

@ -58,6 +58,13 @@ func TestMemBackendLoadNegativeOffset(t *testing.T) {
test.TestLoadNegativeOffset(t)
}
func TestMemBackendGet(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestGet(t)
}
func TestMemBackendSave(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)

40
src/restic/backend/mem/mem_backend.go
View File

@ -1,11 +1,13 @@
package mem
import (
"bytes"
"io"
"io/ioutil"
"restic"
"sync"
"restic/backend"
"restic/errors"
"restic/debug"
@ -121,6 +123,44 @@ func (be *MemoryBackend) Save(h restic.Handle, rd io.Reader) error {
return nil
}
// Get returns a reader that yields the contents of the file at h at the
// given offset. If length is nonzero, only a portion of the file is
// returned. rd must be closed after use.
func (be *MemoryBackend) Get(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
if err := h.Valid(); err != nil {
return nil, err
}
be.m.Lock()
defer be.m.Unlock()
if h.Type == restic.ConfigFile {
h.Name = ""
}
debug.Log("Get %v offset %v len %v", h, offset, length)
if offset < 0 {
return nil, errors.New("offset is negative")
}
if _, ok := be.data[entry{h.Type, h.Name}]; !ok {
return nil, errors.New("no such data")
}
buf := be.data[entry{h.Type, h.Name}]
if offset > int64(len(buf)) {
return nil, errors.New("offset beyond end of file")
}
buf = buf[offset:]
if length > 0 && len(buf) > length {
buf = buf[:length]
}
return backend.Closer{bytes.NewReader(buf)}, nil
}
// Stat returns information about a file in the backend.
func (be *MemoryBackend) Stat(h restic.Handle) (restic.FileInfo, error) {
be.m.Lock()

7
src/restic/backend/rest/backend_test.go
View File

@ -58,6 +58,13 @@ func TestRestBackendLoadNegativeOffset(t *testing.T) {
test.TestLoadNegativeOffset(t)
}
func TestRestBackendGet(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestGet(t)
}
func TestRestBackendSave(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)

57
src/restic/backend/rest/rest.go
View File

@ -11,6 +11,7 @@ import (
"restic"
"strings"
"restic/debug"
"restic/errors"
"restic/backend"
@ -76,10 +77,15 @@ func (b *restBackend) Location() string {
// Load returns the data stored in the backend for h at the given offset
// and saves it in p. Load has the same semantics as io.ReaderAt.
func (b *restBackend) Load(h restic.Handle, p []byte, off int64) (n int, err error) {
debug.Log("Load(%v, length %v, offset %v)", h, len(p), off)
if err := h.Valid(); err != nil {
return 0, err
}
if len(p) == 0 {
return 0, errors.New("buffer length is zero")
}
// invert offset
if off < 0 {
info, err := b.Stat(h)
@ -98,6 +104,7 @@ func (b *restBackend) Load(h restic.Handle, p []byte, off int64) (n int, err err
if err != nil {
return 0, errors.Wrap(err, "http.NewRequest")
}
debug.Log("Load(%v) send range %d-%d", h, off, off+int64(len(p)-1))
req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", off, off+int64(len(p))))
<-b.connChan
resp, err := b.client.Do(req)
@ -156,6 +163,56 @@ func (b *restBackend) Save(h restic.Handle, rd io.Reader) (err error) {
return nil
}
// Get returns a reader that yields the contents of the file at h at the
// given offset. If length is nonzero, only a portion of the file is
// returned. rd must be closed after use.
func (b *restBackend) Get(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
debug.Log("Get %v, length %v, offset %v", h, length, offset)
if err := h.Valid(); err != nil {
return nil, err
}
if offset < 0 {
return nil, errors.New("offset is negative")
}
if length < 0 {
return nil, errors.Errorf("invalid length %d", length)
}
req, err := http.NewRequest("GET", restPath(b.url, h), nil)
if err != nil {
return nil, errors.Wrap(err, "http.NewRequest")
}
byteRange := fmt.Sprintf("bytes=%d-", offset)
if length > 0 {
byteRange = fmt.Sprintf("bytes=%d-%d", offset, offset+int64(length)-1)
}
req.Header.Add("Range", byteRange)
debug.Log("Get(%v) send range %v", h, byteRange)
<-b.connChan
resp, err := b.client.Do(req)
b.connChan <- struct{}{}
if err != nil {
if resp != nil {
io.Copy(ioutil.Discard, resp.Body)
resp.Body.Close()
}
return nil, errors.Wrap(err, "client.Do")
}
if resp.StatusCode != 200 && resp.StatusCode != 206 {
io.Copy(ioutil.Discard, resp.Body)
resp.Body.Close()
return nil, errors.Errorf("unexpected HTTP response code %v", resp.StatusCode)
}
return resp.Body, nil
}
// Stat returns information about a blob.
func (b *restBackend) Stat(h restic.Handle) (restic.FileInfo, error) {
if err := h.Valid(); err != nil {

7
src/restic/backend/s3/backend_test.go
View File

@ -58,6 +58,13 @@ func TestS3BackendLoadNegativeOffset(t *testing.T) {
test.TestLoadNegativeOffset(t)
}
func TestS3BackendGet(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestGet(t)
}
func TestS3BackendSave(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)

82
src/restic/backend/s3/s3.go
View File

@ -1,11 +1,13 @@
package s3
import (
"bytes"
"io"
"path"
"restic"
"strings"
"restic/backend"
"restic/errors"
"github.com/minio/minio-go"
@ -74,7 +76,7 @@ func (be *s3) Location() string {
// Load returns the data stored in the backend for h at the given offset
// and saves it in p. Load has the same semantics as io.ReaderAt.
func (be s3) Load(h restic.Handle, p []byte, off int64) (n int, err error) {
func (be *s3) Load(h restic.Handle, p []byte, off int64) (n int, err error) {
var obj *minio.Object
debug.Log("%v, offset %v, len %v", h, off, len(p))
@ -146,7 +148,7 @@ func (be s3) Load(h restic.Handle, p []byte, off int64) (n int, err error) {
}
// Save stores data in the backend at the handle.
func (be s3) Save(h restic.Handle, rd io.Reader) (err error) {
func (be *s3) Save(h restic.Handle, rd io.Reader) (err error) {
if err := h.Valid(); err != nil {
return err
}
@ -175,8 +177,82 @@ func (be s3) Save(h restic.Handle, rd io.Reader) (err error) {
return errors.Wrap(err, "client.PutObject")
}
// Get returns a reader that yields the contents of the file at h at the
// given offset. If length is nonzero, only a portion of the file is
// returned. rd must be closed after use.
func (be *s3) Get(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
debug.Log("Get %v, length %v, offset %v", h, length, offset)
if err := h.Valid(); err != nil {
return nil, err
}
if offset < 0 {
return nil, errors.New("offset is negative")
}
if length < 0 {
return nil, errors.Errorf("invalid length %d", length)
}
var obj *minio.Object
objName := be.s3path(h.Type, h.Name)
<-be.connChan
defer func() {
be.connChan <- struct{}{}
}()
obj, err := be.client.GetObject(be.bucketname, objName)
if err != nil {
debug.Log(" err %v", err)
return nil, errors.Wrap(err, "client.GetObject")
}
// if we're going to read the whole object, just pass it on.
if length == 0 {
debug.Log("Get %v: pass on object", h)
_, err = obj.Seek(offset, 0)
if err != nil {
return nil, errors.Wrap(err, "obj.Seek")
}
return obj, nil
}
// otherwise use a buffer with ReadAt
info, err := obj.Stat()
if err != nil {
return nil, errors.Wrap(err, "obj.Stat")
}
if offset > info.Size {
return nil, errors.Errorf("offset larger than file size")
}
l := int64(length)
if offset+l > info.Size {
l = info.Size - offset
}
buf := make([]byte, l)
n, err := obj.ReadAt(buf, offset)
debug.Log("Get %v: use buffer with ReadAt: %v, %v", h, n, err)
if err == io.EOF {
debug.Log("Get %v: shorten buffer %v -> %v", h, len(buf), n)
buf = buf[:n]
err = nil
}
if err != nil {
return nil, errors.Wrap(err, "obj.ReadAt")
}
return backend.Closer{Reader: bytes.NewReader(buf)}, nil
}
// Stat returns information about a blob.
func (be s3) Stat(h restic.Handle) (bi restic.FileInfo, err error) {
func (be *s3) Stat(h restic.Handle) (bi restic.FileInfo, err error) {
debug.Log("%v", h)
objName := be.s3path(h.Type, h.Name)

7
src/restic/backend/sftp/backend_test.go
View File

@ -58,6 +58,13 @@ func TestSftpBackendLoadNegativeOffset(t *testing.T) {
test.TestLoadNegativeOffset(t)
}
func TestSftpBackendGet(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestGet(t)
}
func TestSftpBackendSave(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)

33
src/restic/backend/sftp/sftp.go
View File

@ -398,6 +398,39 @@ func (r *SFTP) Save(h restic.Handle, rd io.Reader) (err error) {
return err
}
// Get returns a reader that yields the contents of the file at h at the
// given offset. If length is nonzero, only a portion of the file is
// returned. rd must be closed after use.
func (r *SFTP) Get(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
debug.Log("Get %v, length %v, offset %v", h, length, offset)
if err := h.Valid(); err != nil {
return nil, err
}
if offset < 0 {
return nil, errors.New("offset is negative")
}
f, err := r.c.Open(r.filename(h.Type, h.Name))
if err != nil {
return nil, err
}
if offset > 0 {
_, err = f.Seek(offset, 0)
if err != nil {
f.Close()
return nil, err
}
}
if length > 0 {
return backend.LimitReadCloser(f, int64(length)), nil
}
return f, nil
}
// Stat returns information about a blob.
func (r *SFTP) Stat(h restic.Handle) (restic.FileInfo, error) {
debug.Log("stat %v", h)

7
src/restic/backend/test/backend_test.go
View File

@ -58,6 +58,13 @@ func TestTestBackendLoadNegativeOffset(t *testing.T) {
test.TestLoadNegativeOffset(t)
}
func TestTestBackendGet(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestGet(t)
}
func TestTestBackendSave(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)

94
src/restic/backend/test/tests.go
View File

@ -4,6 +4,7 @@ import (
"bytes"
"fmt"
"io"
"io/ioutil"
"math/rand"
"reflect"
"restic"
@ -369,6 +370,99 @@ func TestLoadNegativeOffset(t testing.TB) {
test.OK(t, b.Remove(restic.DataFile, id.String()))
}
// TestGet tests the backend's Get function.
func TestGet(t testing.TB) {
b := open(t)
defer close(t)
_, err := b.Get(restic.Handle{}, 0, 0)
if err == nil {
t.Fatalf("Get() did not return an error for invalid handle")
}
_, err = b.Get(restic.Handle{Type: restic.DataFile, Name: "foobar"}, 0, 0)
if err == nil {
t.Fatalf("Get() did not return an error for non-existing blob")
}
length := rand.Intn(1<<24) + 2000
data := test.Random(23, length)
id := restic.Hash(data)
handle := restic.Handle{Type: restic.DataFile, Name: id.String()}
err = b.Save(handle, bytes.NewReader(data))
if err != nil {
t.Fatalf("Save() error: %v", err)
}
rd, err := b.Get(handle, 100, -1)
if err == nil {
t.Fatalf("Get() returned no error for negative offset!")
}
if rd != nil {
t.Fatalf("Get() returned a non-nil reader for negative offset!")
}
for i := 0; i < 50; i++ {
l := rand.Intn(length + 2000)
o := rand.Intn(length + 2000)
d := data
if o < len(d) {
d = d[o:]
} else {
o = len(d)
d = d[:0]
}
getlen := l
if l >= len(d) && rand.Float32() >= 0.5 {
getlen = 0
}
if l > 0 && l < len(d) {
d = d[:l]
}
rd, err := b.Get(handle, getlen, int64(o))
if err != nil {
t.Errorf("Get(%d, %d) returned unexpected error: %v", l, o, err)
continue
}
buf, err := ioutil.ReadAll(rd)
if err != nil {
t.Errorf("Get(%d, %d) ReadAll() returned unexpected error: %v", l, o, err)
continue
}
if l <= len(d) && len(buf) != l {
t.Errorf("Get(%d, %d) wrong number of bytes read: want %d, got %d", l, o, l, len(buf))
continue
}
if l > len(d) && len(buf) != len(d) {
t.Errorf("Get(%d, %d) wrong number of bytes read for overlong read: want %d, got %d", l, o, l, len(buf))
continue
}
if !bytes.Equal(buf, d) {
t.Errorf("Get(%d, %d) returned wrong bytes", l, o)
continue
}
err = rd.Close()
if err != nil {
t.Errorf("Get(%d, %d) rd.Close() returned unexpected error: %v", l, o, err)
continue
}
}
test.OK(t, b.Remove(restic.DataFile, id.String()))
}
// TestSave tests saving data in the backend.
func TestSave(t testing.TB) {
b := open(t)

27
src/restic/backend/utils.go
View File

@ -28,3 +28,30 @@ func LoadAll(be restic.Backend, h restic.Handle, buf []byte) ([]byte, error) {
buf = buf[:n]
return buf, err
}
// Closer wraps an io.Reader and adds a Close() method that does nothing.
type Closer struct {
io.Reader
}
// Close is a no-op.
func (c Closer) Close() error {
return nil
}
// LimitedReadCloser wraps io.LimitedReader and exposes the Close() method.
type LimitedReadCloser struct {
io.ReadCloser
io.Reader
}
// Read reads data from the limited reader.
func (l *LimitedReadCloser) Read(p []byte) (int, error) {
return l.Reader.Read(p)
}
// LimitReadCloser returns a new reader wraps r in an io.LimitReader, but also
// exposes the Close() method.
func LimitReadCloser(r io.ReadCloser, n int64) *LimitedReadCloser {
return &LimitedReadCloser{ReadCloser: r, Reader: io.LimitReader(r, n)}
}

10
src/restic/mock/backend.go
View File

@ -12,6 +12,7 @@ type Backend struct {
CloseFn func() error
LoadFn func(h restic.Handle, p []byte, off int64) (int, error)
SaveFn func(h restic.Handle, rd io.Reader) error
GetFn func(h restic.Handle, length int, offset int64) (io.ReadCloser, error)
StatFn func(h restic.Handle) (restic.FileInfo, error)
ListFn func(restic.FileType, <-chan struct{}) <-chan string
RemoveFn func(restic.FileType, string) error
@ -56,6 +57,15 @@ func (m *Backend) Save(h restic.Handle, rd io.Reader) error {
return m.SaveFn(h, rd)
}
// Get loads data from the backend.
func (m *Backend) Get(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
if m.GetFn == nil {
return nil, errors.New("not implemented")
}
return m.GetFn(h, length, offset)
}
// Stat an object in the backend.
func (m *Backend) Stat(h restic.Handle) (restic.FileInfo, error) {
if m.StatFn == nil {