diff --git a/src/restic/backend.go b/src/restic/backend.go index 304c02152..9e0a4b47c 100644 --- a/src/restic/backend.go +++ b/src/restic/backend.go @@ -26,6 +26,11 @@ type Backend interface { // Save stores the data in the backend under the given handle. Save(h Handle, rd io.Reader) error + // Get returns a reader that yields the contents of the file at h at the + // given offset. If length is nonzero, only a portion of the file is + // returned. rd must be closed after use. + Get(h Handle, length int, offset int64) (io.ReadCloser, error) + // Stat returns information about the File identified by h. Stat(h Handle) (FileInfo, error) diff --git a/src/restic/backend/local/backend_test.go b/src/restic/backend/local/backend_test.go index 8954dc83d..c018b4947 100644 --- a/src/restic/backend/local/backend_test.go +++ b/src/restic/backend/local/backend_test.go @@ -58,6 +58,13 @@ func TestLocalBackendLoadNegativeOffset(t *testing.T) { test.TestLoadNegativeOffset(t) } +func TestLocalBackendGet(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestGet(t) +} + func TestLocalBackendSave(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) diff --git a/src/restic/backend/local/local.go b/src/restic/backend/local/local.go index 2f3cabad0..b50587642 100644 --- a/src/restic/backend/local/local.go +++ b/src/restic/backend/local/local.go @@ -206,6 +206,39 @@ func (b *Local) Save(h restic.Handle, rd io.Reader) (err error) { return setNewFileMode(filename, fi) } +// Get returns a reader that yields the contents of the file at h at the +// given offset. If length is nonzero, only a portion of the file is +// returned. rd must be closed after use. +func (b *Local) Get(h restic.Handle, length int, offset int64) (io.ReadCloser, error) { + debug.Log("Get %v, length %v, offset %v", h, length, offset) + if err := h.Valid(); err != nil { + return nil, err + } + + if offset < 0 { + return nil, errors.New("offset is negative") + } + + f, err := os.Open(filename(b.p, h.Type, h.Name)) + if err != nil { + return nil, err + } + + if offset > 0 { + _, err = f.Seek(offset, 0) + if err != nil { + f.Close() + return nil, err + } + } + + if length > 0 { + return backend.LimitReadCloser(f, int64(length)), nil + } + + return f, nil +} + // Stat returns information about a blob. func (b *Local) Stat(h restic.Handle) (restic.FileInfo, error) { debug.Log("Stat %v", h) diff --git a/src/restic/backend/mem/backend_test.go b/src/restic/backend/mem/backend_test.go index 6bf19580f..09857b64d 100644 --- a/src/restic/backend/mem/backend_test.go +++ b/src/restic/backend/mem/backend_test.go @@ -58,6 +58,13 @@ func TestMemBackendLoadNegativeOffset(t *testing.T) { test.TestLoadNegativeOffset(t) } +func TestMemBackendGet(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestGet(t) +} + func TestMemBackendSave(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) diff --git a/src/restic/backend/mem/mem_backend.go b/src/restic/backend/mem/mem_backend.go index ab827d3d9..791753755 100644 --- a/src/restic/backend/mem/mem_backend.go +++ b/src/restic/backend/mem/mem_backend.go @@ -1,11 +1,13 @@ package mem import ( + "bytes" "io" "io/ioutil" "restic" "sync" + "restic/backend" "restic/errors" "restic/debug" @@ -121,6 +123,44 @@ func (be *MemoryBackend) Save(h restic.Handle, rd io.Reader) error { return nil } +// Get returns a reader that yields the contents of the file at h at the +// given offset. If length is nonzero, only a portion of the file is +// returned. rd must be closed after use. +func (be *MemoryBackend) Get(h restic.Handle, length int, offset int64) (io.ReadCloser, error) { + if err := h.Valid(); err != nil { + return nil, err + } + + be.m.Lock() + defer be.m.Unlock() + + if h.Type == restic.ConfigFile { + h.Name = "" + } + + debug.Log("Get %v offset %v len %v", h, offset, length) + + if offset < 0 { + return nil, errors.New("offset is negative") + } + + if _, ok := be.data[entry{h.Type, h.Name}]; !ok { + return nil, errors.New("no such data") + } + + buf := be.data[entry{h.Type, h.Name}] + if offset > int64(len(buf)) { + return nil, errors.New("offset beyond end of file") + } + + buf = buf[offset:] + if length > 0 && len(buf) > length { + buf = buf[:length] + } + + return backend.Closer{bytes.NewReader(buf)}, nil +} + // Stat returns information about a file in the backend. func (be *MemoryBackend) Stat(h restic.Handle) (restic.FileInfo, error) { be.m.Lock() diff --git a/src/restic/backend/rest/backend_test.go b/src/restic/backend/rest/backend_test.go index 9605396d6..a9beec25f 100644 --- a/src/restic/backend/rest/backend_test.go +++ b/src/restic/backend/rest/backend_test.go @@ -58,6 +58,13 @@ func TestRestBackendLoadNegativeOffset(t *testing.T) { test.TestLoadNegativeOffset(t) } +func TestRestBackendGet(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestGet(t) +} + func TestRestBackendSave(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) diff --git a/src/restic/backend/rest/rest.go b/src/restic/backend/rest/rest.go index f66cb634b..b521b4938 100644 --- a/src/restic/backend/rest/rest.go +++ b/src/restic/backend/rest/rest.go @@ -11,6 +11,7 @@ import ( "restic" "strings" + "restic/debug" "restic/errors" "restic/backend" @@ -76,10 +77,15 @@ func (b *restBackend) Location() string { // Load returns the data stored in the backend for h at the given offset // and saves it in p. Load has the same semantics as io.ReaderAt. func (b *restBackend) Load(h restic.Handle, p []byte, off int64) (n int, err error) { + debug.Log("Load(%v, length %v, offset %v)", h, len(p), off) if err := h.Valid(); err != nil { return 0, err } + if len(p) == 0 { + return 0, errors.New("buffer length is zero") + } + // invert offset if off < 0 { info, err := b.Stat(h) @@ -98,6 +104,7 @@ func (b *restBackend) Load(h restic.Handle, p []byte, off int64) (n int, err err if err != nil { return 0, errors.Wrap(err, "http.NewRequest") } + debug.Log("Load(%v) send range %d-%d", h, off, off+int64(len(p)-1)) req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", off, off+int64(len(p)))) <-b.connChan resp, err := b.client.Do(req) @@ -156,6 +163,56 @@ func (b *restBackend) Save(h restic.Handle, rd io.Reader) (err error) { return nil } +// Get returns a reader that yields the contents of the file at h at the +// given offset. If length is nonzero, only a portion of the file is +// returned. rd must be closed after use. +func (b *restBackend) Get(h restic.Handle, length int, offset int64) (io.ReadCloser, error) { + debug.Log("Get %v, length %v, offset %v", h, length, offset) + if err := h.Valid(); err != nil { + return nil, err + } + + if offset < 0 { + return nil, errors.New("offset is negative") + } + + if length < 0 { + return nil, errors.Errorf("invalid length %d", length) + } + + req, err := http.NewRequest("GET", restPath(b.url, h), nil) + if err != nil { + return nil, errors.Wrap(err, "http.NewRequest") + } + + byteRange := fmt.Sprintf("bytes=%d-", offset) + if length > 0 { + byteRange = fmt.Sprintf("bytes=%d-%d", offset, offset+int64(length)-1) + } + req.Header.Add("Range", byteRange) + debug.Log("Get(%v) send range %v", h, byteRange) + + <-b.connChan + resp, err := b.client.Do(req) + b.connChan <- struct{}{} + + if err != nil { + if resp != nil { + io.Copy(ioutil.Discard, resp.Body) + resp.Body.Close() + } + return nil, errors.Wrap(err, "client.Do") + } + + if resp.StatusCode != 200 && resp.StatusCode != 206 { + io.Copy(ioutil.Discard, resp.Body) + resp.Body.Close() + return nil, errors.Errorf("unexpected HTTP response code %v", resp.StatusCode) + } + + return resp.Body, nil +} + // Stat returns information about a blob. func (b *restBackend) Stat(h restic.Handle) (restic.FileInfo, error) { if err := h.Valid(); err != nil { diff --git a/src/restic/backend/s3/backend_test.go b/src/restic/backend/s3/backend_test.go index 9fb4dd3fa..c4709d279 100644 --- a/src/restic/backend/s3/backend_test.go +++ b/src/restic/backend/s3/backend_test.go @@ -58,6 +58,13 @@ func TestS3BackendLoadNegativeOffset(t *testing.T) { test.TestLoadNegativeOffset(t) } +func TestS3BackendGet(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestGet(t) +} + func TestS3BackendSave(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) diff --git a/src/restic/backend/s3/s3.go b/src/restic/backend/s3/s3.go index 2ba688142..793750009 100644 --- a/src/restic/backend/s3/s3.go +++ b/src/restic/backend/s3/s3.go @@ -1,11 +1,13 @@ package s3 import ( + "bytes" "io" "path" "restic" "strings" + "restic/backend" "restic/errors" "github.com/minio/minio-go" @@ -74,7 +76,7 @@ func (be *s3) Location() string { // Load returns the data stored in the backend for h at the given offset // and saves it in p. Load has the same semantics as io.ReaderAt. -func (be s3) Load(h restic.Handle, p []byte, off int64) (n int, err error) { +func (be *s3) Load(h restic.Handle, p []byte, off int64) (n int, err error) { var obj *minio.Object debug.Log("%v, offset %v, len %v", h, off, len(p)) @@ -146,7 +148,7 @@ func (be s3) Load(h restic.Handle, p []byte, off int64) (n int, err error) { } // Save stores data in the backend at the handle. -func (be s3) Save(h restic.Handle, rd io.Reader) (err error) { +func (be *s3) Save(h restic.Handle, rd io.Reader) (err error) { if err := h.Valid(); err != nil { return err } @@ -175,8 +177,82 @@ func (be s3) Save(h restic.Handle, rd io.Reader) (err error) { return errors.Wrap(err, "client.PutObject") } +// Get returns a reader that yields the contents of the file at h at the +// given offset. If length is nonzero, only a portion of the file is +// returned. rd must be closed after use. +func (be *s3) Get(h restic.Handle, length int, offset int64) (io.ReadCloser, error) { + debug.Log("Get %v, length %v, offset %v", h, length, offset) + if err := h.Valid(); err != nil { + return nil, err + } + + if offset < 0 { + return nil, errors.New("offset is negative") + } + + if length < 0 { + return nil, errors.Errorf("invalid length %d", length) + } + + var obj *minio.Object + + objName := be.s3path(h.Type, h.Name) + + <-be.connChan + defer func() { + be.connChan <- struct{}{} + }() + + obj, err := be.client.GetObject(be.bucketname, objName) + if err != nil { + debug.Log(" err %v", err) + return nil, errors.Wrap(err, "client.GetObject") + } + + // if we're going to read the whole object, just pass it on. + if length == 0 { + debug.Log("Get %v: pass on object", h) + _, err = obj.Seek(offset, 0) + if err != nil { + return nil, errors.Wrap(err, "obj.Seek") + } + + return obj, nil + } + + // otherwise use a buffer with ReadAt + info, err := obj.Stat() + if err != nil { + return nil, errors.Wrap(err, "obj.Stat") + } + + if offset > info.Size { + return nil, errors.Errorf("offset larger than file size") + } + + l := int64(length) + if offset+l > info.Size { + l = info.Size - offset + } + + buf := make([]byte, l) + n, err := obj.ReadAt(buf, offset) + debug.Log("Get %v: use buffer with ReadAt: %v, %v", h, n, err) + if err == io.EOF { + debug.Log("Get %v: shorten buffer %v -> %v", h, len(buf), n) + buf = buf[:n] + err = nil + } + + if err != nil { + return nil, errors.Wrap(err, "obj.ReadAt") + } + + return backend.Closer{Reader: bytes.NewReader(buf)}, nil +} + // Stat returns information about a blob. -func (be s3) Stat(h restic.Handle) (bi restic.FileInfo, err error) { +func (be *s3) Stat(h restic.Handle) (bi restic.FileInfo, err error) { debug.Log("%v", h) objName := be.s3path(h.Type, h.Name) diff --git a/src/restic/backend/sftp/backend_test.go b/src/restic/backend/sftp/backend_test.go index c28dd8c99..b066d4966 100644 --- a/src/restic/backend/sftp/backend_test.go +++ b/src/restic/backend/sftp/backend_test.go @@ -58,6 +58,13 @@ func TestSftpBackendLoadNegativeOffset(t *testing.T) { test.TestLoadNegativeOffset(t) } +func TestSftpBackendGet(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestGet(t) +} + func TestSftpBackendSave(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) diff --git a/src/restic/backend/sftp/sftp.go b/src/restic/backend/sftp/sftp.go index 3ad34c384..4dee53b37 100644 --- a/src/restic/backend/sftp/sftp.go +++ b/src/restic/backend/sftp/sftp.go @@ -398,6 +398,39 @@ func (r *SFTP) Save(h restic.Handle, rd io.Reader) (err error) { return err } +// Get returns a reader that yields the contents of the file at h at the +// given offset. If length is nonzero, only a portion of the file is +// returned. rd must be closed after use. +func (r *SFTP) Get(h restic.Handle, length int, offset int64) (io.ReadCloser, error) { + debug.Log("Get %v, length %v, offset %v", h, length, offset) + if err := h.Valid(); err != nil { + return nil, err + } + + if offset < 0 { + return nil, errors.New("offset is negative") + } + + f, err := r.c.Open(r.filename(h.Type, h.Name)) + if err != nil { + return nil, err + } + + if offset > 0 { + _, err = f.Seek(offset, 0) + if err != nil { + f.Close() + return nil, err + } + } + + if length > 0 { + return backend.LimitReadCloser(f, int64(length)), nil + } + + return f, nil +} + // Stat returns information about a blob. func (r *SFTP) Stat(h restic.Handle) (restic.FileInfo, error) { debug.Log("stat %v", h) diff --git a/src/restic/backend/test/backend_test.go b/src/restic/backend/test/backend_test.go index c577092fb..9239d10a7 100644 --- a/src/restic/backend/test/backend_test.go +++ b/src/restic/backend/test/backend_test.go @@ -58,6 +58,13 @@ func TestTestBackendLoadNegativeOffset(t *testing.T) { test.TestLoadNegativeOffset(t) } +func TestTestBackendGet(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestGet(t) +} + func TestTestBackendSave(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) diff --git a/src/restic/backend/test/tests.go b/src/restic/backend/test/tests.go index 36ab3f26c..16147eeae 100644 --- a/src/restic/backend/test/tests.go +++ b/src/restic/backend/test/tests.go @@ -4,6 +4,7 @@ import ( "bytes" "fmt" "io" + "io/ioutil" "math/rand" "reflect" "restic" @@ -369,6 +370,99 @@ func TestLoadNegativeOffset(t testing.TB) { test.OK(t, b.Remove(restic.DataFile, id.String())) } +// TestGet tests the backend's Get function. +func TestGet(t testing.TB) { + b := open(t) + defer close(t) + + _, err := b.Get(restic.Handle{}, 0, 0) + if err == nil { + t.Fatalf("Get() did not return an error for invalid handle") + } + + _, err = b.Get(restic.Handle{Type: restic.DataFile, Name: "foobar"}, 0, 0) + if err == nil { + t.Fatalf("Get() did not return an error for non-existing blob") + } + + length := rand.Intn(1<<24) + 2000 + + data := test.Random(23, length) + id := restic.Hash(data) + + handle := restic.Handle{Type: restic.DataFile, Name: id.String()} + err = b.Save(handle, bytes.NewReader(data)) + if err != nil { + t.Fatalf("Save() error: %v", err) + } + + rd, err := b.Get(handle, 100, -1) + if err == nil { + t.Fatalf("Get() returned no error for negative offset!") + } + + if rd != nil { + t.Fatalf("Get() returned a non-nil reader for negative offset!") + } + + for i := 0; i < 50; i++ { + l := rand.Intn(length + 2000) + o := rand.Intn(length + 2000) + + d := data + if o < len(d) { + d = d[o:] + } else { + o = len(d) + d = d[:0] + } + + getlen := l + if l >= len(d) && rand.Float32() >= 0.5 { + getlen = 0 + } + + if l > 0 && l < len(d) { + d = d[:l] + } + + rd, err := b.Get(handle, getlen, int64(o)) + if err != nil { + t.Errorf("Get(%d, %d) returned unexpected error: %v", l, o, err) + continue + } + + buf, err := ioutil.ReadAll(rd) + if err != nil { + t.Errorf("Get(%d, %d) ReadAll() returned unexpected error: %v", l, o, err) + continue + } + + if l <= len(d) && len(buf) != l { + t.Errorf("Get(%d, %d) wrong number of bytes read: want %d, got %d", l, o, l, len(buf)) + continue + } + + if l > len(d) && len(buf) != len(d) { + t.Errorf("Get(%d, %d) wrong number of bytes read for overlong read: want %d, got %d", l, o, l, len(buf)) + continue + } + + if !bytes.Equal(buf, d) { + t.Errorf("Get(%d, %d) returned wrong bytes", l, o) + continue + } + + err = rd.Close() + if err != nil { + t.Errorf("Get(%d, %d) rd.Close() returned unexpected error: %v", l, o, err) + continue + } + } + + test.OK(t, b.Remove(restic.DataFile, id.String())) +} + // TestSave tests saving data in the backend. func TestSave(t testing.TB) { b := open(t) diff --git a/src/restic/backend/utils.go b/src/restic/backend/utils.go index 82a899515..575e847ed 100644 --- a/src/restic/backend/utils.go +++ b/src/restic/backend/utils.go @@ -28,3 +28,30 @@ func LoadAll(be restic.Backend, h restic.Handle, buf []byte) ([]byte, error) { buf = buf[:n] return buf, err } + +// Closer wraps an io.Reader and adds a Close() method that does nothing. +type Closer struct { + io.Reader +} + +// Close is a no-op. +func (c Closer) Close() error { + return nil +} + +// LimitedReadCloser wraps io.LimitedReader and exposes the Close() method. +type LimitedReadCloser struct { + io.ReadCloser + io.Reader +} + +// Read reads data from the limited reader. +func (l *LimitedReadCloser) Read(p []byte) (int, error) { + return l.Reader.Read(p) +} + +// LimitReadCloser returns a new reader wraps r in an io.LimitReader, but also +// exposes the Close() method. +func LimitReadCloser(r io.ReadCloser, n int64) *LimitedReadCloser { + return &LimitedReadCloser{ReadCloser: r, Reader: io.LimitReader(r, n)} +} diff --git a/src/restic/mock/backend.go b/src/restic/mock/backend.go index c52282450..b2a390661 100644 --- a/src/restic/mock/backend.go +++ b/src/restic/mock/backend.go @@ -12,6 +12,7 @@ type Backend struct { CloseFn func() error LoadFn func(h restic.Handle, p []byte, off int64) (int, error) SaveFn func(h restic.Handle, rd io.Reader) error + GetFn func(h restic.Handle, length int, offset int64) (io.ReadCloser, error) StatFn func(h restic.Handle) (restic.FileInfo, error) ListFn func(restic.FileType, <-chan struct{}) <-chan string RemoveFn func(restic.FileType, string) error @@ -56,6 +57,15 @@ func (m *Backend) Save(h restic.Handle, rd io.Reader) error { return m.SaveFn(h, rd) } +// Get loads data from the backend. +func (m *Backend) Get(h restic.Handle, length int, offset int64) (io.ReadCloser, error) { + if m.GetFn == nil { + return nil, errors.New("not implemented") + } + + return m.GetFn(h, length, offset) +} + // Stat an object in the backend. func (m *Backend) Stat(h restic.Handle) (restic.FileInfo, error) { if m.StatFn == nil {