diff --git a/internal/backend/azure/azure.go b/internal/backend/azure/azure.go index b163ec992..ed401c868 100644 --- a/internal/backend/azure/azure.go +++ b/internal/backend/azure/azure.go @@ -242,7 +242,11 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, return restic.FileInfo{}, errors.Wrap(err, "blob.GetProperties") } - return restic.FileInfo{Size: int64(blob.Properties.ContentLength)}, nil + fi := restic.FileInfo{ + Size: int64(blob.Properties.ContentLength), + Name: h.Name, + } + return fi, nil } // Test returns true if a blob of the given type and name exists in the backend. @@ -271,17 +275,15 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { return errors.Wrap(err, "client.RemoveObject") } -// List returns a channel that yields all names of blobs of type t. A -// goroutine is started for this. If the channel done is closed, sending -// stops. -func (be *Backend) List(ctx context.Context, t restic.FileType) <-chan string { +// List runs fn for each file in the backend which has the type t. When an +// error occurs (or fn returns an error), List stops and returns it. +func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { debug.Log("listing %v", t) - ch := make(chan string) prefix, _ := be.Basedir(t) // make sure prefix ends with a slash - if prefix[len(prefix)-1] != '/' { + if !strings.HasSuffix(prefix, "/") { prefix += "/" } @@ -290,53 +292,57 @@ func (be *Backend) List(ctx context.Context, t restic.FileType) <-chan string { Prefix: prefix, } - go func() { - defer close(ch) + for { + be.sem.GetToken() + obj, err := be.container.ListBlobs(params) + be.sem.ReleaseToken() - for { - be.sem.GetToken() - obj, err := be.container.ListBlobs(params) - be.sem.ReleaseToken() - - if err != nil { - return - } - - debug.Log("got %v objects", len(obj.Blobs)) - - for _, item := range obj.Blobs { - m := strings.TrimPrefix(item.Name, prefix) - if m == "" { - continue - } - - select { - case ch <- path.Base(m): - case <-ctx.Done(): - return - } - } - - if obj.NextMarker == "" { - break - } - params.Marker = obj.NextMarker + if err != nil { + return err } - }() - return ch + debug.Log("got %v objects", len(obj.Blobs)) + + for _, item := range obj.Blobs { + m := strings.TrimPrefix(item.Name, prefix) + if m == "" { + continue + } + + fi := restic.FileInfo{ + Name: path.Base(m), + Size: item.Properties.ContentLength, + } + + if ctx.Err() != nil { + return ctx.Err() + } + + err := fn(fi) + if err != nil { + return err + } + + if ctx.Err() != nil { + return ctx.Err() + } + + } + + if obj.NextMarker == "" { + break + } + params.Marker = obj.NextMarker + } + + return ctx.Err() } // Remove keys for a specified backend type. func (be *Backend) removeKeys(ctx context.Context, t restic.FileType) error { - for key := range be.List(ctx, restic.DataFile) { - err := be.Remove(ctx, restic.Handle{Type: restic.DataFile, Name: key}) - if err != nil { - return err - } - } - - return nil + return be.List(ctx, t, func(fi restic.FileInfo) error { + return be.Remove(ctx, restic.Handle{Type: t, Name: fi.Name}) + }) } // Delete removes all restic keys in the bucket. It will not remove the bucket itself. diff --git a/internal/backend/b2/b2.go b/internal/backend/b2/b2.go index edf9a14f3..64a333017 100644 --- a/internal/backend/b2/b2.go +++ b/internal/backend/b2/b2.go @@ -228,7 +228,7 @@ func (be *b2Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileI debug.Log("Attrs() err %v", err) return restic.FileInfo{}, errors.Wrap(err, "Stat") } - return restic.FileInfo{Size: info.Size}, nil + return restic.FileInfo{Size: info.Size, Name: h.Name}, nil } // Test returns true if a blob of the given type and name exists in the backend. @@ -262,66 +262,76 @@ func (be *b2Backend) Remove(ctx context.Context, h restic.Handle) error { // List returns a channel that yields all names of blobs of type t. A // goroutine is started for this. If the channel done is closed, sending // stops. -func (be *b2Backend) List(ctx context.Context, t restic.FileType) <-chan string { +func (be *b2Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { debug.Log("List %v", t) - ch := make(chan string) + + prefix, _ := be.Basedir(t) + cur := &b2.Cursor{Prefix: prefix} ctx, cancel := context.WithCancel(ctx) + defer cancel() - go func() { - defer close(ch) - defer cancel() + for { + be.sem.GetToken() + objs, c, err := be.bucket.ListCurrentObjects(ctx, be.listMaxItems, cur) + be.sem.ReleaseToken() - prefix, _ := be.Basedir(t) - cur := &b2.Cursor{Prefix: prefix} - - for { - be.sem.GetToken() - objs, c, err := be.bucket.ListCurrentObjects(ctx, be.listMaxItems, cur) - be.sem.ReleaseToken() - if err != nil && err != io.EOF { - // TODO: return err to caller once err handling in List() is improved - debug.Log("List: %v", err) - return - } - debug.Log("returned %v items", len(objs)) - for _, obj := range objs { - // Skip objects returned that do not have the specified prefix. - if !strings.HasPrefix(obj.Name(), prefix) { - continue - } - - m := path.Base(obj.Name()) - if m == "" { - continue - } - - select { - case ch <- m: - case <-ctx.Done(): - return - } - } - if err == io.EOF { - return - } - cur = c + if err != nil && err != io.EOF { + debug.Log("List: %v", err) + return err } - }() - return ch + debug.Log("returned %v items", len(objs)) + for _, obj := range objs { + // Skip objects returned that do not have the specified prefix. + if !strings.HasPrefix(obj.Name(), prefix) { + continue + } + + m := path.Base(obj.Name()) + if m == "" { + continue + } + + if ctx.Err() != nil { + return ctx.Err() + } + + attrs, err := obj.Attrs(ctx) + if err != nil { + return err + } + + fi := restic.FileInfo{ + Name: m, + Size: attrs.Size, + } + + err = fn(fi) + if err != nil { + return err + } + + if ctx.Err() != nil { + return ctx.Err() + } + } + + if err == io.EOF { + return ctx.Err() + } + cur = c + } + + return ctx.Err() } // Remove keys for a specified backend type. func (be *b2Backend) removeKeys(ctx context.Context, t restic.FileType) error { debug.Log("removeKeys %v", t) - for key := range be.List(ctx, t) { - err := be.Remove(ctx, restic.Handle{Type: t, Name: key}) - if err != nil { - return err - } - } - return nil + return be.List(ctx, t, func(fi restic.FileInfo) error { + return be.Remove(ctx, restic.Handle{Type: t, Name: fi.Name}) + }) } // Delete removes all restic keys in the bucket. It will not remove the bucket itself. diff --git a/internal/backend/gs/gs.go b/internal/backend/gs/gs.go index 8d0e66d23..e88d49f45 100644 --- a/internal/backend/gs/gs.go +++ b/internal/backend/gs/gs.go @@ -333,7 +333,7 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInf return restic.FileInfo{}, errors.Wrap(err, "service.Objects.Get") } - return restic.FileInfo{Size: int64(obj.Size)}, nil + return restic.FileInfo{Size: int64(obj.Size), Name: h.Name}, nil } // Test returns true if a blob of the given type and name exists in the backend. @@ -370,69 +370,72 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { return errors.Wrap(err, "client.RemoveObject") } -// List returns a channel that yields all names of blobs of type t. A -// goroutine is started for this. If the channel done is closed, sending -// stops. -func (be *Backend) List(ctx context.Context, t restic.FileType) <-chan string { +// List runs fn for each file in the backend which has the type t. When an +// error occurs (or fn returns an error), List stops and returns it. +func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { debug.Log("listing %v", t) - ch := make(chan string) prefix, _ := be.Basedir(t) // make sure prefix ends with a slash - if prefix[len(prefix)-1] != '/' { + if !strings.HasSuffix(prefix, "/") { prefix += "/" } - go func() { - defer close(ch) + ctx, cancel := context.WithCancel(ctx) + defer cancel() - listReq := be.service.Objects.List(be.bucketName).Prefix(prefix).MaxResults(int64(be.listMaxItems)) - for { - be.sem.GetToken() - obj, err := listReq.Do() - be.sem.ReleaseToken() + listReq := be.service.Objects.List(be.bucketName).Context(ctx).Prefix(prefix).MaxResults(int64(be.listMaxItems)) + for { + be.sem.GetToken() + obj, err := listReq.Do() + be.sem.ReleaseToken() - if err != nil { - fmt.Fprintf(os.Stderr, "error listing %v: %v\n", prefix, err) - return - } - - debug.Log("returned %v items", len(obj.Items)) - - for _, item := range obj.Items { - m := strings.TrimPrefix(item.Name, prefix) - if m == "" { - continue - } - - select { - case ch <- path.Base(m): - case <-ctx.Done(): - return - } - } - - if obj.NextPageToken == "" { - break - } - listReq.PageToken(obj.NextPageToken) + if err != nil { + return err } - }() - return ch + debug.Log("returned %v items", len(obj.Items)) + + for _, item := range obj.Items { + m := strings.TrimPrefix(item.Name, prefix) + if m == "" { + continue + } + + if ctx.Err() != nil { + return ctx.Err() + } + + fi := restic.FileInfo{ + Name: path.Base(m), + Size: int64(item.Size), + } + + err := fn(fi) + if err != nil { + return err + } + + if ctx.Err() != nil { + return ctx.Err() + } + } + + if obj.NextPageToken == "" { + break + } + listReq.PageToken(obj.NextPageToken) + } + + return ctx.Err() } // Remove keys for a specified backend type. func (be *Backend) removeKeys(ctx context.Context, t restic.FileType) error { - for key := range be.List(ctx, restic.DataFile) { - err := be.Remove(ctx, restic.Handle{Type: restic.DataFile, Name: key}) - if err != nil { - return err - } - } - - return nil + return be.List(ctx, t, func(fi restic.FileInfo) error { + return be.Remove(ctx, restic.Handle{Type: t, Name: fi.Name}) + }) } // Delete removes all restic keys in the bucket. It will not remove the bucket itself. diff --git a/internal/backend/local/local.go b/internal/backend/local/local.go index 02b3be4b2..8bf949f37 100644 --- a/internal/backend/local/local.go +++ b/internal/backend/local/local.go @@ -191,7 +191,7 @@ func (b *Local) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, err return restic.FileInfo{}, errors.Wrap(err, "Stat") } - return restic.FileInfo{Size: fi.Size()}, nil + return restic.FileInfo{Size: fi.Size(), Name: h.Name}, nil } // Test returns true if a blob of the given type and name exists in the backend. @@ -226,13 +226,13 @@ func isFile(fi os.FileInfo) bool { return fi.Mode()&(os.ModeType|os.ModeCharDevice) == 0 } -// List returns a channel that yields all names of blobs of type t. A -// goroutine is started for this. +// List runs fn for each file in the backend which has the type t. When an +// error occurs (or fn returns an error), List stops and returns it. func (b *Local) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { debug.Log("List %v", t) basedir, subdirs := b.Basedir(t) - err := fs.Walk(basedir, func(path string, fi os.FileInfo, err error) error { + return fs.Walk(basedir, func(path string, fi os.FileInfo, err error) error { debug.Log("walk on %v\n", path) if err != nil { return err @@ -257,24 +257,17 @@ func (b *Local) List(ctx context.Context, t restic.FileType, fn func(restic.File Size: fi.Size(), } + if ctx.Err() != nil { + return ctx.Err() + } + err = fn(rfi) if err != nil { return err } - if ctx.Err() != nil { - return ctx.Err() - } - - return nil + return ctx.Err() }) - - if err != nil { - debug.Log("Walk %v", err) - return err - } - - return ctx.Err() } // Delete removes the repository and all files. diff --git a/internal/backend/mem/mem_backend.go b/internal/backend/mem/mem_backend.go index 68041ac55..ba0ede583 100644 --- a/internal/backend/mem/mem_backend.go +++ b/internal/backend/mem/mem_backend.go @@ -143,7 +143,7 @@ func (be *MemoryBackend) Stat(ctx context.Context, h restic.Handle) (restic.File return restic.FileInfo{}, errNotFound } - return restic.FileInfo{Size: int64(len(e))}, nil + return restic.FileInfo{Size: int64(len(e)), Name: h.Name}, nil } // Remove deletes a file from the backend. @@ -177,6 +177,10 @@ func (be *MemoryBackend) List(ctx context.Context, t restic.FileType, fn func(re Size: int64(len(buf)), } + if ctx.Err() != nil { + return ctx.Err() + } + err := fn(fi) if err != nil { return err diff --git a/internal/backend/rest/rest.go b/internal/backend/rest/rest.go index 825202f20..ed879dc88 100644 --- a/internal/backend/rest/rest.go +++ b/internal/backend/rest/rest.go @@ -241,6 +241,7 @@ func (b *restBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInf bi := restic.FileInfo{ Size: resp.ContentLength, + Name: h.Name, } return bi, nil @@ -291,12 +292,9 @@ func (b *restBackend) Remove(ctx context.Context, h restic.Handle) error { return errors.Wrap(resp.Body.Close(), "Close") } -// List returns a channel that yields all names of blobs of type t. A -// goroutine is started for this. If the channel done is closed, sending -// stops. -func (b *restBackend) List(ctx context.Context, t restic.FileType) <-chan string { - ch := make(chan string) - +// List runs fn for each file in the backend which has the type t. When an +// error occurs (or fn returns an error), List stops and returns it. +func (b *restBackend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { url := b.Dirname(restic.Handle{Type: t}) if !strings.HasSuffix(url, "/") { url += "/" @@ -306,41 +304,38 @@ func (b *restBackend) List(ctx context.Context, t restic.FileType) <-chan string resp, err := ctxhttp.Get(ctx, b.client, url) b.sem.ReleaseToken() - if resp != nil { - defer func() { - _, _ = io.Copy(ioutil.Discard, resp.Body) - e := resp.Body.Close() - - if err == nil { - err = errors.Wrap(e, "Close") - } - }() - } - if err != nil { - close(ch) - return ch + return errors.Wrap(err, "Get") } dec := json.NewDecoder(resp.Body) var list []string if err = dec.Decode(&list); err != nil { - close(ch) - return ch + return errors.Wrap(err, "Decode") } - go func() { - defer close(ch) - for _, m := range list { - select { - case ch <- m: - case <-ctx.Done(): - return - } + for _, m := range list { + fi, err := b.Stat(ctx, restic.Handle{Name: m, Type: t}) + if err != nil { + return err } - }() - return ch + if ctx.Err() != nil { + return ctx.Err() + } + + fi.Name = m + err = fn(fi) + if err != nil { + return err + } + + if ctx.Err() != nil { + return ctx.Err() + } + } + + return ctx.Err() } // Close closes all open files. @@ -352,14 +347,9 @@ func (b *restBackend) Close() error { // Remove keys for a specified backend type. func (b *restBackend) removeKeys(ctx context.Context, t restic.FileType) error { - for key := range b.List(ctx, restic.DataFile) { - err := b.Remove(ctx, restic.Handle{Type: restic.DataFile, Name: key}) - if err != nil { - return err - } - } - - return nil + return b.List(ctx, t, func(fi restic.FileInfo) error { + return b.Remove(ctx, restic.Handle{Type: t, Name: fi.Name}) + }) } // Delete removes all data in the backend. diff --git a/internal/backend/s3/s3.go b/internal/backend/s3/s3.go index b0818e9b2..b33e76f64 100644 --- a/internal/backend/s3/s3.go +++ b/internal/backend/s3/s3.go @@ -365,7 +365,7 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInf return restic.FileInfo{}, errors.Wrap(err, "Stat") } - return restic.FileInfo{Size: fi.Size}, nil + return restic.FileInfo{Size: fi.Size, Name: h.Name}, nil } // Test returns true if a blob of the given type and name exists in the backend. @@ -402,54 +402,59 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { return errors.Wrap(err, "client.RemoveObject") } -// List returns a channel that yields all names of blobs of type t. A -// goroutine is started for this. If the channel done is closed, sending -// stops. -func (be *Backend) List(ctx context.Context, t restic.FileType) <-chan string { +// List runs fn for each file in the backend which has the type t. When an +// error occurs (or fn returns an error), List stops and returns it. +func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { debug.Log("listing %v", t) - ch := make(chan string) prefix, recursive := be.Basedir(t) // make sure prefix ends with a slash - if prefix[len(prefix)-1] != '/' { + if !strings.HasSuffix(prefix, "/") { prefix += "/" } + ctx, cancel := context.WithCancel(ctx) + defer cancel() + // NB: unfortunately we can't protect this with be.sem.GetToken() here. // Doing so would enable a deadlock situation (gh-1399), as ListObjects() // starts its own goroutine and returns results via a channel. listresp := be.client.ListObjects(be.cfg.Bucket, prefix, recursive, ctx.Done()) - go func() { - defer close(ch) - for obj := range listresp { - m := strings.TrimPrefix(obj.Key, prefix) - if m == "" { - continue - } - - select { - case ch <- path.Base(m): - case <-ctx.Done(): - return - } + for obj := range listresp { + m := strings.TrimPrefix(obj.Key, prefix) + if m == "" { + continue } - }() - return ch + fi := restic.FileInfo{ + Name: path.Base(m), + Size: obj.Size, + } + + if ctx.Err() != nil { + return ctx.Err() + } + + err := fn(fi) + if err != nil { + return err + } + + if ctx.Err() != nil { + return ctx.Err() + } + } + + return ctx.Err() } // Remove keys for a specified backend type. func (be *Backend) removeKeys(ctx context.Context, t restic.FileType) error { - for key := range be.List(ctx, restic.DataFile) { - err := be.Remove(ctx, restic.Handle{Type: restic.DataFile, Name: key}) - if err != nil { - return err - } - } - - return nil + return be.List(ctx, restic.DataFile, func(fi restic.FileInfo) error { + return be.Remove(ctx, restic.Handle{Type: t, Name: fi.Name}) + }) } // Delete removes all restic keys in the bucket. It will not remove the bucket itself. diff --git a/internal/backend/sftp/layout_test.go b/internal/backend/sftp/layout_test.go index db1f1a870..81e5f3240 100644 --- a/internal/backend/sftp/layout_test.go +++ b/internal/backend/sftp/layout_test.go @@ -56,9 +56,10 @@ func TestLayout(t *testing.T) { } datafiles := make(map[string]bool) - for id := range be.List(context.TODO(), restic.DataFile) { - datafiles[id] = false - } + err = be.List(context.TODO(), restic.DataFile, func(fi restic.FileInfo) error { + datafiles[fi.Name] = false + return nil + }) if len(datafiles) == 0 { t.Errorf("List() returned zero data files") diff --git a/internal/backend/sftp/sftp.go b/internal/backend/sftp/sftp.go index 7dfa2951e..a0e20101a 100644 --- a/internal/backend/sftp/sftp.go +++ b/internal/backend/sftp/sftp.go @@ -376,7 +376,7 @@ func (r *SFTP) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, erro return restic.FileInfo{}, errors.Wrap(err, "Lstat") } - return restic.FileInfo{Size: fi.Size()}, nil + return restic.FileInfo{Size: fi.Size(), Name: h.Name}, nil } // Test returns true if a blob of the given type and name exists in the backend. @@ -408,47 +408,54 @@ func (r *SFTP) Remove(ctx context.Context, h restic.Handle) error { return r.c.Remove(r.Filename(h)) } -// List returns a channel that yields all names of blobs of type t. A -// goroutine is started for this. If the channel done is closed, sending -// stops. -func (r *SFTP) List(ctx context.Context, t restic.FileType) <-chan string { +// List runs fn for each file in the backend which has the type t. When an +// error occurs (or fn returns an error), List stops and returns it. +func (r *SFTP) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { debug.Log("List %v", t) - ch := make(chan string) - - go func() { - defer close(ch) - - basedir, subdirs := r.Basedir(t) - walker := r.c.Walk(basedir) - for walker.Step() { - if walker.Err() != nil { - continue - } - - if walker.Path() == basedir { - continue - } - - if walker.Stat().IsDir() && !subdirs { - walker.SkipDir() - continue - } - - if !walker.Stat().Mode().IsRegular() { - continue - } - - select { - case ch <- path.Base(walker.Path()): - case <-ctx.Done(): - return - } + basedir, subdirs := r.Basedir(t) + walker := r.c.Walk(basedir) + for walker.Step() { + if walker.Err() != nil { + return walker.Err() } - }() - return ch + if walker.Path() == basedir { + continue + } + if walker.Stat().IsDir() && !subdirs { + walker.SkipDir() + continue + } + + fi := walker.Stat() + if !fi.Mode().IsRegular() { + continue + } + + debug.Log("send %v\n", path.Base(walker.Path())) + + rfi := restic.FileInfo{ + Name: path.Base(walker.Path()), + Size: fi.Size(), + } + + if ctx.Err() != nil { + return ctx.Err() + } + + err := fn(rfi) + if err != nil { + return err + } + + if ctx.Err() != nil { + return ctx.Err() + } + } + + return ctx.Err() } var closeTimeout = 2 * time.Second diff --git a/internal/backend/swift/swift.go b/internal/backend/swift/swift.go index 48aeba600..27df0d55a 100644 --- a/internal/backend/swift/swift.go +++ b/internal/backend/swift/swift.go @@ -6,7 +6,6 @@ import ( "io" "net/http" "path" - "path/filepath" "strings" "time" @@ -203,7 +202,7 @@ func (be *beSwift) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInf return restic.FileInfo{}, errors.Wrap(err, "conn.Object") } - return restic.FileInfo{Size: obj.Bytes}, nil + return restic.FileInfo{Size: obj.Bytes, Name: h.Name}, nil } // Test returns true if a blob of the given type and name exists in the backend. @@ -237,61 +236,62 @@ func (be *beSwift) Remove(ctx context.Context, h restic.Handle) error { return errors.Wrap(err, "conn.ObjectDelete") } -// List returns a channel that yields all names of blobs of type t. A -// goroutine is started for this. If the channel done is closed, sending -// stops. -func (be *beSwift) List(ctx context.Context, t restic.FileType) <-chan string { +// List runs fn for each file in the backend which has the type t. When an +// error occurs (or fn returns an error), List stops and returns it. +func (be *beSwift) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { debug.Log("listing %v", t) - ch := make(chan string) prefix, _ := be.Basedir(t) prefix += "/" - go func() { - defer close(ch) + err := be.conn.ObjectsWalk(be.container, &swift.ObjectsOpts{Prefix: prefix}, + func(opts *swift.ObjectsOpts) (interface{}, error) { + be.sem.GetToken() + newObjects, err := be.conn.Objects(be.container, opts) + be.sem.ReleaseToken() - err := be.conn.ObjectsWalk(be.container, &swift.ObjectsOpts{Prefix: prefix}, - func(opts *swift.ObjectsOpts) (interface{}, error) { - be.sem.GetToken() - newObjects, err := be.conn.ObjectNames(be.container, opts) - be.sem.ReleaseToken() + if err != nil { + return nil, errors.Wrap(err, "conn.ObjectNames") + } + for _, obj := range newObjects { + m := path.Base(strings.TrimPrefix(obj.Name, prefix)) + if m == "" { + continue + } + fi := restic.FileInfo{ + Name: m, + Size: obj.Bytes, + } + + if ctx.Err() != nil { + return nil, ctx.Err() + } + + err := fn(fi) if err != nil { - return nil, errors.Wrap(err, "conn.ObjectNames") + return nil, err } - for _, obj := range newObjects { - m := filepath.Base(strings.TrimPrefix(obj, prefix)) - if m == "" { - continue - } - select { - case ch <- m: - case <-ctx.Done(): - return nil, io.EOF - } + if ctx.Err() != nil { + return nil, ctx.Err() } - return newObjects, nil - }) + } + return newObjects, nil + }) - if err != nil { - debug.Log("ObjectsWalk returned error: %v", err) - } - }() + if err != nil { + return err + } - return ch + return ctx.Err() } // Remove keys for a specified backend type. func (be *beSwift) removeKeys(ctx context.Context, t restic.FileType) error { - for key := range be.List(ctx, t) { - err := be.Remove(ctx, restic.Handle{Type: t, Name: key}) - if err != nil { - return err - } - } - - return nil + return be.List(ctx, t, func(fi restic.FileInfo) error { + return be.Remove(ctx, restic.Handle{Type: t, Name: fi.Name}) + }) } // IsNotExist returns true if the error is caused by a not existing file. diff --git a/internal/backend/test/tests.go b/internal/backend/test/tests.go index dac23d21a..7c7735cf9 100644 --- a/internal/backend/test/tests.go +++ b/internal/backend/test/tests.go @@ -249,17 +249,17 @@ func (s *Suite) TestList(t *testing.T) { b := s.open(t) defer s.close(t, b) - list1 := restic.NewIDSet() + list1 := make(map[restic.ID]int64) for i := 0; i < numTestFiles; i++ { - data := []byte(fmt.Sprintf("random test blob %v", i)) + data := test.Random(rand.Int(), rand.Intn(100)+55) id := restic.Hash(data) h := restic.Handle{Type: restic.DataFile, Name: id.String()} err := b.Save(context.TODO(), h, bytes.NewReader(data)) if err != nil { t.Fatal(err) } - list1.Insert(id) + list1[id] = int64(len(data)) } t.Logf("wrote %v files", len(list1)) @@ -272,7 +272,7 @@ func (s *Suite) TestList(t *testing.T) { for _, test := range tests { t.Run(fmt.Sprintf("max-%v", test.maxItems), func(t *testing.T) { - list2 := restic.NewIDSet() + list2 := make(map[restic.ID]int64) type setter interface { SetListMaxItems(int) @@ -288,7 +288,7 @@ func (s *Suite) TestList(t *testing.T) { if err != nil { t.Fatal(err) } - list2.Insert(id) + list2[id] = fi.Size return nil }) @@ -298,9 +298,22 @@ func (s *Suite) TestList(t *testing.T) { t.Logf("loaded %v IDs from backend", len(list2)) - if !list1.Equals(list2) { - t.Errorf("lists are not equal, list1 %d entries, list2 %d entries", - len(list1), len(list2)) + for id, size := range list1 { + size2, ok := list2[id] + if !ok { + t.Errorf("id %v not returned by List()", id.Str()) + } + + if size != size2 { + t.Errorf("wrong size for id %v returned: want %v, got %v", id.Str(), size, size2) + } + } + + for id := range list2 { + _, ok := list1[id] + if !ok { + t.Errorf("extra id %v returned by List()", id.Str()) + } } }) } @@ -349,8 +362,8 @@ func (s *Suite) TestListCancel(t *testing.T) { return nil }) - if err != context.Canceled { - t.Fatalf("expected error not found, want %v, got %v", context.Canceled, err) + if errors.Cause(err) != context.Canceled { + t.Fatalf("expected error not found, want %v, got %v", context.Canceled, errors.Cause(err)) } }) @@ -404,7 +417,7 @@ func (s *Suite) TestListCancel(t *testing.T) { ctx, cancel := context.WithCancel(context.TODO()) defer cancel() - timeout := 500 * time.Millisecond + timeout := 200 * time.Millisecond ctxTimeout, _ := context.WithTimeout(ctx, timeout) @@ -414,7 +427,7 @@ func (s *Suite) TestListCancel(t *testing.T) { i++ // wait until the context is cancelled - time.Sleep(timeout) + time.Sleep(timeout + 200*time.Millisecond) return nil }) @@ -487,8 +500,12 @@ func (s *Suite) TestSave(t *testing.T) { fi, err := b.Stat(context.TODO(), h) test.OK(t, err) + if fi.Name != h.Name { + t.Errorf("Stat() returned wrong name, want %q, got %q", h.Name, fi.Name) + } + if fi.Size != int64(len(data)) { - t.Fatalf("Stat() returned different size, want %q, got %d", len(data), fi.Size) + t.Errorf("Stat() returned different size, want %q, got %d", len(data), fi.Size) } err = b.Remove(context.TODO(), h) diff --git a/internal/mock/backend.go b/internal/mock/backend.go index b011131c4..29543c5fe 100644 --- a/internal/mock/backend.go +++ b/internal/mock/backend.go @@ -15,7 +15,7 @@ type Backend struct { SaveFn func(ctx context.Context, h restic.Handle, rd io.Reader) error LoadFn func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) StatFn func(ctx context.Context, h restic.Handle) (restic.FileInfo, error) - ListFn func(ctx context.Context, t restic.FileType) <-chan string + ListFn func(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error RemoveFn func(ctx context.Context, h restic.Handle) error TestFn func(ctx context.Context, h restic.Handle) (bool, error) DeleteFn func(ctx context.Context) error @@ -77,14 +77,12 @@ func (m *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, e } // List items of type t. -func (m *Backend) List(ctx context.Context, t restic.FileType) <-chan string { +func (m *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { if m.ListFn == nil { - ch := make(chan string) - close(ch) - return ch + return nil } - return m.ListFn(ctx, t) + return m.ListFn(ctx, t, fn) } // Remove data from the backend.