Merge pull request #1482 from restic/fix-1478

backend: Only return top-level files for most dirs

internal/backend/azure/azure.go

@@ -278,7 +278,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType) <-chan string {
 	debug.Log("listing %v", t)
 	ch := make(chan string)
 
-	prefix := be.Dirname(restic.Handle{Type: t})
+	prefix, _ := be.Basedir(t)
 
 	// make sure prefix ends with a slash
 	if prefix[len(prefix)-1] != '/' {

internal/backend/b2/b2.go

@@ -272,7 +272,7 @@ func (be *b2Backend) List(ctx context.Context, t restic.FileType) <-chan string
 		defer close(ch)
 		defer cancel()
 
-		prefix := be.Dirname(restic.Handle{Type: t})
+		prefix, _ := be.Basedir(t)
 		cur := &b2.Cursor{Prefix: prefix}
 
 		for {

internal/backend/gs/gs.go

@@ -377,7 +377,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType) <-chan string {
 	debug.Log("listing %v", t)
 	ch := make(chan string)
 
-	prefix := be.Dirname(restic.Handle{Type: t})
+	prefix, _ := be.Basedir(t)
 
 	// make sure prefix ends with a slash
 	if prefix[len(prefix)-1] != '/' {

internal/backend/layout.go

@@ -16,7 +16,7 @@ import (
 type Layout interface {
 	Filename(restic.Handle) string
 	Dirname(restic.Handle) string
-	Basedir(restic.FileType) string
+	Basedir(restic.FileType) (dir string, subdirs bool)
 	Paths() []string
 	Name() string
 }

internal/backend/layout_default.go

@@ -69,6 +69,11 @@ func (l *DefaultLayout) Paths() (dirs []string) {
 }
 
 // Basedir returns the base dir name for type t.
-func (l *DefaultLayout) Basedir(t restic.FileType) string {
+func (l *DefaultLayout) Basedir(t restic.FileType) (dirname string, subdirs bool) {
-	return l.Join(l.Path, defaultLayoutPaths[t])
+	if t == restic.DataFile {
+		subdirs = true
+	}
+
+	dirname = l.Join(l.Path, defaultLayoutPaths[t])
+	return
 }

internal/backend/layout_rest.go

@@ -49,6 +49,6 @@ func (l *RESTLayout) Paths() (dirs []string) {
 }
 
 // Basedir returns the base dir name for files of type t.
-func (l *RESTLayout) Basedir(t restic.FileType) string {
+func (l *RESTLayout) Basedir(t restic.FileType) (dirname string, subdirs bool) {
-	return l.URL + l.Join(l.Path, restLayoutPaths[t])
+	return l.URL + l.Join(l.Path, restLayoutPaths[t]), false
 }

internal/backend/layout_s3legacy.go

@@ -72,6 +72,6 @@ func (l *S3LegacyLayout) Paths() (dirs []string) {
 }
 
 // Basedir returns the base dir name for type t.
-func (l *S3LegacyLayout) Basedir(t restic.FileType) string {
+func (l *S3LegacyLayout) Basedir(t restic.FileType) (dirname string, subdirs bool) {
-	return l.Join(l.Path, s3LayoutPaths[t])
+	return l.Join(l.Path, s3LayoutPaths[t]), false
 }

internal/backend/local/local.go

@@ -252,15 +252,27 @@ func (b *Local) List(ctx context.Context, t restic.FileType) <-chan string {
 	go func() {
 		defer close(ch)
 
-		err := fs.Walk(b.Basedir(t), func(path string, fi os.FileInfo, err error) error {
+		basedir, subdirs := b.Basedir(t)
+		err := fs.Walk(basedir, func(path string, fi os.FileInfo, err error) error {
+			debug.Log("walk on %v, %v\n", path, fi.IsDir())
 			if err != nil {
 				return err
 			}
 
+			if path == basedir {
+				return nil
+			}
+
 			if !isFile(fi) {
 				return nil
 			}
 
+			if fi.IsDir() && !subdirs {
+				return filepath.SkipDir
+			}
+
+			debug.Log("send %v\n", filepath.Base(path))
+
 			select {
 			case ch <- filepath.Base(path):
 			case <-ctx.Done():

internal/backend/s3/s3.go

@@ -409,7 +409,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType) <-chan string {
 	debug.Log("listing %v", t)
 	ch := make(chan string)
 
-	prefix := be.Dirname(restic.Handle{Type: t})
+	prefix, recursive := be.Basedir(t)
 
 	// make sure prefix ends with a slash
 	if prefix[len(prefix)-1] != '/' {
@@ -419,7 +419,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType) <-chan string {
 	// NB: unfortunately we can't protect this with be.sem.GetToken() here.
 	// Doing so would enable a deadlock situation (gh-1399), as ListObjects()
 	// starts its own goroutine and returns results via a channel.
-	listresp := be.client.ListObjects(be.cfg.Bucket, prefix, true, ctx.Done())
+	listresp := be.client.ListObjects(be.cfg.Bucket, prefix, recursive, ctx.Done())
 
 	go func() {
 		defer close(ch)

internal/backend/sftp/sftp.go

@@ -455,12 +455,22 @@ func (r *SFTP) List(ctx context.Context, t restic.FileType) <-chan string {
 	go func() {
 		defer close(ch)
 
-		walker := r.c.Walk(r.Basedir(t))
+		basedir, subdirs := r.Basedir(t)
+		walker := r.c.Walk(basedir)
 		for walker.Step() {
 			if walker.Err() != nil {
 				continue
 			}
 
+			if walker.Path() == basedir {
+				continue
+			}
+
+			if walker.Stat().IsDir() && !subdirs {
+				walker.SkipDir()
+				continue
+			}
+
 			if !walker.Stat().Mode().IsRegular() {
 				continue
 			}

internal/backend/swift/swift.go

@@ -244,7 +244,8 @@ func (be *beSwift) List(ctx context.Context, t restic.FileType) <-chan string {
 	debug.Log("listing %v", t)
 	ch := make(chan string)
 
-	prefix := be.Filename(restic.Handle{Type: t}) + "/"
+	prefix, _ := be.Basedir(t)
+	prefix += "/"
 
 	go func() {
 		defer close(ch)

internal/repository/key.go

@@ -122,6 +122,12 @@ func SearchKey(ctx context.Context, s *Repository, password string, maxKeys int)
 			return nil, ErrMaxKeysReached
 		}
 
+		_, err := restic.ParseID(name)
+		if err != nil {
+			debug.Log("rejecting key with invalid name: %v", name)
+			continue
+		}
+
 		debug.Log("trying key %q", name)
 		key, err := OpenKey(ctx, s, name, password)
 		if err != nil {