Merge pull request #966 from restic/unify-repo-layout

WIP: Unify repository layout

doc/design.rst

@@ -76,8 +76,8 @@ identifies the repository, regardless if it is accessed via SFTP or
 locally. The field ``chunker_polynomial`` contains a parameter that is
 used for splitting large files into smaller chunks (see below).
 
-Filesystem-Based Repositories
+Repository Layout
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~
 
 The ``local`` and ``sftp`` backends are implemented using files and
 directories stored in a file system. The directory layout is the same
@@ -117,44 +117,20 @@ e.g.:
 
     $ restic -r /tmp/restic-repo init
 
-The local and sftp backends will also accept the repository layout
+The local and sftp backends will auto-detect and accept all layouts described
-described in the following section, so that remote repositories mounted
+in the following sections, so that remote repositories mounted locally e.g. via
-locally e.g. via fuse can be accessed. The layout auto-detection can be
+fuse can be accessed. The layout auto-detection can be overridden by specifying
-overridden by specifying the option ``-o local.layout=default``, valid
+the option ``-o local.layout=default``, valid values are ``default`` and
-values are ``default``, ``cloud`` and ``s3``. The option for the sftp
+``s3legacy``. The option for the sftp backend is named ``sftp.layout``, for the
-backend is named ``sftp.layout``.
+s3 backend ``s3.layout``.
 
-Object-Storage-Based Repositories
+S3 Legacy Layout
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~
 
-Repositories in a backend based on an object store (e.g. Amazon s3) have
+Unfortunately during development the AWS S3 backend uses slightly different
-the same basic layout, with the exception that all data pack files are
-directly saved in the ``data`` path, without the sub-directories listed
-for the filesystem-based backends as listed in the previous section. The
-layout looks like this:
-
-::
-
-    /config
-    /data
-     ├── 2159dd48f8a24f33c307b750592773f8b71ff8d11452132a7b2e2a6a01611be1
-     ├── 32ea976bc30771cebad8285cd99120ac8786f9ffd42141d452458089985043a5
-     ├── 59fe4bcde59bd6222eba87795e35a90d82cd2f138a27b6835032b7b58173a426
-     ├── 73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c
-    [...]
-    /index
-     ├── c38f5fb68307c6a3e3aa945d556e325dc38f5fb68307c6a3e3aa945d556e325d
-     └── ca171b1b7394d90d330b265d90f506f9984043b342525f019788f97e745c71fd
-    /keys
-     └── b02de829beeb3c01a63e6b25cbd421a98fef144f03b9a02e46eff9e2ca3f0bd7
-    /locks
-    /snapshots
-     └── 22a5af1bdc6e616f8a29579458c49627e01b32210d09adb288d1ecda7c5711ec
-
-Unfortunately during development the s3 backend uses slightly different
 paths (directory names use singular instead of plural for ``key``,
-``lock``, and ``snapshot`` files), for s3 the repository layout looks
+``lock``, and ``snapshot`` files), and the data files are stored directly below
-like this:
+the ``data`` directory. The S3 Legacy repository layout looks like this:
 
 ::
 
@@ -174,8 +150,8 @@ like this:
     /snapshot
      └── 22a5af1bdc6e616f8a29579458c49627e01b32210d09adb288d1ecda7c5711ec
 
-The s3 backend understands and accepts both forms, new backends are
+The S3 backend understands and accepts both forms, new backends are
-always created with the former layout for compatibility reasons.
+always created with the default layout for compatibility reasons.
 
 Pack Format
 -----------

src/cmds/restic/flags_test.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"io/ioutil"
 	"testing"
 )
 
@@ -13,6 +14,7 @@ func TestFlags(t *testing.T) {
 
 	for _, cmd := range cmdRoot.Commands() {
 		t.Run(cmd.Name(), func(t *testing.T) {
+			cmd.Flags().SetOutput(ioutil.Discard)
 			err := cmd.ParseFlags([]string{"--help"})
 			if err.Error() == "pflag: help requested" {
 				err = nil

src/cmds/restic/local_layout_test.go

@@ -12,12 +12,10 @@ func TestRestoreLocalLayout(t *testing.T) {
 			filename string
 			layout   string
 		}{
-			{"repo-layout-cloud.tar.gz", ""},
+			{"repo-layout-default.tar.gz", ""},
-			{"repo-layout-local.tar.gz", ""},
+			{"repo-layout-s3legacy.tar.gz", ""},
-			{"repo-layout-s3-old.tar.gz", ""},
+			{"repo-layout-default.tar.gz", "default"},
-			{"repo-layout-cloud.tar.gz", "cloud"},
+			{"repo-layout-s3legacy.tar.gz", "s3legacy"},
-			{"repo-layout-local.tar.gz", "default"},
-			{"repo-layout-s3-old.tar.gz", "s3"},
 		}
 
 		for _, test := range tests {

src/restic/backend/layout.go

@@ -85,36 +85,6 @@ func hasBackendFile(fs Filesystem, dir string) (bool, error) {
 	return false, nil
 }
 
-var dataSubdirName = regexp.MustCompile("^[a-fA-F0-9]{2}$")
-
-func hasSubdirBackendFile(fs Filesystem, dir string) (bool, error) {
-	entries, err := fs.ReadDir(dir)
-	if err != nil && fs.IsNotExist(errors.Cause(err)) {
-		return false, nil
-	}
-
-	if err != nil {
-		return false, errors.Wrap(err, "ReadDir")
-	}
-
-	for _, subdir := range entries {
-		if !dataSubdirName.MatchString(subdir.Name()) {
-			continue
-		}
-
-		present, err := hasBackendFile(fs, fs.Join(dir, subdir.Name()))
-		if err != nil {
-			return false, err
-		}
-
-		if present {
-			return true, nil
-		}
-	}
-
-	return false, nil
-}
-
 // ErrLayoutDetectionFailed is returned by DetectLayout() when the layout
 // cannot be detected automatically.
 var ErrLayoutDetectionFailed = errors.New("auto-detecting the filesystem layout failed")
@@ -128,39 +98,19 @@ func DetectLayout(repo Filesystem, dir string) (Layout, error) {
 		repo = &LocalFilesystem{}
 	}
 
-	// key file in the "keys" dir (DefaultLayout or CloudLayout)
+	// key file in the "keys" dir (DefaultLayout)
 	foundKeysFile, err := hasBackendFile(repo, repo.Join(dir, defaultLayoutPaths[restic.KeyFile]))
 	if err != nil {
 		return nil, err
 	}
 
-	// key file in the "key" dir (S3Layout)
+	// key file in the "key" dir (S3LegacyLayout)
 	foundKeyFile, err := hasBackendFile(repo, repo.Join(dir, s3LayoutPaths[restic.KeyFile]))
 	if err != nil {
 		return nil, err
 	}
 
-	// data file in "data" directory (S3Layout or CloudLayout)
+	if foundKeysFile && !foundKeyFile {
-	foundDataFile, err := hasBackendFile(repo, repo.Join(dir, s3LayoutPaths[restic.DataFile]))
-	if err != nil {
-		return nil, err
-	}
-
-	// data file in subdir of "data" directory (DefaultLayout)
-	foundDataSubdirFile, err := hasSubdirBackendFile(repo, repo.Join(dir, s3LayoutPaths[restic.DataFile]))
-	if err != nil {
-		return nil, err
-	}
-
-	if foundKeysFile && foundDataFile && !foundKeyFile && !foundDataSubdirFile {
-		debug.Log("found cloud layout at %v", dir)
-		return &CloudLayout{
-			Path: dir,
-			Join: repo.Join,
-		}, nil
-	}
-
-	if foundKeysFile && foundDataSubdirFile && !foundKeyFile && !foundDataFile {
 		debug.Log("found default layout at %v", dir)
 		return &DefaultLayout{
 			Path: dir,
@@ -168,9 +118,9 @@ func DetectLayout(repo Filesystem, dir string) (Layout, error) {
 		}, nil
 	}
 
-	if foundKeyFile && foundDataFile && !foundKeysFile && !foundDataSubdirFile {
+	if foundKeyFile && !foundKeysFile {
 		debug.Log("found s3 layout at %v", dir)
-		return &S3Layout{
+		return &S3LegacyLayout{
 			Path: dir,
 			Join: repo.Join,
 		}, nil
@@ -190,13 +140,8 @@ func ParseLayout(repo Filesystem, layout, defaultLayout, path string) (l Layout,
 			Path: path,
 			Join: repo.Join,
 		}
-	case "cloud":
+	case "s3legacy":
-		l = &CloudLayout{
+		l = &S3LegacyLayout{
-			Path: path,
-			Join: repo.Join,
-		}
-	case "s3":
-		l = &S3Layout{
 			Path: path,
 			Join: repo.Join,
 		}
@@ -205,7 +150,7 @@ func ParseLayout(repo Filesystem, layout, defaultLayout, path string) (l Layout,
 
 		// use the default layout if auto detection failed
 		if errors.Cause(err) == ErrLayoutDetectionFailed && defaultLayout != "" {
-			debug.Log("error: %v, use default layout %v", defaultLayout)
+			debug.Log("error: %v, use default layout %v", err, defaultLayout)
 			return ParseLayout(repo, defaultLayout, "", path)
 		}
 
@@ -214,7 +159,7 @@ func ParseLayout(repo Filesystem, layout, defaultLayout, path string) (l Layout,
 		}
 		debug.Log("layout detected: %v", l)
 	default:
-		return nil, errors.Errorf("unknown backend layout string %q, may be one of: default, cloud, s3", layout)
+		return nil, errors.Errorf("unknown backend layout string %q, may be one of: default, s3legacy", layout)
 	}
 
 	return l, nil

src/restic/backend/layout_cloud.go

@@ -1,46 +0,0 @@
-package backend
-
-import "restic"
-
-// CloudLayout implements the default layout for cloud storage backends, as
-// described in the Design document.
-type CloudLayout struct {
-	URL  string
-	Path string
-	Join func(...string) string
-}
-
-var cloudLayoutPaths = defaultLayoutPaths
-
-// Dirname returns the directory path for a given file type and name.
-func (l *CloudLayout) Dirname(h restic.Handle) string {
-	if h.Type == restic.ConfigFile {
-		return l.URL + l.Join(l.Path, "/")
-	}
-
-	return l.URL + l.Join(l.Path, "/", cloudLayoutPaths[h.Type]) + "/"
-}
-
-// Filename returns a path to a file, including its name.
-func (l *CloudLayout) Filename(h restic.Handle) string {
-	name := h.Name
-
-	if h.Type == restic.ConfigFile {
-		name = "config"
-	}
-
-	return l.URL + l.Join(l.Path, "/", cloudLayoutPaths[h.Type], name)
-}
-
-// Paths returns all directory names
-func (l *CloudLayout) Paths() (dirs []string) {
-	for _, p := range cloudLayoutPaths {
-		dirs = append(dirs, l.URL+l.Join(l.Path, p))
-	}
-	return dirs
-}
-
-// Basedir returns the base dir name for files of type t.
-func (l *CloudLayout) Basedir(t restic.FileType) string {
-	return l.URL + l.Join(l.Path, cloudLayoutPaths[t])
-}

src/restic/backend/layout_default.go

@@ -24,10 +24,10 @@ func (l *DefaultLayout) Dirname(h restic.Handle) string {
 	p := defaultLayoutPaths[h.Type]
 
 	if h.Type == restic.DataFile && len(h.Name) > 2 {
-		p = l.Join(p, h.Name[:2])
+		p = l.Join(p, h.Name[:2]) + "/"
 	}
 
-	return l.Join(l.Path, p)
+	return l.Join(l.Path, p) + "/"
 }
 
 // Filename returns a path to a file, including its name.

src/restic/backend/layout_rest.go

@@ -0,0 +1,45 @@
+package backend
+
+import "restic"
+
+// RESTLayout implements the default layout for the REST protocol.
+type RESTLayout struct {
+	URL  string
+	Path string
+	Join func(...string) string
+}
+
+var restLayoutPaths = defaultLayoutPaths
+
+// Dirname returns the directory path for a given file type and name.
+func (l *RESTLayout) Dirname(h restic.Handle) string {
+	if h.Type == restic.ConfigFile {
+		return l.URL + l.Join(l.Path, "/")
+	}
+
+	return l.URL + l.Join(l.Path, "/", restLayoutPaths[h.Type]) + "/"
+}
+
+// Filename returns a path to a file, including its name.
+func (l *RESTLayout) Filename(h restic.Handle) string {
+	name := h.Name
+
+	if h.Type == restic.ConfigFile {
+		name = "config"
+	}
+
+	return l.URL + l.Join(l.Path, "/", restLayoutPaths[h.Type], name)
+}
+
+// Paths returns all directory names
+func (l *RESTLayout) Paths() (dirs []string) {
+	for _, p := range restLayoutPaths {
+		dirs = append(dirs, l.URL+l.Join(l.Path, p))
+	}
+	return dirs
+}
+
+// Basedir returns the base dir name for files of type t.
+func (l *RESTLayout) Basedir(t restic.FileType) string {
+	return l.URL + l.Join(l.Path, restLayoutPaths[t])
+}

src/restic/backend/layout_s3legacy.go

@@ -2,9 +2,9 @@ package backend
 
 import "restic"
 
-// S3Layout implements the old layout used for s3 cloud storage backends, as
+// S3LegacyLayout implements the old layout used for s3 cloud storage backends, as
 // described in the Design document.
-type S3Layout struct {
+type S3LegacyLayout struct {
 	URL  string
 	Path string
 	Join func(...string) string
@@ -19,7 +19,7 @@ var s3LayoutPaths = map[restic.FileType]string{
 }
 
 // join calls Join with the first empty elements removed.
-func (l *S3Layout) join(url string, items ...string) string {
+func (l *S3LegacyLayout) join(url string, items ...string) string {
 	for len(items) > 0 && items[0] == "" {
 		items = items[1:]
 	}
@@ -35,7 +35,7 @@ func (l *S3Layout) join(url string, items ...string) string {
 }
 
 // Dirname returns the directory path for a given file type and name.
-func (l *S3Layout) Dirname(h restic.Handle) string {
+func (l *S3LegacyLayout) Dirname(h restic.Handle) string {
 	if h.Type == restic.ConfigFile {
 		return l.URL + l.Join(l.Path, "/")
 	}
@@ -44,7 +44,7 @@ func (l *S3Layout) Dirname(h restic.Handle) string {
 }
 
 // Filename returns a path to a file, including its name.
-func (l *S3Layout) Filename(h restic.Handle) string {
+func (l *S3LegacyLayout) Filename(h restic.Handle) string {
 	name := h.Name
 
 	if h.Type == restic.ConfigFile {
@@ -55,7 +55,7 @@ func (l *S3Layout) Filename(h restic.Handle) string {
 }
 
 // Paths returns all directory names
-func (l *S3Layout) Paths() (dirs []string) {
+func (l *S3LegacyLayout) Paths() (dirs []string) {
 	for _, p := range s3LayoutPaths {
 		dirs = append(dirs, l.Join(l.Path, p))
 	}
@@ -63,6 +63,6 @@ func (l *S3Layout) Paths() (dirs []string) {
 }
 
 // Basedir returns the base dir name for type t.
-func (l *S3Layout) Basedir(t restic.FileType) string {
+func (l *S3LegacyLayout) Basedir(t restic.FileType) string {
 	return l.Join(l.Path, s3LayoutPaths[t])
 }

src/restic/backend/layout_test.go

@@ -79,7 +79,7 @@ func TestDefaultLayout(t *testing.T) {
 	}
 }
 
-func TestCloudLayout(t *testing.T) {
+func TestRESTLayout(t *testing.T) {
 	path, cleanup := TempDir(t)
 	defer cleanup()
 
@@ -113,7 +113,7 @@ func TestCloudLayout(t *testing.T) {
 		},
 	}
 
-	l := &CloudLayout{
+	l := &RESTLayout{
 		Path: path,
 		Join: filepath.Join,
 	}
@@ -147,7 +147,7 @@ func TestCloudLayout(t *testing.T) {
 	}
 }
 
-func TestCloudLayoutURLs(t *testing.T) {
+func TestRESTLayoutURLs(t *testing.T) {
 	var tests = []struct {
 		l   Layout
 		h   restic.Handle
@@ -155,55 +155,55 @@ func TestCloudLayoutURLs(t *testing.T) {
 		dir string
 	}{
 		{
-			&CloudLayout{URL: "https://hostname.foo", Path: "", Join: path.Join},
+			&RESTLayout{URL: "https://hostname.foo", Path: "", Join: path.Join},
 			restic.Handle{Type: restic.DataFile, Name: "foobar"},
 			"https://hostname.foo/data/foobar",
 			"https://hostname.foo/data/",
 		},
 		{
-			&CloudLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join},
+			&RESTLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join},
 			restic.Handle{Type: restic.LockFile, Name: "foobar"},
 			"https://hostname.foo:1234/prefix/repo/locks/foobar",
 			"https://hostname.foo:1234/prefix/repo/locks/",
 		},
 		{
-			&CloudLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join},
+			&RESTLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join},
 			restic.Handle{Type: restic.ConfigFile, Name: "foobar"},
 			"https://hostname.foo:1234/prefix/repo/config",
 			"https://hostname.foo:1234/prefix/repo/",
 		},
 		{
-			&S3Layout{URL: "https://hostname.foo", Path: "/", Join: path.Join},
+			&S3LegacyLayout{URL: "https://hostname.foo", Path: "/", Join: path.Join},
 			restic.Handle{Type: restic.DataFile, Name: "foobar"},
 			"https://hostname.foo/data/foobar",
 			"https://hostname.foo/data/",
 		},
 		{
-			&S3Layout{URL: "https://hostname.foo:1234/prefix/repo", Path: "", Join: path.Join},
+			&S3LegacyLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "", Join: path.Join},
 			restic.Handle{Type: restic.LockFile, Name: "foobar"},
 			"https://hostname.foo:1234/prefix/repo/lock/foobar",
 			"https://hostname.foo:1234/prefix/repo/lock/",
 		},
 		{
-			&S3Layout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join},
+			&S3LegacyLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join},
 			restic.Handle{Type: restic.ConfigFile, Name: "foobar"},
 			"https://hostname.foo:1234/prefix/repo/config",
 			"https://hostname.foo:1234/prefix/repo/",
 		},
 		{
-			&S3Layout{URL: "", Path: "", Join: path.Join},
+			&S3LegacyLayout{URL: "", Path: "", Join: path.Join},
 			restic.Handle{Type: restic.DataFile, Name: "foobar"},
 			"data/foobar",
 			"data/",
 		},
 		{
-			&S3Layout{URL: "", Path: "", Join: path.Join},
+			&S3LegacyLayout{URL: "", Path: "", Join: path.Join},
 			restic.Handle{Type: restic.LockFile, Name: "foobar"},
 			"lock/foobar",
 			"lock/",
 		},
 		{
-			&S3Layout{URL: "", Path: "/", Join: path.Join},
+			&S3LegacyLayout{URL: "", Path: "/", Join: path.Join},
 			restic.Handle{Type: restic.ConfigFile, Name: "foobar"},
 			"/config",
 			"/",
@@ -225,7 +225,7 @@ func TestCloudLayoutURLs(t *testing.T) {
 	}
 }
 
-func TestS3Layout(t *testing.T) {
+func TestS3LegacyLayout(t *testing.T) {
 	path, cleanup := TempDir(t)
 	defer cleanup()
 
@@ -259,7 +259,7 @@ func TestS3Layout(t *testing.T) {
 		},
 	}
 
-	l := &S3Layout{
+	l := &S3LegacyLayout{
 		Path: path,
 		Join: filepath.Join,
 	}
@@ -301,9 +301,8 @@ func TestDetectLayout(t *testing.T) {
 		filename string
 		want     string
 	}{
-		{"repo-layout-local.tar.gz", "*backend.DefaultLayout"},
+		{"repo-layout-default.tar.gz", "*backend.DefaultLayout"},
-		{"repo-layout-cloud.tar.gz", "*backend.CloudLayout"},
+		{"repo-layout-s3legacy.tar.gz", "*backend.S3LegacyLayout"},
-		{"repo-layout-s3-old.tar.gz", "*backend.S3Layout"},
 	}
 
 	var fs = &LocalFilesystem{}
@@ -342,12 +341,11 @@ func TestParseLayout(t *testing.T) {
 		want              string
 	}{
 		{"default", "", "*backend.DefaultLayout"},
-		{"cloud", "", "*backend.CloudLayout"},
+		{"s3legacy", "", "*backend.S3LegacyLayout"},
-		{"s3", "", "*backend.S3Layout"},
+		{"", "", "*backend.DefaultLayout"},
-		{"", "", "*backend.CloudLayout"},
 	}
 
-	SetupTarTestFixture(t, path, filepath.Join("testdata", "repo-layout-cloud.tar.gz"))
+	SetupTarTestFixture(t, path, filepath.Join("testdata", "repo-layout-default.tar.gz"))
 
 	for _, test := range tests {
 		t.Run(test.layoutName, func(t *testing.T) {

src/restic/backend/local/layout_test.go

@@ -17,17 +17,12 @@ func TestLayout(t *testing.T) {
 		failureExpected bool
 		datafiles       map[string]bool
 	}{
-		{"repo-layout-local.tar.gz", "", false, map[string]bool{
+		{"repo-layout-default.tar.gz", "", false, map[string]bool{
 			"aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false,
 			"fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false,
 			"c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false,
 		}},
-		{"repo-layout-cloud.tar.gz", "", false, map[string]bool{
+		{"repo-layout-s3legacy.tar.gz", "", false, map[string]bool{
-			"fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false,
-			"c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false,
-			"aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false,
-		}},
-		{"repo-layout-s3-old.tar.gz", "", false, map[string]bool{
 			"fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false,
 			"c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false,
 			"aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false,

src/restic/backend/rest/rest.go

@@ -48,7 +48,7 @@ func Open(cfg Config) (restic.Backend, error) {
 		url:      cfg.URL,
 		connChan: connChan,
 		client:   client,
-		Layout:   &backend.CloudLayout{URL: url, Join: path.Join},
+		Layout:   &backend.RESTLayout{URL: url, Join: path.Join},
 	}
 
 	return be, nil

src/restic/backend/s3/config.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	"restic/errors"
+	"restic/options"
 )
 
 // Config contains all configuration necessary to connect to an s3 compatible
@@ -16,6 +17,11 @@ type Config struct {
 	KeyID, Secret string
 	Bucket        string
 	Prefix        string
+	Layout        string `option:"layout" help:"use this backend layout (default: auto-detect)"`
+}
+
+func init() {
+	options.Register("s3", Config{})
 }
 
 const defaultPrefix = "restic"

src/restic/backend/s3/s3.go

@@ -8,6 +8,7 @@ import (
 	"restic"
 	"strings"
 	"sync"
+	"time"
 
 	"restic/backend"
 	"restic/errors"
@@ -30,6 +31,8 @@ type s3 struct {
 	backend.Layout
 }
 
+const defaultLayout = "s3legacy"
+
 // Open opens the S3 backend at bucket and region. The bucket is created if it
 // does not exist yet.
 func Open(cfg Config) (restic.Backend, error) {
@@ -45,11 +48,17 @@ func Open(cfg Config) (restic.Backend, error) {
 		bucketname:   cfg.Bucket,
 		prefix:       cfg.Prefix,
 		cacheObjSize: make(map[string]int64),
-		Layout:       &backend.S3Layout{Path: cfg.Prefix, Join: path.Join},
 	}
 
 	client.SetCustomTransport(backend.Transport())
 
+	l, err := backend.ParseLayout(be, cfg.Layout, defaultLayout, cfg.Prefix)
+	if err != nil {
+		return nil, err
+	}
+
+	be.Layout = l
+
 	be.createConnections()
 
 	found, err := client.BucketExists(cfg.Bucket)
@@ -76,6 +85,77 @@ func (be *s3) createConnections() {
 	}
 }
 
+// IsNotExist returns true if the error is caused by a not existing file.
+func (be *s3) IsNotExist(err error) bool {
+	debug.Log("IsNotExist(%T, %#v)", err, err)
+	if os.IsNotExist(err) {
+		return true
+	}
+
+	return false
+}
+
+// Join combines path components with slashes.
+func (be *s3) Join(p ...string) string {
+	return path.Join(p...)
+}
+
+type fileInfo struct {
+	name    string
+	size    int64
+	mode    os.FileMode
+	modTime time.Time
+	isDir   bool
+}
+
+func (fi fileInfo) Name() string       { return fi.name }    // base name of the file
+func (fi fileInfo) Size() int64        { return fi.size }    // length in bytes for regular files; system-dependent for others
+func (fi fileInfo) Mode() os.FileMode  { return fi.mode }    // file mode bits
+func (fi fileInfo) ModTime() time.Time { return fi.modTime } // modification time
+func (fi fileInfo) IsDir() bool        { return fi.isDir }   // abbreviation for Mode().IsDir()
+func (fi fileInfo) Sys() interface{}   { return nil }        // underlying data source (can return nil)
+
+// ReadDir returns the entries for a directory.
+func (be *s3) ReadDir(dir string) (list []os.FileInfo, err error) {
+	debug.Log("ReadDir(%v)", dir)
+
+	// make sure dir ends with a slash
+	if dir[len(dir)-1] != '/' {
+		dir += "/"
+	}
+
+	done := make(chan struct{})
+	defer close(done)
+
+	for obj := range be.client.ListObjects(be.bucketname, dir, false, done) {
+		if obj.Key == "" {
+			continue
+		}
+
+		name := strings.TrimPrefix(obj.Key, dir)
+		if name == "" {
+			return nil, errors.Errorf("invalid key name %v, removing prefix %v yielded empty string", obj.Key, dir)
+		}
+		entry := fileInfo{
+			name:    name,
+			size:    obj.Size,
+			modTime: obj.LastModified,
+		}
+
+		if name[len(name)-1] == '/' {
+			entry.isDir = true
+			entry.mode = os.ModeDir | 0755
+			entry.name = name[:len(name)-1]
+		} else {
+			entry.mode = 0644
+		}
+
+		list = append(list, entry)
+	}
+
+	return list, nil
+}
+
 // Location returns this backend's location (the bucket name).
 func (be *s3) Location() string {
 	return be.bucketname
@@ -290,6 +370,11 @@ func (be *s3) List(t restic.FileType, done <-chan struct{}) <-chan string {
 
 	prefix := be.Dirname(restic.Handle{Type: t})
 
+	// make sure prefix ends with a slash
+	if prefix[len(prefix)-1] != '/' {
+		prefix += "/"
+	}
+
 	listresp := be.client.ListObjects(be.bucketname, prefix, true, done)
 
 	go func() {

src/restic/backend/sftp/layout_test.go

@@ -23,17 +23,12 @@ func TestLayout(t *testing.T) {
 		failureExpected bool
 		datafiles       map[string]bool
 	}{
-		{"repo-layout-local.tar.gz", "", false, map[string]bool{
+		{"repo-layout-default.tar.gz", "", false, map[string]bool{
 			"aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false,
 			"fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false,
 			"c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false,
 		}},
-		{"repo-layout-cloud.tar.gz", "", false, map[string]bool{
+		{"repo-layout-s3legacy.tar.gz", "", false, map[string]bool{
-			"fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false,
-			"c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false,
-			"aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false,
-		}},
-		{"repo-layout-s3-old.tar.gz", "", false, map[string]bool{
 			"fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false,
 			"c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false,
 			"aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false,