diff --git a/doc/design.rst b/doc/design.rst index c3f32a67c..16fa8e7ae 100644 --- a/doc/design.rst +++ b/doc/design.rst @@ -76,8 +76,8 @@ identifies the repository, regardless if it is accessed via SFTP or locally. The field ``chunker_polynomial`` contains a parameter that is used for splitting large files into smaller chunks (see below). -Filesystem-Based Repositories -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Repository Layout +~~~~~~~~~~~~~~~~~ The ``local`` and ``sftp`` backends are implemented using files and directories stored in a file system. The directory layout is the same @@ -117,44 +117,20 @@ e.g.: $ restic -r /tmp/restic-repo init -The local and sftp backends will also accept the repository layout -described in the following section, so that remote repositories mounted -locally e.g. via fuse can be accessed. The layout auto-detection can be -overridden by specifying the option ``-o local.layout=default``, valid -values are ``default``, ``cloud`` and ``s3``. The option for the sftp -backend is named ``sftp.layout``. +The local and sftp backends will auto-detect and accept all layouts described +in the following sections, so that remote repositories mounted locally e.g. via +fuse can be accessed. The layout auto-detection can be overridden by specifying +the option ``-o local.layout=default``, valid values are ``default`` and +``s3legacy``. The option for the sftp backend is named ``sftp.layout``, for the +s3 backend ``s3.layout``. -Object-Storage-Based Repositories -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +S3 Legacy Layout +~~~~~~~~~~~~~~~~ -Repositories in a backend based on an object store (e.g. Amazon s3) have -the same basic layout, with the exception that all data pack files are -directly saved in the ``data`` path, without the sub-directories listed -for the filesystem-based backends as listed in the previous section. The -layout looks like this: - -:: - - /config - /data - ├── 2159dd48f8a24f33c307b750592773f8b71ff8d11452132a7b2e2a6a01611be1 - ├── 32ea976bc30771cebad8285cd99120ac8786f9ffd42141d452458089985043a5 - ├── 59fe4bcde59bd6222eba87795e35a90d82cd2f138a27b6835032b7b58173a426 - ├── 73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c - [...] - /index - ├── c38f5fb68307c6a3e3aa945d556e325dc38f5fb68307c6a3e3aa945d556e325d - └── ca171b1b7394d90d330b265d90f506f9984043b342525f019788f97e745c71fd - /keys - └── b02de829beeb3c01a63e6b25cbd421a98fef144f03b9a02e46eff9e2ca3f0bd7 - /locks - /snapshots - └── 22a5af1bdc6e616f8a29579458c49627e01b32210d09adb288d1ecda7c5711ec - -Unfortunately during development the s3 backend uses slightly different +Unfortunately during development the AWS S3 backend uses slightly different paths (directory names use singular instead of plural for ``key``, -``lock``, and ``snapshot`` files), for s3 the repository layout looks -like this: +``lock``, and ``snapshot`` files), and the data files are stored directly below +the ``data`` directory. The S3 Legacy repository layout looks like this: :: @@ -174,8 +150,8 @@ like this: /snapshot └── 22a5af1bdc6e616f8a29579458c49627e01b32210d09adb288d1ecda7c5711ec -The s3 backend understands and accepts both forms, new backends are -always created with the former layout for compatibility reasons. +The S3 backend understands and accepts both forms, new backends are +always created with the default layout for compatibility reasons. Pack Format ----------- diff --git a/src/cmds/restic/flags_test.go b/src/cmds/restic/flags_test.go index 3b313b68c..7cd758592 100644 --- a/src/cmds/restic/flags_test.go +++ b/src/cmds/restic/flags_test.go @@ -1,6 +1,7 @@ package main import ( + "io/ioutil" "testing" ) @@ -13,6 +14,7 @@ func TestFlags(t *testing.T) { for _, cmd := range cmdRoot.Commands() { t.Run(cmd.Name(), func(t *testing.T) { + cmd.Flags().SetOutput(ioutil.Discard) err := cmd.ParseFlags([]string{"--help"}) if err.Error() == "pflag: help requested" { err = nil diff --git a/src/cmds/restic/local_layout_test.go b/src/cmds/restic/local_layout_test.go index eb6268e72..fbb1f13af 100644 --- a/src/cmds/restic/local_layout_test.go +++ b/src/cmds/restic/local_layout_test.go @@ -12,12 +12,10 @@ func TestRestoreLocalLayout(t *testing.T) { filename string layout string }{ - {"repo-layout-cloud.tar.gz", ""}, - {"repo-layout-local.tar.gz", ""}, - {"repo-layout-s3-old.tar.gz", ""}, - {"repo-layout-cloud.tar.gz", "cloud"}, - {"repo-layout-local.tar.gz", "default"}, - {"repo-layout-s3-old.tar.gz", "s3"}, + {"repo-layout-default.tar.gz", ""}, + {"repo-layout-s3legacy.tar.gz", ""}, + {"repo-layout-default.tar.gz", "default"}, + {"repo-layout-s3legacy.tar.gz", "s3legacy"}, } for _, test := range tests { diff --git a/src/restic/backend/layout.go b/src/restic/backend/layout.go index de72f07d4..3d0953de8 100644 --- a/src/restic/backend/layout.go +++ b/src/restic/backend/layout.go @@ -85,36 +85,6 @@ func hasBackendFile(fs Filesystem, dir string) (bool, error) { return false, nil } -var dataSubdirName = regexp.MustCompile("^[a-fA-F0-9]{2}$") - -func hasSubdirBackendFile(fs Filesystem, dir string) (bool, error) { - entries, err := fs.ReadDir(dir) - if err != nil && fs.IsNotExist(errors.Cause(err)) { - return false, nil - } - - if err != nil { - return false, errors.Wrap(err, "ReadDir") - } - - for _, subdir := range entries { - if !dataSubdirName.MatchString(subdir.Name()) { - continue - } - - present, err := hasBackendFile(fs, fs.Join(dir, subdir.Name())) - if err != nil { - return false, err - } - - if present { - return true, nil - } - } - - return false, nil -} - // ErrLayoutDetectionFailed is returned by DetectLayout() when the layout // cannot be detected automatically. var ErrLayoutDetectionFailed = errors.New("auto-detecting the filesystem layout failed") @@ -128,39 +98,19 @@ func DetectLayout(repo Filesystem, dir string) (Layout, error) { repo = &LocalFilesystem{} } - // key file in the "keys" dir (DefaultLayout or CloudLayout) + // key file in the "keys" dir (DefaultLayout) foundKeysFile, err := hasBackendFile(repo, repo.Join(dir, defaultLayoutPaths[restic.KeyFile])) if err != nil { return nil, err } - // key file in the "key" dir (S3Layout) + // key file in the "key" dir (S3LegacyLayout) foundKeyFile, err := hasBackendFile(repo, repo.Join(dir, s3LayoutPaths[restic.KeyFile])) if err != nil { return nil, err } - // data file in "data" directory (S3Layout or CloudLayout) - foundDataFile, err := hasBackendFile(repo, repo.Join(dir, s3LayoutPaths[restic.DataFile])) - if err != nil { - return nil, err - } - - // data file in subdir of "data" directory (DefaultLayout) - foundDataSubdirFile, err := hasSubdirBackendFile(repo, repo.Join(dir, s3LayoutPaths[restic.DataFile])) - if err != nil { - return nil, err - } - - if foundKeysFile && foundDataFile && !foundKeyFile && !foundDataSubdirFile { - debug.Log("found cloud layout at %v", dir) - return &CloudLayout{ - Path: dir, - Join: repo.Join, - }, nil - } - - if foundKeysFile && foundDataSubdirFile && !foundKeyFile && !foundDataFile { + if foundKeysFile && !foundKeyFile { debug.Log("found default layout at %v", dir) return &DefaultLayout{ Path: dir, @@ -168,9 +118,9 @@ func DetectLayout(repo Filesystem, dir string) (Layout, error) { }, nil } - if foundKeyFile && foundDataFile && !foundKeysFile && !foundDataSubdirFile { + if foundKeyFile && !foundKeysFile { debug.Log("found s3 layout at %v", dir) - return &S3Layout{ + return &S3LegacyLayout{ Path: dir, Join: repo.Join, }, nil @@ -190,13 +140,8 @@ func ParseLayout(repo Filesystem, layout, defaultLayout, path string) (l Layout, Path: path, Join: repo.Join, } - case "cloud": - l = &CloudLayout{ - Path: path, - Join: repo.Join, - } - case "s3": - l = &S3Layout{ + case "s3legacy": + l = &S3LegacyLayout{ Path: path, Join: repo.Join, } @@ -205,7 +150,7 @@ func ParseLayout(repo Filesystem, layout, defaultLayout, path string) (l Layout, // use the default layout if auto detection failed if errors.Cause(err) == ErrLayoutDetectionFailed && defaultLayout != "" { - debug.Log("error: %v, use default layout %v", defaultLayout) + debug.Log("error: %v, use default layout %v", err, defaultLayout) return ParseLayout(repo, defaultLayout, "", path) } @@ -214,7 +159,7 @@ func ParseLayout(repo Filesystem, layout, defaultLayout, path string) (l Layout, } debug.Log("layout detected: %v", l) default: - return nil, errors.Errorf("unknown backend layout string %q, may be one of: default, cloud, s3", layout) + return nil, errors.Errorf("unknown backend layout string %q, may be one of: default, s3legacy", layout) } return l, nil diff --git a/src/restic/backend/layout_cloud.go b/src/restic/backend/layout_cloud.go deleted file mode 100644 index 6331d4709..000000000 --- a/src/restic/backend/layout_cloud.go +++ /dev/null @@ -1,46 +0,0 @@ -package backend - -import "restic" - -// CloudLayout implements the default layout for cloud storage backends, as -// described in the Design document. -type CloudLayout struct { - URL string - Path string - Join func(...string) string -} - -var cloudLayoutPaths = defaultLayoutPaths - -// Dirname returns the directory path for a given file type and name. -func (l *CloudLayout) Dirname(h restic.Handle) string { - if h.Type == restic.ConfigFile { - return l.URL + l.Join(l.Path, "/") - } - - return l.URL + l.Join(l.Path, "/", cloudLayoutPaths[h.Type]) + "/" -} - -// Filename returns a path to a file, including its name. -func (l *CloudLayout) Filename(h restic.Handle) string { - name := h.Name - - if h.Type == restic.ConfigFile { - name = "config" - } - - return l.URL + l.Join(l.Path, "/", cloudLayoutPaths[h.Type], name) -} - -// Paths returns all directory names -func (l *CloudLayout) Paths() (dirs []string) { - for _, p := range cloudLayoutPaths { - dirs = append(dirs, l.URL+l.Join(l.Path, p)) - } - return dirs -} - -// Basedir returns the base dir name for files of type t. -func (l *CloudLayout) Basedir(t restic.FileType) string { - return l.URL + l.Join(l.Path, cloudLayoutPaths[t]) -} diff --git a/src/restic/backend/layout_default.go b/src/restic/backend/layout_default.go index 77cb27508..836700b33 100644 --- a/src/restic/backend/layout_default.go +++ b/src/restic/backend/layout_default.go @@ -24,10 +24,10 @@ func (l *DefaultLayout) Dirname(h restic.Handle) string { p := defaultLayoutPaths[h.Type] if h.Type == restic.DataFile && len(h.Name) > 2 { - p = l.Join(p, h.Name[:2]) + p = l.Join(p, h.Name[:2]) + "/" } - return l.Join(l.Path, p) + return l.Join(l.Path, p) + "/" } // Filename returns a path to a file, including its name. diff --git a/src/restic/backend/layout_rest.go b/src/restic/backend/layout_rest.go new file mode 100644 index 000000000..2d01ece79 --- /dev/null +++ b/src/restic/backend/layout_rest.go @@ -0,0 +1,45 @@ +package backend + +import "restic" + +// RESTLayout implements the default layout for the REST protocol. +type RESTLayout struct { + URL string + Path string + Join func(...string) string +} + +var restLayoutPaths = defaultLayoutPaths + +// Dirname returns the directory path for a given file type and name. +func (l *RESTLayout) Dirname(h restic.Handle) string { + if h.Type == restic.ConfigFile { + return l.URL + l.Join(l.Path, "/") + } + + return l.URL + l.Join(l.Path, "/", restLayoutPaths[h.Type]) + "/" +} + +// Filename returns a path to a file, including its name. +func (l *RESTLayout) Filename(h restic.Handle) string { + name := h.Name + + if h.Type == restic.ConfigFile { + name = "config" + } + + return l.URL + l.Join(l.Path, "/", restLayoutPaths[h.Type], name) +} + +// Paths returns all directory names +func (l *RESTLayout) Paths() (dirs []string) { + for _, p := range restLayoutPaths { + dirs = append(dirs, l.URL+l.Join(l.Path, p)) + } + return dirs +} + +// Basedir returns the base dir name for files of type t. +func (l *RESTLayout) Basedir(t restic.FileType) string { + return l.URL + l.Join(l.Path, restLayoutPaths[t]) +} diff --git a/src/restic/backend/layout_s3.go b/src/restic/backend/layout_s3legacy.go similarity index 74% rename from src/restic/backend/layout_s3.go rename to src/restic/backend/layout_s3legacy.go index 91273d63a..601d29bc5 100644 --- a/src/restic/backend/layout_s3.go +++ b/src/restic/backend/layout_s3legacy.go @@ -2,9 +2,9 @@ package backend import "restic" -// S3Layout implements the old layout used for s3 cloud storage backends, as +// S3LegacyLayout implements the old layout used for s3 cloud storage backends, as // described in the Design document. -type S3Layout struct { +type S3LegacyLayout struct { URL string Path string Join func(...string) string @@ -19,7 +19,7 @@ var s3LayoutPaths = map[restic.FileType]string{ } // join calls Join with the first empty elements removed. -func (l *S3Layout) join(url string, items ...string) string { +func (l *S3LegacyLayout) join(url string, items ...string) string { for len(items) > 0 && items[0] == "" { items = items[1:] } @@ -35,7 +35,7 @@ func (l *S3Layout) join(url string, items ...string) string { } // Dirname returns the directory path for a given file type and name. -func (l *S3Layout) Dirname(h restic.Handle) string { +func (l *S3LegacyLayout) Dirname(h restic.Handle) string { if h.Type == restic.ConfigFile { return l.URL + l.Join(l.Path, "/") } @@ -44,7 +44,7 @@ func (l *S3Layout) Dirname(h restic.Handle) string { } // Filename returns a path to a file, including its name. -func (l *S3Layout) Filename(h restic.Handle) string { +func (l *S3LegacyLayout) Filename(h restic.Handle) string { name := h.Name if h.Type == restic.ConfigFile { @@ -55,7 +55,7 @@ func (l *S3Layout) Filename(h restic.Handle) string { } // Paths returns all directory names -func (l *S3Layout) Paths() (dirs []string) { +func (l *S3LegacyLayout) Paths() (dirs []string) { for _, p := range s3LayoutPaths { dirs = append(dirs, l.Join(l.Path, p)) } @@ -63,6 +63,6 @@ func (l *S3Layout) Paths() (dirs []string) { } // Basedir returns the base dir name for type t. -func (l *S3Layout) Basedir(t restic.FileType) string { +func (l *S3LegacyLayout) Basedir(t restic.FileType) string { return l.Join(l.Path, s3LayoutPaths[t]) } diff --git a/src/restic/backend/layout_test.go b/src/restic/backend/layout_test.go index 606da3241..cd06d4e63 100644 --- a/src/restic/backend/layout_test.go +++ b/src/restic/backend/layout_test.go @@ -79,7 +79,7 @@ func TestDefaultLayout(t *testing.T) { } } -func TestCloudLayout(t *testing.T) { +func TestRESTLayout(t *testing.T) { path, cleanup := TempDir(t) defer cleanup() @@ -113,7 +113,7 @@ func TestCloudLayout(t *testing.T) { }, } - l := &CloudLayout{ + l := &RESTLayout{ Path: path, Join: filepath.Join, } @@ -147,7 +147,7 @@ func TestCloudLayout(t *testing.T) { } } -func TestCloudLayoutURLs(t *testing.T) { +func TestRESTLayoutURLs(t *testing.T) { var tests = []struct { l Layout h restic.Handle @@ -155,55 +155,55 @@ func TestCloudLayoutURLs(t *testing.T) { dir string }{ { - &CloudLayout{URL: "https://hostname.foo", Path: "", Join: path.Join}, + &RESTLayout{URL: "https://hostname.foo", Path: "", Join: path.Join}, restic.Handle{Type: restic.DataFile, Name: "foobar"}, "https://hostname.foo/data/foobar", "https://hostname.foo/data/", }, { - &CloudLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join}, + &RESTLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join}, restic.Handle{Type: restic.LockFile, Name: "foobar"}, "https://hostname.foo:1234/prefix/repo/locks/foobar", "https://hostname.foo:1234/prefix/repo/locks/", }, { - &CloudLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join}, + &RESTLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join}, restic.Handle{Type: restic.ConfigFile, Name: "foobar"}, "https://hostname.foo:1234/prefix/repo/config", "https://hostname.foo:1234/prefix/repo/", }, { - &S3Layout{URL: "https://hostname.foo", Path: "/", Join: path.Join}, + &S3LegacyLayout{URL: "https://hostname.foo", Path: "/", Join: path.Join}, restic.Handle{Type: restic.DataFile, Name: "foobar"}, "https://hostname.foo/data/foobar", "https://hostname.foo/data/", }, { - &S3Layout{URL: "https://hostname.foo:1234/prefix/repo", Path: "", Join: path.Join}, + &S3LegacyLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "", Join: path.Join}, restic.Handle{Type: restic.LockFile, Name: "foobar"}, "https://hostname.foo:1234/prefix/repo/lock/foobar", "https://hostname.foo:1234/prefix/repo/lock/", }, { - &S3Layout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join}, + &S3LegacyLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join}, restic.Handle{Type: restic.ConfigFile, Name: "foobar"}, "https://hostname.foo:1234/prefix/repo/config", "https://hostname.foo:1234/prefix/repo/", }, { - &S3Layout{URL: "", Path: "", Join: path.Join}, + &S3LegacyLayout{URL: "", Path: "", Join: path.Join}, restic.Handle{Type: restic.DataFile, Name: "foobar"}, "data/foobar", "data/", }, { - &S3Layout{URL: "", Path: "", Join: path.Join}, + &S3LegacyLayout{URL: "", Path: "", Join: path.Join}, restic.Handle{Type: restic.LockFile, Name: "foobar"}, "lock/foobar", "lock/", }, { - &S3Layout{URL: "", Path: "/", Join: path.Join}, + &S3LegacyLayout{URL: "", Path: "/", Join: path.Join}, restic.Handle{Type: restic.ConfigFile, Name: "foobar"}, "/config", "/", @@ -225,7 +225,7 @@ func TestCloudLayoutURLs(t *testing.T) { } } -func TestS3Layout(t *testing.T) { +func TestS3LegacyLayout(t *testing.T) { path, cleanup := TempDir(t) defer cleanup() @@ -259,7 +259,7 @@ func TestS3Layout(t *testing.T) { }, } - l := &S3Layout{ + l := &S3LegacyLayout{ Path: path, Join: filepath.Join, } @@ -301,9 +301,8 @@ func TestDetectLayout(t *testing.T) { filename string want string }{ - {"repo-layout-local.tar.gz", "*backend.DefaultLayout"}, - {"repo-layout-cloud.tar.gz", "*backend.CloudLayout"}, - {"repo-layout-s3-old.tar.gz", "*backend.S3Layout"}, + {"repo-layout-default.tar.gz", "*backend.DefaultLayout"}, + {"repo-layout-s3legacy.tar.gz", "*backend.S3LegacyLayout"}, } var fs = &LocalFilesystem{} @@ -342,12 +341,11 @@ func TestParseLayout(t *testing.T) { want string }{ {"default", "", "*backend.DefaultLayout"}, - {"cloud", "", "*backend.CloudLayout"}, - {"s3", "", "*backend.S3Layout"}, - {"", "", "*backend.CloudLayout"}, + {"s3legacy", "", "*backend.S3LegacyLayout"}, + {"", "", "*backend.DefaultLayout"}, } - SetupTarTestFixture(t, path, filepath.Join("testdata", "repo-layout-cloud.tar.gz")) + SetupTarTestFixture(t, path, filepath.Join("testdata", "repo-layout-default.tar.gz")) for _, test := range tests { t.Run(test.layoutName, func(t *testing.T) { diff --git a/src/restic/backend/local/layout_test.go b/src/restic/backend/local/layout_test.go index da0b0bfc8..16b6b16e3 100644 --- a/src/restic/backend/local/layout_test.go +++ b/src/restic/backend/local/layout_test.go @@ -17,17 +17,12 @@ func TestLayout(t *testing.T) { failureExpected bool datafiles map[string]bool }{ - {"repo-layout-local.tar.gz", "", false, map[string]bool{ + {"repo-layout-default.tar.gz", "", false, map[string]bool{ "aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false, "fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false, "c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false, }}, - {"repo-layout-cloud.tar.gz", "", false, map[string]bool{ - "fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false, - "c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false, - "aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false, - }}, - {"repo-layout-s3-old.tar.gz", "", false, map[string]bool{ + {"repo-layout-s3legacy.tar.gz", "", false, map[string]bool{ "fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false, "c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false, "aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false, diff --git a/src/restic/backend/rest/rest.go b/src/restic/backend/rest/rest.go index 5a0e6226b..db1118077 100644 --- a/src/restic/backend/rest/rest.go +++ b/src/restic/backend/rest/rest.go @@ -48,7 +48,7 @@ func Open(cfg Config) (restic.Backend, error) { url: cfg.URL, connChan: connChan, client: client, - Layout: &backend.CloudLayout{URL: url, Join: path.Join}, + Layout: &backend.RESTLayout{URL: url, Join: path.Join}, } return be, nil diff --git a/src/restic/backend/s3/config.go b/src/restic/backend/s3/config.go index 2df02b58c..6cf7db9c1 100644 --- a/src/restic/backend/s3/config.go +++ b/src/restic/backend/s3/config.go @@ -6,6 +6,7 @@ import ( "strings" "restic/errors" + "restic/options" ) // Config contains all configuration necessary to connect to an s3 compatible @@ -16,6 +17,11 @@ type Config struct { KeyID, Secret string Bucket string Prefix string + Layout string `option:"layout" help:"use this backend layout (default: auto-detect)"` +} + +func init() { + options.Register("s3", Config{}) } const defaultPrefix = "restic" diff --git a/src/restic/backend/s3/s3.go b/src/restic/backend/s3/s3.go index 9afd13df5..3b6a60b54 100644 --- a/src/restic/backend/s3/s3.go +++ b/src/restic/backend/s3/s3.go @@ -8,6 +8,7 @@ import ( "restic" "strings" "sync" + "time" "restic/backend" "restic/errors" @@ -30,6 +31,8 @@ type s3 struct { backend.Layout } +const defaultLayout = "s3legacy" + // Open opens the S3 backend at bucket and region. The bucket is created if it // does not exist yet. func Open(cfg Config) (restic.Backend, error) { @@ -45,11 +48,17 @@ func Open(cfg Config) (restic.Backend, error) { bucketname: cfg.Bucket, prefix: cfg.Prefix, cacheObjSize: make(map[string]int64), - Layout: &backend.S3Layout{Path: cfg.Prefix, Join: path.Join}, } client.SetCustomTransport(backend.Transport()) + l, err := backend.ParseLayout(be, cfg.Layout, defaultLayout, cfg.Prefix) + if err != nil { + return nil, err + } + + be.Layout = l + be.createConnections() found, err := client.BucketExists(cfg.Bucket) @@ -76,6 +85,77 @@ func (be *s3) createConnections() { } } +// IsNotExist returns true if the error is caused by a not existing file. +func (be *s3) IsNotExist(err error) bool { + debug.Log("IsNotExist(%T, %#v)", err, err) + if os.IsNotExist(err) { + return true + } + + return false +} + +// Join combines path components with slashes. +func (be *s3) Join(p ...string) string { + return path.Join(p...) +} + +type fileInfo struct { + name string + size int64 + mode os.FileMode + modTime time.Time + isDir bool +} + +func (fi fileInfo) Name() string { return fi.name } // base name of the file +func (fi fileInfo) Size() int64 { return fi.size } // length in bytes for regular files; system-dependent for others +func (fi fileInfo) Mode() os.FileMode { return fi.mode } // file mode bits +func (fi fileInfo) ModTime() time.Time { return fi.modTime } // modification time +func (fi fileInfo) IsDir() bool { return fi.isDir } // abbreviation for Mode().IsDir() +func (fi fileInfo) Sys() interface{} { return nil } // underlying data source (can return nil) + +// ReadDir returns the entries for a directory. +func (be *s3) ReadDir(dir string) (list []os.FileInfo, err error) { + debug.Log("ReadDir(%v)", dir) + + // make sure dir ends with a slash + if dir[len(dir)-1] != '/' { + dir += "/" + } + + done := make(chan struct{}) + defer close(done) + + for obj := range be.client.ListObjects(be.bucketname, dir, false, done) { + if obj.Key == "" { + continue + } + + name := strings.TrimPrefix(obj.Key, dir) + if name == "" { + return nil, errors.Errorf("invalid key name %v, removing prefix %v yielded empty string", obj.Key, dir) + } + entry := fileInfo{ + name: name, + size: obj.Size, + modTime: obj.LastModified, + } + + if name[len(name)-1] == '/' { + entry.isDir = true + entry.mode = os.ModeDir | 0755 + entry.name = name[:len(name)-1] + } else { + entry.mode = 0644 + } + + list = append(list, entry) + } + + return list, nil +} + // Location returns this backend's location (the bucket name). func (be *s3) Location() string { return be.bucketname @@ -290,6 +370,11 @@ func (be *s3) List(t restic.FileType, done <-chan struct{}) <-chan string { prefix := be.Dirname(restic.Handle{Type: t}) + // make sure prefix ends with a slash + if prefix[len(prefix)-1] != '/' { + prefix += "/" + } + listresp := be.client.ListObjects(be.bucketname, prefix, true, done) go func() { diff --git a/src/restic/backend/sftp/layout_test.go b/src/restic/backend/sftp/layout_test.go index 856bd5ce4..166fa97e3 100644 --- a/src/restic/backend/sftp/layout_test.go +++ b/src/restic/backend/sftp/layout_test.go @@ -23,17 +23,12 @@ func TestLayout(t *testing.T) { failureExpected bool datafiles map[string]bool }{ - {"repo-layout-local.tar.gz", "", false, map[string]bool{ + {"repo-layout-default.tar.gz", "", false, map[string]bool{ "aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false, "fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false, "c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false, }}, - {"repo-layout-cloud.tar.gz", "", false, map[string]bool{ - "fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false, - "c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false, - "aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false, - }}, - {"repo-layout-s3-old.tar.gz", "", false, map[string]bool{ + {"repo-layout-s3legacy.tar.gz", "", false, map[string]bool{ "fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false, "c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false, "aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false, diff --git a/src/restic/backend/testdata/repo-layout-cloud.tar.gz b/src/restic/backend/testdata/repo-layout-cloud.tar.gz deleted file mode 100644 index 189832589..000000000 Binary files a/src/restic/backend/testdata/repo-layout-cloud.tar.gz and /dev/null differ diff --git a/src/restic/backend/testdata/repo-layout-local.tar.gz b/src/restic/backend/testdata/repo-layout-default.tar.gz similarity index 100% rename from src/restic/backend/testdata/repo-layout-local.tar.gz rename to src/restic/backend/testdata/repo-layout-default.tar.gz diff --git a/src/restic/backend/testdata/repo-layout-s3-old.tar.gz b/src/restic/backend/testdata/repo-layout-s3legacy.tar.gz similarity index 100% rename from src/restic/backend/testdata/repo-layout-s3-old.tar.gz rename to src/restic/backend/testdata/repo-layout-s3legacy.tar.gz