octoleo/restic
2
2
Fork 0
mirror of https://github.com/octoleo/restic.git synced 2024-11-22 12:55:18 +00:00
Code Issues Releases Wiki Activity

drop support for s3legacy layout

Browse Source
This commit is contained in:
Michael Eischer 2024-08-26 20:28:39 +02:00
parent 943b6ccfba
commit 6024597028
19 changed files with 34 additions and 698 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
cmd/restic/cmd_check.go
View File

@ -296,9 +296,6 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
errorsFound = true
printer.E("%v\n", err)
}
} else if err == checker.ErrLegacyLayout {
errorsFound = true
printer.E("error: repository still uses the S3 legacy layout\nYou must run `restic migrate s3legacy` to correct this.\n")
} else {
errorsFound = true
printer.E("%v\n", err)

36
cmd/restic/cmd_restore_integration_test.go
View File

@ -12,7 +12,6 @@ import (
"testing"
"time"
"github.com/restic/restic/internal/feature"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
"github.com/restic/restic/internal/ui/termstatus"
@ -403,36 +402,21 @@ func TestRestoreNoMetadataOnIgnoredIntermediateDirs(t *testing.T) {
"meta data of intermediate directory hasn't been restore")
}
func TestRestoreLocalLayout(t *testing.T) {
defer feature.TestSetFlag(t, feature.Flag, feature.DeprecateS3LegacyLayout, false)()
func TestRestoreDefaultLayout(t *testing.T) {
env, cleanup := withTestEnvironment(t)
defer cleanup()
var tests = []struct {
filename string
layout string
}{
{"repo-layout-default.tar.gz", ""},
{"repo-layout-s3legacy.tar.gz", ""},
{"repo-layout-default.tar.gz", "default"},
{"repo-layout-s3legacy.tar.gz", "s3legacy"},
}
datafile := filepath.Join("..", "..", "internal", "backend", "testdata", "repo-layout-default.tar.gz")
for _, test := range tests {
datafile := filepath.Join("..", "..", "internal", "backend", "testdata", test.filename)
rtest.SetupTarTestFixture(t, env.base, datafile)
rtest.SetupTarTestFixture(t, env.base, datafile)
// check the repo
testRunCheck(t, env.gopts)
env.gopts.extended["local.layout"] = test.layout
// restore latest snapshot
target := filepath.Join(env.base, "restore")
testRunRestoreLatest(t, env.gopts, target, nil, nil)
// check the repo
testRunCheck(t, env.gopts)
// restore latest snapshot
target := filepath.Join(env.base, "restore")
testRunRestoreLatest(t, env.gopts, target, nil, nil)
rtest.RemoveAll(t, filepath.Join(env.base, "repo"))
rtest.RemoveAll(t, target)
}
rtest.RemoveAll(t, filepath.Join(env.base, "repo"))
rtest.RemoveAll(t, target)
}

11
doc/design.rst
View File

@ -119,16 +119,11 @@ A local repository can be initialized with the ``restic init`` command, e.g.:
$ restic -r /tmp/restic-repo init
The local and sftp backends will auto-detect and accept all layouts described
in the following sections, so that remote repositories mounted locally e.g. via
fuse can be accessed. The layout auto-detection can be overridden by specifying
the option ``-o local.layout=default``, valid values are ``default`` and
``s3legacy``. The option for the sftp backend is named ``sftp.layout``, for the
s3 backend ``s3.layout``.
S3 Legacy Layout (deprecated)
-----------------------------
Restic 0.17 is the last version that supports the legacy layout.
Unfortunately during development the Amazon S3 backend uses slightly different
paths (directory names use singular instead of plural for ``key``,
``lock``, and ``snapshot`` files), and the pack files are stored directly below
@ -152,8 +147,6 @@ the ``data`` directory. The S3 Legacy repository layout looks like this:
/snapshot
└── 22a5af1bdc6e616f8a29579458c49627e01b32210d09adb288d1ecda7c5711ec
Restic 0.17 is the last version that supports the legacy layout.
Pack Format
===========

167
internal/backend/layout/layout.go
View File

@ -1,18 +1,7 @@
package layout
import (
"context"
"fmt"
"os"
"path/filepath"
"regexp"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/feature"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/restic"
)
// Layout computes paths for file name storage.
@ -23,159 +12,3 @@ type Layout interface {
Paths() []string
Name() string
}
// Filesystem is the abstraction of a file system used for a backend.
type Filesystem interface {
Join(...string) string
ReadDir(context.Context, string) ([]os.FileInfo, error)
IsNotExist(error) bool
}
// ensure statically that *LocalFilesystem implements Filesystem.
var _ Filesystem = &LocalFilesystem{}
// LocalFilesystem implements Filesystem in a local path.
type LocalFilesystem struct {
}
// ReadDir returns all entries of a directory.
func (l *LocalFilesystem) ReadDir(_ context.Context, dir string) ([]os.FileInfo, error) {
f, err := fs.Open(dir)
if err != nil {
return nil, err
}
entries, err := f.Readdir(-1)
if err != nil {
return nil, errors.Wrap(err, "Readdir")
}
err = f.Close()
if err != nil {
return nil, errors.Wrap(err, "Close")
}
return entries, nil
}
// Join combines several path components to one.
func (l *LocalFilesystem) Join(paths ...string) string {
return filepath.Join(paths...)
}
// IsNotExist returns true for errors that are caused by not existing files.
func (l *LocalFilesystem) IsNotExist(err error) bool {
return os.IsNotExist(err)
}
var backendFilenameLength = len(restic.ID{}) * 2
var backendFilename = regexp.MustCompile(fmt.Sprintf("^[a-fA-F0-9]{%d}$", backendFilenameLength))
func hasBackendFile(ctx context.Context, fs Filesystem, dir string) (bool, error) {
entries, err := fs.ReadDir(ctx, dir)
if err != nil && fs.IsNotExist(err) {
return false, nil
}
if err != nil {
return false, errors.Wrap(err, "ReadDir")
}
for _, e := range entries {
if backendFilename.MatchString(e.Name()) {
return true, nil
}
}
return false, nil
}
// ErrLayoutDetectionFailed is returned by DetectLayout() when the layout
// cannot be detected automatically.
var ErrLayoutDetectionFailed = errors.New("auto-detecting the filesystem layout failed")
var ErrLegacyLayoutFound = errors.New("detected legacy S3 layout. Use `RESTIC_FEATURES=deprecate-s3-legacy-layout=false restic migrate s3_layout` to migrate your repository")
// DetectLayout tries to find out which layout is used in a local (or sftp)
// filesystem at the given path. If repo is nil, an instance of LocalFilesystem
// is used.
func DetectLayout(ctx context.Context, repo Filesystem, dir string) (Layout, error) {
debug.Log("detect layout at %v", dir)
if repo == nil {
repo = &LocalFilesystem{}
}
// key file in the "keys" dir (DefaultLayout)
foundKeysFile, err := hasBackendFile(ctx, repo, repo.Join(dir, defaultLayoutPaths[backend.KeyFile]))
if err != nil {
return nil, err
}
// key file in the "key" dir (S3LegacyLayout)
foundKeyFile, err := hasBackendFile(ctx, repo, repo.Join(dir, s3LayoutPaths[backend.KeyFile]))
if err != nil {
return nil, err
}
if foundKeysFile && !foundKeyFile {
debug.Log("found default layout at %v", dir)
return &DefaultLayout{
Path: dir,
Join: repo.Join,
}, nil
}
if foundKeyFile && !foundKeysFile {
if feature.Flag.Enabled(feature.DeprecateS3LegacyLayout) {
return nil, ErrLegacyLayoutFound
}
debug.Log("found s3 layout at %v", dir)
return &S3LegacyLayout{
Path: dir,
Join: repo.Join,
}, nil
}
debug.Log("layout detection failed")
return nil, ErrLayoutDetectionFailed
}
// ParseLayout parses the config string and returns a Layout. When layout is
// the empty string, DetectLayout is used. If that fails, defaultLayout is used.
func ParseLayout(ctx context.Context, repo Filesystem, layout, defaultLayout, path string) (l Layout, err error) {
debug.Log("parse layout string %q for backend at %v", layout, path)
switch layout {
case "default":
l = &DefaultLayout{
Path: path,
Join: repo.Join,
}
case "s3legacy":
if feature.Flag.Enabled(feature.DeprecateS3LegacyLayout) {
return nil, ErrLegacyLayoutFound
}
l = &S3LegacyLayout{
Path: path,
Join: repo.Join,
}
case "":
l, err = DetectLayout(ctx, repo, path)
// use the default layout if auto detection failed
if errors.Is(err, ErrLayoutDetectionFailed) && defaultLayout != "" {
debug.Log("error: %v, use default layout %v", err, defaultLayout)
return ParseLayout(ctx, repo, defaultLayout, "", path)
}
if err != nil {
return nil, err
}
debug.Log("layout detected: %v", l)
default:
return nil, errors.Errorf("unknown backend layout string %q, may be one of: default, s3legacy", layout)
}
return l, nil
}

7
internal/backend/layout/layout_default.go
View File

@ -23,6 +23,13 @@ var defaultLayoutPaths = map[backend.FileType]string{
backend.KeyFile: "keys",
}
func NewDefaultLayout(path string, join func(...string) string) *DefaultLayout {
return &DefaultLayout{
Path: path,
Join: join,
}
}
func (l *DefaultLayout) String() string {
return "<DefaultLayout>"
}

79
internal/backend/layout/layout_s3legacy.go
View File

@ -1,79 +0,0 @@
package layout
import (
"github.com/restic/restic/internal/backend"
)
// S3LegacyLayout implements the old layout used for s3 cloud storage backends, as
// described in the Design document.
type S3LegacyLayout struct {
URL string
Path string
Join func(...string) string
}
var s3LayoutPaths = map[backend.FileType]string{
backend.PackFile: "data",
backend.SnapshotFile: "snapshot",
backend.IndexFile: "index",
backend.LockFile: "lock",
backend.KeyFile: "key",
}
func (l *S3LegacyLayout) String() string {
return "<S3LegacyLayout>"
}
// Name returns the name for this layout.
func (l *S3LegacyLayout) Name() string {
return "s3legacy"
}
// join calls Join with the first empty elements removed.
func (l *S3LegacyLayout) join(url string, items ...string) string {
for len(items) > 0 && items[0] == "" {
items = items[1:]
}
path := l.Join(items...)
if path == "" || path[0] != '/' {
if url != "" && url[len(url)-1] != '/' {
url += "/"
}
}
return url + path
}
// Dirname returns the directory path for a given file type and name.
func (l *S3LegacyLayout) Dirname(h backend.Handle) string {
if h.Type == backend.ConfigFile {
return l.URL + l.Join(l.Path, "/")
}
return l.join(l.URL, l.Path, s3LayoutPaths[h.Type]) + "/"
}
// Filename returns a path to a file, including its name.
func (l *S3LegacyLayout) Filename(h backend.Handle) string {
name := h.Name
if h.Type == backend.ConfigFile {
name = "config"
}
return l.join(l.URL, l.Path, s3LayoutPaths[h.Type], name)
}
// Paths returns all directory names
func (l *S3LegacyLayout) Paths() (dirs []string) {
for _, p := range s3LayoutPaths {
dirs = append(dirs, l.Join(l.Path, p))
}
return dirs
}
// Basedir returns the base dir name for type t.
func (l *S3LegacyLayout) Basedir(t backend.FileType) (dirname string, subdirs bool) {
return l.Join(l.Path, s3LayoutPaths[t]), false
}

200
internal/backend/layout/layout_test.go
View File

@ -1,7 +1,6 @@
package layout
import (
"context"
"fmt"
"path"
"path/filepath"
@ -10,7 +9,6 @@ import (
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/feature"
rtest "github.com/restic/restic/internal/test"
)
@ -232,42 +230,6 @@ func TestRESTLayoutURLs(t *testing.T) {
"https://hostname.foo:1234/prefix/repo/config",
"https://hostname.foo:1234/prefix/repo/",
},
{
&S3LegacyLayout{URL: "https://hostname.foo", Path: "/", Join: path.Join},
backend.Handle{Type: backend.PackFile, Name: "foobar"},
"https://hostname.foo/data/foobar",
"https://hostname.foo/data/",
},
{
&S3LegacyLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "", Join: path.Join},
backend.Handle{Type: backend.LockFile, Name: "foobar"},
"https://hostname.foo:1234/prefix/repo/lock/foobar",
"https://hostname.foo:1234/prefix/repo/lock/",
},
{
&S3LegacyLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join},
backend.Handle{Type: backend.ConfigFile, Name: "foobar"},
"https://hostname.foo:1234/prefix/repo/config",
"https://hostname.foo:1234/prefix/repo/",
},
{
&S3LegacyLayout{URL: "", Path: "", Join: path.Join},
backend.Handle{Type: backend.PackFile, Name: "foobar"},
"data/foobar",
"data/",
},
{
&S3LegacyLayout{URL: "", Path: "", Join: path.Join},
backend.Handle{Type: backend.LockFile, Name: "foobar"},
"lock/foobar",
"lock/",
},
{
&S3LegacyLayout{URL: "", Path: "/", Join: path.Join},
backend.Handle{Type: backend.ConfigFile, Name: "foobar"},
"/config",
"/",
},
}
for _, test := range tests {
@ -284,165 +246,3 @@ func TestRESTLayoutURLs(t *testing.T) {
})
}
}
func TestS3LegacyLayout(t *testing.T) {
path := rtest.TempDir(t)
var tests = []struct {
backend.Handle
filename string
}{
{
backend.Handle{Type: backend.PackFile, Name: "0123456"},
filepath.Join(path, "data", "0123456"),
},
{
backend.Handle{Type: backend.ConfigFile, Name: "CFG"},
filepath.Join(path, "config"),
},
{
backend.Handle{Type: backend.SnapshotFile, Name: "123456"},
filepath.Join(path, "snapshot", "123456"),
},
{
backend.Handle{Type: backend.IndexFile, Name: "123456"},
filepath.Join(path, "index", "123456"),
},
{
backend.Handle{Type: backend.LockFile, Name: "123456"},
filepath.Join(path, "lock", "123456"),
},
{
backend.Handle{Type: backend.KeyFile, Name: "123456"},
filepath.Join(path, "key", "123456"),
},
}
l := &S3LegacyLayout{
Path: path,
Join: filepath.Join,
}
t.Run("Paths", func(t *testing.T) {
dirs := l.Paths()
want := []string{
filepath.Join(path, "data"),
filepath.Join(path, "snapshot"),
filepath.Join(path, "index"),
filepath.Join(path, "lock"),
filepath.Join(path, "key"),
}
sort.Strings(want)
sort.Strings(dirs)
if !reflect.DeepEqual(dirs, want) {
t.Fatalf("wrong paths returned, want:\n %v\ngot:\n %v", want, dirs)
}
})
for _, test := range tests {
t.Run(fmt.Sprintf("%v/%v", test.Type, test.Handle.Name), func(t *testing.T) {
filename := l.Filename(test.Handle)
if filename != test.filename {
t.Fatalf("wrong filename, want %v, got %v", test.filename, filename)
}
})
}
}
func TestDetectLayout(t *testing.T) {
defer feature.TestSetFlag(t, feature.Flag, feature.DeprecateS3LegacyLayout, false)()
path := rtest.TempDir(t)
var tests = []struct {
filename string
want string
}{
{"repo-layout-default.tar.gz", "*layout.DefaultLayout"},
{"repo-layout-s3legacy.tar.gz", "*layout.S3LegacyLayout"},
}
var fs = &LocalFilesystem{}
for _, test := range tests {
for _, fs := range []Filesystem{fs, nil} {
t.Run(fmt.Sprintf("%v/fs-%T", test.filename, fs), func(t *testing.T) {
rtest.SetupTarTestFixture(t, path, filepath.Join("../testdata", test.filename))
layout, err := DetectLayout(context.TODO(), fs, filepath.Join(path, "repo"))
if err != nil {
t.Fatal(err)
}
if layout == nil {
t.Fatal("wanted some layout, but detect returned nil")
}
layoutName := fmt.Sprintf("%T", layout)
if layoutName != test.want {
t.Fatalf("want layout %v, got %v", test.want, layoutName)
}
rtest.RemoveAll(t, filepath.Join(path, "repo"))
})
}
}
}
func TestParseLayout(t *testing.T) {
defer feature.TestSetFlag(t, feature.Flag, feature.DeprecateS3LegacyLayout, false)()
path := rtest.TempDir(t)
var tests = []struct {
layoutName string
defaultLayoutName string
want string
}{
{"default", "", "*layout.DefaultLayout"},
{"s3legacy", "", "*layout.S3LegacyLayout"},
{"", "", "*layout.DefaultLayout"},
}
rtest.SetupTarTestFixture(t, path, filepath.Join("..", "testdata", "repo-layout-default.tar.gz"))
for _, test := range tests {
t.Run(test.layoutName, func(t *testing.T) {
layout, err := ParseLayout(context.TODO(), &LocalFilesystem{}, test.layoutName, test.defaultLayoutName, filepath.Join(path, "repo"))
if err != nil {
t.Fatal(err)
}
if layout == nil {
t.Fatal("wanted some layout, but detect returned nil")
}
// test that the functions work (and don't panic)
_ = layout.Dirname(backend.Handle{Type: backend.PackFile})
_ = layout.Filename(backend.Handle{Type: backend.PackFile, Name: "1234"})
_ = layout.Paths()
layoutName := fmt.Sprintf("%T", layout)
if layoutName != test.want {
t.Fatalf("want layout %v, got %v", test.want, layoutName)
}
})
}
}
func TestParseLayoutInvalid(t *testing.T) {
path := rtest.TempDir(t)
var invalidNames = []string{
"foo", "bar", "local",
}
for _, name := range invalidNames {
t.Run(name, func(t *testing.T) {
layout, err := ParseLayout(context.TODO(), nil, name, "", path)
if err == nil {
t.Fatalf("expected error not found for layout name %v, layout is %v", name, layout)
}
})
}
}

3
internal/backend/local/config.go
View File

@ -9,8 +9,7 @@ import (
// Config holds all information needed to open a local repository.
type Config struct {
Path string
Layout string `option:"layout" help:"use this backend directory layout (default: auto-detect) (deprecated)"`
Path string
Connections uint `option:"connections" help:"set a limit for the number of concurrent operations (default: 2)"`
}

11
internal/backend/local/layout_test.go
View File

@ -6,30 +6,22 @@ import (
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/feature"
rtest "github.com/restic/restic/internal/test"
)
func TestLayout(t *testing.T) {
defer feature.TestSetFlag(t, feature.Flag, feature.DeprecateS3LegacyLayout, false)()
path := rtest.TempDir(t)
var tests = []struct {
filename string
layout string
failureExpected bool
packfiles map[string]bool
}{
{"repo-layout-default.tar.gz", "", false, map[string]bool{
{"repo-layout-default.tar.gz", false, map[string]bool{
"aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false,
"fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false,
"c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false,
}},
{"repo-layout-s3legacy.tar.gz", "", false, map[string]bool{
"fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false,
"c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false,
"aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false,
}},
}
for _, test := range tests {
@ -39,7 +31,6 @@ func TestLayout(t *testing.T) {
repo := filepath.Join(path, "repo")
be, err := Open(context.TODO(), Config{
Path: repo,
Layout: test.layout,
Connections: 2,
})
if err != nil {

11
internal/backend/local/local.go
View File

@ -37,13 +37,8 @@ func NewFactory() location.Factory {
return location.NewLimitedBackendFactory("local", ParseConfig, location.NoPassword, limiter.WrapBackendConstructor(Create), limiter.WrapBackendConstructor(Open))
}
const defaultLayout = "default"
func open(ctx context.Context, cfg Config) (*Local, error) {
l, err := layout.ParseLayout(ctx, &layout.LocalFilesystem{}, cfg.Layout, defaultLayout, cfg.Path)
if err != nil {
return nil, err
}
l := layout.NewDefaultLayout(cfg.Path, filepath.Join)
fi, err := fs.Stat(l.Filename(backend.Handle{Type: backend.ConfigFile}))
m := util.DeriveModesFromFileInfo(fi, err)
@ -58,14 +53,14 @@ func open(ctx context.Context, cfg Config) (*Local, error) {
// Open opens the local backend as specified by config.
func Open(ctx context.Context, cfg Config) (*Local, error) {
debug.Log("open local backend at %v (layout %q)", cfg.Path, cfg.Layout)
debug.Log("open local backend at %v", cfg.Path)
return open(ctx, cfg)
}
// Create creates all the necessary files and directories for a new local
// backend at dir. Afterwards a new config blob should be created.
func Create(ctx context.Context, cfg Config) (*Local, error) {
debug.Log("create local backend at %v (layout %q)", cfg.Path, cfg.Layout)
debug.Log("create local backend at %v", cfg.Path)
be, err := open(ctx, cfg)
if err != nil {

10
internal/backend/s3/s3.go
View File

@ -37,8 +37,6 @@ func NewFactory() location.Factory {
return location.NewHTTPBackendFactory("s3", ParseConfig, location.NoPassword, Create, Open)
}
const defaultLayout = "default"
func open(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, error) {
debug.Log("open, config %#v", cfg)
@ -83,15 +81,9 @@ func open(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, erro
be := &Backend{
client: client,
cfg: cfg,
Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
}
l, err := layout.ParseLayout(ctx, be, cfg.Layout, defaultLayout, cfg.Prefix)
if err != nil {
return nil, err
}
be.Layout = l
return be, nil
}

1
internal/backend/sftp/config.go
View File

@ -13,7 +13,6 @@ import (
type Config struct {
User, Host, Port, Path string
Layout string `option:"layout" help:"use this backend directory layout (default: auto-detect) (deprecated)"`
Command string `option:"command" help:"specify command to create sftp connection"`
Args string `option:"args" help:"specify arguments for ssh"`

11
internal/backend/sftp/layout_test.go
View File

@ -8,7 +8,6 @@ import (
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/sftp"
"github.com/restic/restic/internal/feature"
rtest "github.com/restic/restic/internal/test"
)
@ -17,25 +16,18 @@ func TestLayout(t *testing.T) {
t.Skip("sftp server binary not available")
}
defer feature.TestSetFlag(t, feature.Flag, feature.DeprecateS3LegacyLayout, false)()
path := rtest.TempDir(t)
var tests = []struct {
filename string
layout string
failureExpected bool
packfiles map[string]bool
}{
{"repo-layout-default.tar.gz", "", false, map[string]bool{
{"repo-layout-default.tar.gz", false, map[string]bool{
"aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false,
"fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false,
"c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false,
}},
{"repo-layout-s3legacy.tar.gz", "", false, map[string]bool{
"fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false,
"c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false,
"aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false,
}},
}
for _, test := range tests {
@ -46,7 +38,6 @@ func TestLayout(t *testing.T) {
be, err := sftp.Open(context.TODO(), sftp.Config{
Command: fmt.Sprintf("%q -e", sftpServer),
Path: repo,
Layout: test.layout,
Connections: 5,
})
if err != nil {

28
internal/backend/sftp/sftp.go
View File

@ -121,7 +121,13 @@ func startClient(cfg Config) (*SFTP, error) {
}
_, posixRename := client.HasExtension("posix-rename@openssh.com")
return &SFTP{c: client, cmd: cmd, result: ch, posixRename: posixRename}, nil
return &SFTP{
c: client,
cmd: cmd,
result: ch,
posixRename: posixRename,
Layout: layout.NewDefaultLayout(cfg.Path, path.Join),
}, nil
}
// clientError returns an error if the client has exited. Otherwise, nil is
@ -152,14 +158,6 @@ func Open(ctx context.Context, cfg Config) (*SFTP, error) {
}
func open(ctx context.Context, sftp *SFTP, cfg Config) (*SFTP, error) {
var err error
sftp.Layout, err = layout.ParseLayout(ctx, sftp, cfg.Layout, defaultLayout, cfg.Path)
if err != nil {
return nil, err
}
debug.Log("layout: %v\n", sftp.Layout)
fi, err := sftp.c.Stat(sftp.Layout.Filename(backend.Handle{Type: backend.ConfigFile}))
m := util.DeriveModesFromFileInfo(fi, err)
debug.Log("using (%03O file, %03O dir) permissions", m.File, m.Dir)
@ -195,11 +193,6 @@ func (r *SFTP) mkdirAllDataSubdirs(ctx context.Context, nconn uint) error {
return g.Wait()
}
// Join combines path components with slashes (according to the sftp spec).
func (r *SFTP) Join(p ...string) string {
return path.Join(p...)
}
// ReadDir returns the entries for a directory.
func (r *SFTP) ReadDir(_ context.Context, dir string) ([]os.FileInfo, error) {
fi, err := r.c.ReadDir(dir)
@ -266,11 +259,6 @@ func Create(ctx context.Context, cfg Config) (*SFTP, error) {
return nil, err
}
sftp.Layout, err = layout.ParseLayout(ctx, sftp, cfg.Layout, defaultLayout, cfg.Path)
if err != nil {
return nil, err
}
sftp.Modes = util.DefaultModes
// test if config file already exists
@ -582,7 +570,7 @@ func (r *SFTP) deleteRecursive(ctx context.Context, name string) error {
return ctx.Err()
}
itemName := r.Join(name, fi.Name())
itemName := path.Join(name, fi.Name())
if fi.IsDir() {
err := r.deleteRecursive(ctx, itemName)
if err != nil {

BIN
internal/backend/testdata/repo-layout-s3legacy.tar.gz vendored
View File

Binary file not shown.

17
internal/checker/checker.go
View File

@ -8,8 +8,6 @@ import (
"sync"
"github.com/klauspost/compress/zstd"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/s3"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/repository"
@ -53,9 +51,6 @@ func New(repo restic.Repository, trackUnused bool) *Checker {
return c
}
// ErrLegacyLayout is returned when the repository uses the S3 legacy layout.
var ErrLegacyLayout = errors.New("repository uses S3 legacy layout")
// ErrDuplicatePacks is returned when a pack is found in more than one index.
type ErrDuplicatePacks struct {
PackID restic.ID
@ -177,23 +172,11 @@ func (e *PackError) Error() string {
return "pack " + e.ID.String() + ": " + e.Err.Error()
}
func isS3Legacy(b backend.Backend) bool {
be := backend.AsBackend[*s3.Backend](b)
return be != nil && be.Layout.Name() == "s3legacy"
}
// Packs checks that all packs referenced in the index are still available and
// there are no packs that aren't in an index. errChan is closed after all
// packs have been checked.
func (c *Checker) Packs(ctx context.Context, errChan chan<- error) {
defer close(errChan)
if r, ok := c.repo.(*repository.Repository); ok {
if isS3Legacy(repository.AsS3Backend(r)) {
errChan <- ErrLegacyLayout
}
}
debug.Log("checking for %d packs", len(c.packs))
debug.Log("listing repository packs")

2
internal/feature/registry.go
View File

@ -6,7 +6,6 @@ var Flag = New()
// flag names are written in kebab-case
const (
BackendErrorRedesign FlagName = "backend-error-redesign"
DeprecateS3LegacyLayout FlagName = "deprecate-s3-legacy-layout"
DeviceIDForHardlinks FlagName = "device-id-for-hardlinks"
ExplicitS3AnonymousAuth FlagName = "explicit-s3-anonymous-auth"
SafeForgetKeepTags FlagName = "safe-forget-keep-tags"
@ -15,7 +14,6 @@ const (
func init() {
Flag.SetFlags(map[FlagName]FlagDesc{
BackendErrorRedesign: {Type: Beta, Description: "enforce timeouts for stuck HTTP requests and use new backend error handling design."},
DeprecateS3LegacyLayout: {Type: Beta, Description: "disable support for S3 legacy layout used up to restic 0.7.0. Use `RESTIC_FEATURES=deprecate-s3-legacy-layout=false restic migrate s3_layout` to migrate your S3 repository if necessary."},
DeviceIDForHardlinks: {Type: Alpha, Description: "store deviceID only for hardlinks to reduce metadata changes for example when using btrfs subvolumes. Will be removed in a future restic version after repository format 3 is available"},
ExplicitS3AnonymousAuth: {Type: Beta, Description: "forbid anonymous S3 authentication unless `-o s3.unsafe-anonymous-auth=true` is set"},
SafeForgetKeepTags: {Type: Beta, Description: "prevent deleting all snapshots if the tag passed to `forget --keep-tags tagname` does not exist"},

123
internal/migrations/s3_layout.go
View File

@ -1,123 +0,0 @@
package migrations
import (
"context"
"fmt"
"os"
"path"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/s3"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
)
func init() {
register(&S3Layout{})
}
// S3Layout migrates a repository on an S3 backend from the "s3legacy" to the
// "default" layout.
type S3Layout struct{}
// Check tests whether the migration can be applied.
func (m *S3Layout) Check(_ context.Context, repo restic.Repository) (bool, string, error) {
be := repository.AsS3Backend(repo.(*repository.Repository))
if be == nil {
debug.Log("backend is not s3")
return false, "backend is not s3", nil
}
if be.Layout.Name() != "s3legacy" {
debug.Log("layout is not s3legacy")
return false, "not using the legacy s3 layout", nil
}
return true, "", nil
}
func (m *S3Layout) RepoCheck() bool {
return false
}
func retry(max int, fail func(err error), f func() error) error {
var err error
for i := 0; i < max; i++ {
err = f()
if err == nil {
return nil
}
if fail != nil {
fail(err)
}
}
return err
}
// maxErrors for retrying renames on s3.
const maxErrors = 20
func (m *S3Layout) moveFiles(ctx context.Context, be *s3.Backend, l layout.Layout, t restic.FileType) error {
printErr := func(err error) {
fmt.Fprintf(os.Stderr, "renaming file returned error: %v\n", err)
}
return be.List(ctx, t, func(fi backend.FileInfo) error {
h := backend.Handle{Type: t, Name: fi.Name}
debug.Log("move %v", h)
return retry(maxErrors, printErr, func() error {
return be.Rename(ctx, h, l)
})
})
}
// Apply runs the migration.
func (m *S3Layout) Apply(ctx context.Context, repo restic.Repository) error {
be := repository.AsS3Backend(repo.(*repository.Repository))
if be == nil {
debug.Log("backend is not s3")
return errors.New("backend is not s3")
}
oldLayout := &layout.S3LegacyLayout{
Path: be.Path(),
Join: path.Join,
}
newLayout := &layout.DefaultLayout{
Path: be.Path(),
Join: path.Join,
}
be.Layout = oldLayout
for _, t := range []restic.FileType{
restic.SnapshotFile,
restic.PackFile,
restic.KeyFile,
restic.LockFile,
} {
err := m.moveFiles(ctx, be, newLayout, t)
if err != nil {
return err
}
}
be.Layout = newLayout
return nil
}
// Name returns the name for this migration.
func (m *S3Layout) Name() string {
return "s3_layout"
}
// Desc returns a short description what the migration does.
func (m *S3Layout) Desc() string {
return "move files from 's3legacy' to the 'default' repository layout"
}

12
internal/repository/s3_backend.go
View File

@ -1,12 +0,0 @@
package repository
import (
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/s3"
)
// AsS3Backend extracts the S3 backend from a repository
// TODO remove me once restic 0.17 was released
func AsS3Backend(repo *Repository) *s3.Backend {
return backend.AsBackend[*s3.Backend](repo.be)
}