Merge pull request #1075 from restic/migrate-s3-continue

s3: Improve migration to new layout

CHANGELOG.md

@@ -4,6 +4,12 @@ released version of restic from the perspective of the user.
 Important Changes in 0.X.Y
 ==========================
 
+ * The `migrate` command for chaning the `s3legacy` layout to the `default`
+   layout for s3 backends has been improved: It can now be restarted with
+   `restic migrate --force s3_layout` and automatically retries operations on
+   error.
+   https://github.com/restic/restic/issues/1073
+   https://github.com/restic/restic/pull/1075
 
 Important Changes in 0.7.0
 ==========================

src/cmds/restic/cmd_migrate.go

@@ -21,12 +21,15 @@ name is explicitely given, a list of migrations that can be applied is printed.
 
 // MigrateOptions bundles all options for the 'check' command.
 type MigrateOptions struct {
+	Force bool
 }
 
 var migrateOptions MigrateOptions
 
 func init() {
 	cmdRoot.AddCommand(cmdMigrate)
+	f := cmdMigrate.Flags()
+	f.BoolVarP(&migrateOptions.Force, "force", "f", false, `apply a migration a second time`)
 }
 
 func checkMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repository) error {
@@ -59,10 +62,14 @@ func applyMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repos
 				}
 
 				if !ok {
-					Warnf("migration %v cannot be applied: check failed\n", m.Name())
+					if !opts.Force {
+						Warnf("migration %v cannot be applied: check failed\nIf you want to apply this migration anyway, re-run with option --force\n", m.Name())
 						continue
 					}
 
+					Warnf("check for migration %v failed, continuing anyway\n", m.Name())
+				}
+
 				Printf("applying migration %v...\n", m.Name())
 				if err = m.Apply(ctx, repo); err != nil {
 					Warnf("migration %v failed: %v\n", m.Name(), err)

src/restic/backend/s3/s3.go

@@ -453,10 +453,20 @@ func (be *Backend) Rename(h restic.Handle, l backend.Layout) error {
 	oldname := be.Filename(h)
 	newname := l.Filename(h)
 
+	if oldname == newname {
+		debug.Log("  %v is already renamed", newname)
+		return nil
+	}
+
 	debug.Log("  %v -> %v", oldname, newname)
 
 	coreClient := minio.Core{Client: be.client}
 	err := coreClient.CopyObject(be.cfg.Bucket, newname, path.Join(be.cfg.Bucket, oldname), minio.CopyConditions{})
+	if err != nil && be.IsNotExist(err) {
+		debug.Log("copy failed: %v, seems to already have been renamed", err)
+		return nil
+	}
+
 	if err != nil {
 		debug.Log("copy failed: %v", err)
 		return err

src/restic/migrations/s3_layout.go

@@ -2,6 +2,8 @@ package migrations
 
 import (
 	"context"
+	"fmt"
+	"os"
 	"path"
 	"restic"
 	"restic/backend"
@@ -34,13 +36,35 @@ func (m *S3Layout) Check(ctx context.Context, repo restic.Repository) (bool, err
 	return true, nil
 }
 
+func retry(max int, fail func(err error), f func() error) error {
+	var err error
+	for i := 0; i < max; i++ {
+		err = f()
+		if err == nil {
+			return err
+		}
+		if fail != nil {
+			fail(err)
+		}
+	}
+	return err
+}
+
+// maxErrors for retrying renames on s3.
+const maxErrors = 20
+
 func (m *S3Layout) moveFiles(ctx context.Context, be *s3.Backend, l backend.Layout, t restic.FileType) error {
+	printErr := func(err error) {
+		fmt.Fprintf(os.Stderr, "renaming file returned error: %v\n", err)
+	}
+
 	for name := range be.List(ctx, t) {
 		h := restic.Handle{Type: t, Name: name}
 		debug.Log("move %v", h)
-		if err := be.Rename(h, l); err != nil {
-			return err
-		}
+
+		retry(maxErrors, printErr, func() error {
+			return be.Rename(h, l)
+		})
 	}
 
 	return nil
@@ -54,15 +78,22 @@ func (m *S3Layout) Apply(ctx context.Context, repo restic.Repository) error {
 		return errors.New("backend is not s3")
 	}
 
+	oldLayout := &backend.S3LegacyLayout{
+		Path: be.Path(),
+		Join: path.Join,
+	}
+
 	newLayout := &backend.DefaultLayout{
 		Path: be.Path(),
 		Join: path.Join,
 	}
 
+	be.Layout = oldLayout
+
 	for _, t := range []restic.FileType{
-		restic.KeyFile,
 		restic.SnapshotFile,
 		restic.DataFile,
+		restic.KeyFile,
 		restic.LockFile,
 	} {
 		err := m.moveFiles(ctx, be, newLayout, t)