Add migrate

This commits adds a 'migrate' command and a migration to move s3
repositories from the 's3legacy' to the 'default' layout.

src/cmds/restic/cmd_migrate.go

@@ -0,0 +1,99 @@
+package main
+
+import (
+	"restic"
+	"restic/migrations"
+
+	"github.com/spf13/cobra"
+)
+
+var cmdMigrate = &cobra.Command{
+	Use:   "migrate [name]",
+	Short: "apply migrations",
+	Long: `
+The "migrate" command applies migrations to a repository. When no migration
+name is explicitely given, a list of migrations that can be applied is printed.
+`,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return runMigrate(migrateOptions, globalOptions, args)
+	},
+}
+
+// MigrateOptions bundles all options for the 'check' command.
+type MigrateOptions struct {
+}
+
+var migrateOptions MigrateOptions
+
+func init() {
+	cmdRoot.AddCommand(cmdMigrate)
+}
+
+func checkMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repository) error {
+	ctx := gopts.ctx
+	Printf("available migrations:\n")
+	for _, m := range migrations.All {
+		ok, err := m.Check(ctx, repo)
+		if err != nil {
+			return err
+		}
+
+		if ok {
+			Printf("  %v\n", m.Name())
+		}
+	}
+
+	return nil
+}
+
+func applyMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repository, args []string) error {
+	ctx := gopts.ctx
+
+	var firsterr error
+	for _, name := range args {
+		for _, m := range migrations.All {
+			if m.Name() == name {
+				ok, err := m.Check(ctx, repo)
+				if err != nil {
+					return err
+				}
+
+				if !ok {
+					Warnf("migration %v cannot be applied: check failed\n")
+					continue
+				}
+
+				if err = m.Apply(ctx, repo); err != nil {
+					Warnf("migration %v failed: %v\n", m.Name(), err)
+					if firsterr == nil {
+						firsterr = err
+					}
+					continue
+				}
+
+				Printf("migration %v: success\n", m.Name())
+			}
+		}
+	}
+
+	return firsterr
+}
+
+func runMigrate(opts MigrateOptions, gopts GlobalOptions, args []string) error {
+	repo, err := OpenRepository(gopts)
+	if err != nil {
+		return err
+	}
+
+	lock, err := lockRepoExclusive(repo)
+	defer unlockRepo(lock)
+	if err != nil {
+		return err
+	}
+
+	if len(args) == 0 {
+		return checkMigrations(opts, gopts, repo)
+	}
+
+	return applyMigrations(opts, gopts, repo, args)
+}

src/restic/backend/s3/s3.go

@@ -155,7 +155,12 @@ func (be *Backend) ReadDir(dir string) (list []os.FileInfo, err error) {
 
 // Location returns this backend's location (the bucket name).
 func (be *Backend) Location() string {
-	return be.bucketname
+	return be.Join(be.bucketname, be.prefix)
+}
+
+// Path returns the path in the bucket that is used for this backend.
+func (be *Backend) Path() string {
+	return be.prefix
 }
 
 // getRemainingSize returns number of bytes remaining. If it is not possible to
@@ -422,3 +427,21 @@ func (be *Backend) Delete(ctx context.Context) error {
 
 // Close does nothing
 func (be *Backend) Close() error { return nil }
+
+// Rename moves a file based on the new layout l.
+func (be *Backend) Rename(h restic.Handle, l backend.Layout) error {
+	debug.Log("Rename %v to %v", h, l)
+	oldname := be.Filename(h)
+	newname := l.Filename(h)
+
+	debug.Log("  %v -> %v", oldname, newname)
+
+	coreClient := minio.Core{Client: be.client}
+	err := coreClient.CopyObject(be.bucketname, newname, path.Join(be.bucketname, oldname), minio.CopyConditions{})
+	if err != nil {
+		debug.Log("copy failed: %v", err)
+		return err
+	}
+
+	return be.client.RemoveObject(be.bucketname, oldname)
+}

src/restic/migrations/doc.go

@@ -0,0 +1,2 @@
+// Package migrations contains migrations that can be applied to a repository and/or backend.
+package migrations

src/restic/migrations/interface.go

@@ -0,0 +1,21 @@
+package migrations
+
+import (
+	"context"
+	"restic"
+)
+
+// Migration implements a data migration.
+type Migration interface {
+	// Check returns true if the migration can be applied to a repo.
+	Check(context.Context, restic.Repository) (bool, error)
+
+	// Apply runs the migration.
+	Apply(context.Context, restic.Repository) error
+
+	// Name returns a short name.
+	Name() string
+
+	// Descr returns a description what the migration does.
+	Desc() string
+}

src/restic/migrations/list.go

@@ -0,0 +1,8 @@
+package migrations
+
+// All contains all migrations.
+var All []Migration
+
+func register(m Migration) {
+	All = append(All, m)
+}

src/restic/migrations/s3_layout.go

@@ -0,0 +1,87 @@
+package migrations
+
+import (
+	"context"
+	"path"
+	"restic"
+	"restic/backend"
+	"restic/backend/s3"
+	"restic/debug"
+	"restic/errors"
+)
+
+func init() {
+	register(&S3Layout{})
+}
+
+// S3Layout migrates a repository on an S3 backend from the "s3legacy" to the
+// "default" layout.
+type S3Layout struct{}
+
+// Check tests whether the migration can be applied.
+func (m *S3Layout) Check(ctx context.Context, repo restic.Repository) (bool, error) {
+	be, ok := repo.Backend().(*s3.Backend)
+	if !ok {
+		debug.Log("backend is not s3")
+		return false, nil
+	}
+
+	if be.Layout.Name() != "s3legacy" {
+		debug.Log("layout is not s3legacy")
+		return false, nil
+	}
+
+	return true, nil
+}
+
+func (m *S3Layout) moveFiles(ctx context.Context, be *s3.Backend, l backend.Layout, t restic.FileType) error {
+	for name := range be.List(ctx, t) {
+		h := restic.Handle{Type: t, Name: name}
+		debug.Log("move %v", h)
+		if err := be.Rename(h, l); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Apply runs the migration.
+func (m *S3Layout) Apply(ctx context.Context, repo restic.Repository) error {
+	be, ok := repo.Backend().(*s3.Backend)
+	if !ok {
+		debug.Log("backend is not s3")
+		return errors.New("backend is not s3")
+	}
+
+	newLayout := &backend.DefaultLayout{
+		Path: be.Path(),
+		Join: path.Join,
+	}
+
+	for _, t := range []restic.FileType{
+		restic.KeyFile,
+		restic.SnapshotFile,
+		restic.DataFile,
+		restic.LockFile,
+	} {
+		err := m.moveFiles(ctx, be, newLayout, t)
+		if err != nil {
+			return err
+		}
+	}
+
+	be.Layout = newLayout
+
+	return nil
+}
+
+// Name returns the name for this migration.
+func (m *S3Layout) Name() string {
+	return "s3_layout"
+}
+
+// Desc returns a short description what the migration does.
+func (m *S3Layout) Desc() string {
+	return "move files from 's3legacy' to the 'default' repository layout"
+}