From 482a6e9840154cb166dc37796d4c12467aed5971 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 25 Mar 2017 13:20:03 +0100 Subject: [PATCH 1/9] local: Add Config struct --- src/cmds/restic/global.go | 4 +-- src/restic/backend/local/config.go | 7 +++++- src/restic/backend/local/local.go | 34 +++++++++++++------------- src/restic/backend/local/local_test.go | 4 +-- src/restic/location/location_test.go | 31 ++++++++++++++++++----- src/restic/repository/testing.go | 4 +-- 6 files changed, 54 insertions(+), 30 deletions(-) diff --git a/src/cmds/restic/global.go b/src/cmds/restic/global.go index ae3b59ecb..a41194a1c 100644 --- a/src/cmds/restic/global.go +++ b/src/cmds/restic/global.go @@ -322,7 +322,7 @@ func open(s string) (restic.Backend, error) { switch loc.Scheme { case "local": debug.Log("opening local repository at %#v", loc.Config) - be, err = local.Open(loc.Config.(string)) + be, err = local.Open(loc.Config.(local.Config)) case "sftp": debug.Log("opening sftp repository at %#v", loc.Config) be, err = sftp.OpenWithConfig(loc.Config.(sftp.Config)) @@ -362,7 +362,7 @@ func create(s string) (restic.Backend, error) { switch loc.Scheme { case "local": debug.Log("create local repository at %#v", loc.Config) - return local.Create(loc.Config.(string)) + return local.Create(loc.Config.(local.Config)) case "sftp": debug.Log("create sftp repository at %#v", loc.Config) return sftp.CreateWithConfig(loc.Config.(sftp.Config)) diff --git a/src/restic/backend/local/config.go b/src/restic/backend/local/config.go index 8a5c67a2c..746accd27 100644 --- a/src/restic/backend/local/config.go +++ b/src/restic/backend/local/config.go @@ -6,11 +6,16 @@ import ( "restic/errors" ) +// Config holds all information needed to open a local repository. +type Config struct { + Path string +} + // ParseConfig parses a local backend config. func ParseConfig(cfg string) (interface{}, error) { if !strings.HasPrefix(cfg, "local:") { return nil, errors.New(`invalid format, prefix "local" not found`) } - return cfg[6:], nil + return Config{Path: cfg[6:]}, nil } diff --git a/src/restic/backend/local/local.go b/src/restic/backend/local/local.go index 03d812d99..510569654 100644 --- a/src/restic/backend/local/local.go +++ b/src/restic/backend/local/local.go @@ -16,7 +16,7 @@ import ( // Local is a backend in a local directory. type Local struct { - p string + Config } var _ restic.Backend = &Local{} @@ -34,28 +34,28 @@ func paths(dir string) []string { } // Open opens the local backend as specified by config. -func Open(dir string) (*Local, error) { +func Open(cfg Config) (*Local, error) { // test if all necessary dirs are there - for _, d := range paths(dir) { + for _, d := range paths(cfg.Path) { if _, err := fs.Stat(d); err != nil { return nil, errors.Wrap(err, "Open") } } - return &Local{p: dir}, nil + return &Local{Config: cfg}, nil } // Create creates all the necessary files and directories for a new local // backend at dir. Afterwards a new config blob should be created. -func Create(dir string) (*Local, error) { +func Create(cfg Config) (*Local, error) { // test if config file already exists - _, err := fs.Lstat(filepath.Join(dir, backend.Paths.Config)) + _, err := fs.Lstat(filepath.Join(cfg.Path, backend.Paths.Config)) if err == nil { return nil, errors.New("config file already exists") } // create paths for data, refs and temp - for _, d := range paths(dir) { + for _, d := range paths(cfg.Path) { err := fs.MkdirAll(d, backend.Modes.Dir) if err != nil { return nil, errors.Wrap(err, "MkdirAll") @@ -63,12 +63,12 @@ func Create(dir string) (*Local, error) { } // open backend - return Open(dir) + return Open(cfg) } // Location returns this backend's location (the directory name). func (b *Local) Location() string { - return b.p + return b.Path } // Construct path for given Type and name. @@ -132,13 +132,13 @@ func (b *Local) Save(h restic.Handle, rd io.Reader) (err error) { return err } - tmpfile, err := copyToTempfile(filepath.Join(b.p, backend.Paths.Temp), rd) + tmpfile, err := copyToTempfile(filepath.Join(b.Path, backend.Paths.Temp), rd) debug.Log("saved %v to %v", h, tmpfile) if err != nil { return err } - filename := filename(b.p, h.Type, h.Name) + filename := filename(b.Path, h.Type, h.Name) // test if new path already exists if _, err := fs.Stat(filename); err == nil { @@ -183,7 +183,7 @@ func (b *Local) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, return nil, errors.New("offset is negative") } - f, err := os.Open(filename(b.p, h.Type, h.Name)) + f, err := os.Open(filename(b.Path, h.Type, h.Name)) if err != nil { return nil, err } @@ -210,7 +210,7 @@ func (b *Local) Stat(h restic.Handle) (restic.FileInfo, error) { return restic.FileInfo{}, err } - fi, err := fs.Stat(filename(b.p, h.Type, h.Name)) + fi, err := fs.Stat(filename(b.Path, h.Type, h.Name)) if err != nil { return restic.FileInfo{}, errors.Wrap(err, "Stat") } @@ -221,7 +221,7 @@ func (b *Local) Stat(h restic.Handle) (restic.FileInfo, error) { // Test returns true if a blob of the given type and name exists in the backend. func (b *Local) Test(h restic.Handle) (bool, error) { debug.Log("Test %v", h) - _, err := fs.Stat(filename(b.p, h.Type, h.Name)) + _, err := fs.Stat(filename(b.Path, h.Type, h.Name)) if err != nil { if os.IsNotExist(errors.Cause(err)) { return false, nil @@ -235,7 +235,7 @@ func (b *Local) Test(h restic.Handle) (bool, error) { // Remove removes the blob with the given name and type. func (b *Local) Remove(h restic.Handle) error { debug.Log("Remove %v", h) - fn := filename(b.p, h.Type, h.Name) + fn := filename(b.Path, h.Type, h.Name) // reset read-only flag err := fs.Chmod(fn, 0666) @@ -316,7 +316,7 @@ func (b *Local) List(t restic.FileType, done <-chan struct{}) <-chan string { } ch := make(chan string) - items, err := lister(filepath.Join(dirname(b.p, t, ""))) + items, err := lister(filepath.Join(dirname(b.Path, t, ""))) if err != nil { close(ch) return ch @@ -343,7 +343,7 @@ func (b *Local) List(t restic.FileType, done <-chan struct{}) <-chan string { // Delete removes the repository and all files. func (b *Local) Delete() error { debug.Log("Delete()") - return fs.RemoveAll(b.p) + return fs.RemoveAll(b.Path) } // Close closes all open files. diff --git a/src/restic/backend/local/local_test.go b/src/restic/backend/local/local_test.go index 3bae88753..7e7440561 100644 --- a/src/restic/backend/local/local_test.go +++ b/src/restic/backend/local/local_test.go @@ -35,7 +35,7 @@ func init() { if err != nil { return nil, err } - return local.Create(tempBackendDir) + return local.Create(local.Config{Path: tempBackendDir}) } test.OpenFn = func() (restic.Backend, error) { @@ -43,7 +43,7 @@ func init() { if err != nil { return nil, err } - return local.Open(tempBackendDir) + return local.Open(local.Config{Path: tempBackendDir}) } test.CleanupFn = func() error { diff --git a/src/restic/location/location_test.go b/src/restic/location/location_test.go index bb4ac64c9..2961ecd54 100644 --- a/src/restic/location/location_test.go +++ b/src/restic/location/location_test.go @@ -5,6 +5,7 @@ import ( "reflect" "testing" + "restic/backend/local" "restic/backend/rest" "restic/backend/s3" "restic/backend/sftp" @@ -23,12 +24,30 @@ var parseTests = []struct { s string u Location }{ - {"local:/srv/repo", Location{Scheme: "local", Config: "/srv/repo"}}, - {"local:dir1/dir2", Location{Scheme: "local", Config: "dir1/dir2"}}, - {"local:dir1/dir2", Location{Scheme: "local", Config: "dir1/dir2"}}, - {"dir1/dir2", Location{Scheme: "local", Config: "dir1/dir2"}}, - {"local:../dir1/dir2", Location{Scheme: "local", Config: "../dir1/dir2"}}, - {"/dir1/dir2", Location{Scheme: "local", Config: "/dir1/dir2"}}, + {"local:/srv/repo", Location{Scheme: "local", + Config: local.Config{ + Path: "/srv/repo", + }}}, + {"local:dir1/dir2", Location{Scheme: "local", + Config: local.Config{ + Path: "dir1/dir2", + }}}, + {"local:dir1/dir2", Location{Scheme: "local", + Config: local.Config{ + Path: "dir1/dir2", + }}}, + {"dir1/dir2", Location{Scheme: "local", + Config: local.Config{ + Path: "dir1/dir2", + }}}, + {"local:../dir1/dir2", Location{Scheme: "local", + Config: local.Config{ + Path: "../dir1/dir2", + }}}, + {"/dir1/dir2", Location{Scheme: "local", + Config: local.Config{ + Path: "/dir1/dir2", + }}}, {"sftp:user@host:/srv/repo", Location{Scheme: "sftp", Config: sftp.Config{ diff --git a/src/restic/repository/testing.go b/src/restic/repository/testing.go index 7650ad8b9..6f590e13a 100644 --- a/src/restic/repository/testing.go +++ b/src/restic/repository/testing.go @@ -67,7 +67,7 @@ func TestRepository(t testing.TB) (r restic.Repository, cleanup func()) { if dir != "" { _, err := os.Stat(dir) if err != nil { - be, err := local.Create(dir) + be, err := local.Create(local.Config{Path: dir}) if err != nil { t.Fatalf("error creating local backend at %v: %v", dir, err) } @@ -84,7 +84,7 @@ func TestRepository(t testing.TB) (r restic.Repository, cleanup func()) { // TestOpenLocal opens a local repository. func TestOpenLocal(t testing.TB, dir string) (r restic.Repository) { - be, err := local.Open(dir) + be, err := local.Open(local.Config{Path: dir}) if err != nil { t.Fatal(err) } From 9861f3d43597cf65a83780208224d0170d74e077 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 25 Mar 2017 13:27:14 +0100 Subject: [PATCH 2/9] location: Reformat tests, use sub tests --- src/restic/location/location_test.go | 296 ++++++++++++++++----------- 1 file changed, 182 insertions(+), 114 deletions(-) diff --git a/src/restic/location/location_test.go b/src/restic/location/location_test.go index 2961ecd54..fe07ee506 100644 --- a/src/restic/location/location_test.go +++ b/src/restic/location/location_test.go @@ -24,136 +24,204 @@ var parseTests = []struct { s string u Location }{ - {"local:/srv/repo", Location{Scheme: "local", - Config: local.Config{ - Path: "/srv/repo", - }}}, - {"local:dir1/dir2", Location{Scheme: "local", - Config: local.Config{ - Path: "dir1/dir2", - }}}, - {"local:dir1/dir2", Location{Scheme: "local", - Config: local.Config{ - Path: "dir1/dir2", - }}}, - {"dir1/dir2", Location{Scheme: "local", - Config: local.Config{ - Path: "dir1/dir2", - }}}, - {"local:../dir1/dir2", Location{Scheme: "local", - Config: local.Config{ - Path: "../dir1/dir2", - }}}, - {"/dir1/dir2", Location{Scheme: "local", - Config: local.Config{ - Path: "/dir1/dir2", - }}}, + { + "local:/srv/repo", + Location{Scheme: "local", + Config: local.Config{ + Path: "/srv/repo", + }, + }, + }, + { + "local:dir1/dir2", + Location{Scheme: "local", + Config: local.Config{ + Path: "dir1/dir2", + }, + }, + }, + { + "local:dir1/dir2", + Location{Scheme: "local", + Config: local.Config{ + Path: "dir1/dir2", + }, + }, + }, + { + "dir1/dir2", + Location{Scheme: "local", + Config: local.Config{ + Path: "dir1/dir2", + }, + }, + }, + { + "local:../dir1/dir2", + Location{Scheme: "local", + Config: local.Config{ + Path: "../dir1/dir2", + }, + }, + }, + { + "/dir1/dir2", + Location{Scheme: "local", + Config: local.Config{ + Path: "/dir1/dir2", + }, + }, + }, - {"sftp:user@host:/srv/repo", Location{Scheme: "sftp", - Config: sftp.Config{ - User: "user", - Host: "host", - Dir: "/srv/repo", - }}}, - {"sftp:host:/srv/repo", Location{Scheme: "sftp", - Config: sftp.Config{ - User: "", - Host: "host", - Dir: "/srv/repo", - }}}, - {"sftp://user@host/srv/repo", Location{Scheme: "sftp", - Config: sftp.Config{ - User: "user", - Host: "host", - Dir: "srv/repo", - }}}, - {"sftp://user@host//srv/repo", Location{Scheme: "sftp", - Config: sftp.Config{ - User: "user", - Host: "host", - Dir: "/srv/repo", - }}}, + { + "sftp:user@host:/srv/repo", + Location{Scheme: "sftp", + Config: sftp.Config{ + User: "user", + Host: "host", + Dir: "/srv/repo", + }, + }, + }, + { + "sftp:host:/srv/repo", + Location{Scheme: "sftp", + Config: sftp.Config{ + User: "", + Host: "host", + Dir: "/srv/repo", + }, + }, + }, + { + "sftp://user@host/srv/repo", + Location{Scheme: "sftp", + Config: sftp.Config{ + User: "user", + Host: "host", + Dir: "srv/repo", + }, + }, + }, + { + "sftp://user@host//srv/repo", + Location{Scheme: "sftp", + Config: sftp.Config{ + User: "user", + Host: "host", + Dir: "/srv/repo", + }, + }, + }, - {"s3://eu-central-1/bucketname", Location{Scheme: "s3", - Config: s3.Config{ - Endpoint: "eu-central-1", - Bucket: "bucketname", - Prefix: "restic", - }}, + { + "s3://eu-central-1/bucketname", + Location{Scheme: "s3", + Config: s3.Config{ + Endpoint: "eu-central-1", + Bucket: "bucketname", + Prefix: "restic", + }, + }, }, - {"s3://hostname.foo/bucketname", Location{Scheme: "s3", - Config: s3.Config{ - Endpoint: "hostname.foo", - Bucket: "bucketname", - Prefix: "restic", - }}, + { + "s3://hostname.foo/bucketname", + Location{Scheme: "s3", + Config: s3.Config{ + Endpoint: "hostname.foo", + Bucket: "bucketname", + Prefix: "restic", + }, + }, }, - {"s3://hostname.foo/bucketname/prefix/directory", Location{Scheme: "s3", - Config: s3.Config{ - Endpoint: "hostname.foo", - Bucket: "bucketname", - Prefix: "prefix/directory", - }}, + { + "s3://hostname.foo/bucketname/prefix/directory", + Location{Scheme: "s3", + Config: s3.Config{ + Endpoint: "hostname.foo", + Bucket: "bucketname", + Prefix: "prefix/directory", + }, + }, }, - {"s3:eu-central-1/repo", Location{Scheme: "s3", - Config: s3.Config{ - Endpoint: "eu-central-1", - Bucket: "repo", - Prefix: "restic", - }}, + { + "s3:eu-central-1/repo", + Location{Scheme: "s3", + Config: s3.Config{ + Endpoint: "eu-central-1", + Bucket: "repo", + Prefix: "restic", + }, + }, }, - {"s3:eu-central-1/repo/prefix/directory", Location{Scheme: "s3", - Config: s3.Config{ - Endpoint: "eu-central-1", - Bucket: "repo", - Prefix: "prefix/directory", - }}, + { + "s3:eu-central-1/repo/prefix/directory", + Location{Scheme: "s3", + Config: s3.Config{ + Endpoint: "eu-central-1", + Bucket: "repo", + Prefix: "prefix/directory", + }, + }, }, - {"s3:https://hostname.foo/repo", Location{Scheme: "s3", - Config: s3.Config{ - Endpoint: "hostname.foo", - Bucket: "repo", - Prefix: "restic", - }}, + { + "s3:https://hostname.foo/repo", + Location{Scheme: "s3", + Config: s3.Config{ + Endpoint: "hostname.foo", + Bucket: "repo", + Prefix: "restic", + }, + }, }, - {"s3:https://hostname.foo/repo/prefix/directory", Location{Scheme: "s3", - Config: s3.Config{ - Endpoint: "hostname.foo", - Bucket: "repo", - Prefix: "prefix/directory", - }}, + { + "s3:https://hostname.foo/repo/prefix/directory", + Location{Scheme: "s3", + Config: s3.Config{ + Endpoint: "hostname.foo", + Bucket: "repo", + Prefix: "prefix/directory", + }, + }, }, - {"s3:http://hostname.foo/repo", Location{Scheme: "s3", - Config: s3.Config{ - Endpoint: "hostname.foo", - Bucket: "repo", - Prefix: "restic", - UseHTTP: true, - }}, + { + "s3:http://hostname.foo/repo", + Location{Scheme: "s3", + Config: s3.Config{ + Endpoint: "hostname.foo", + Bucket: "repo", + Prefix: "restic", + UseHTTP: true, + }, + }, }, - {"rest:http://hostname.foo:1234/", Location{Scheme: "rest", - Config: rest.Config{ - URL: parseURL("http://hostname.foo:1234/"), - }}, + { + "rest:http://hostname.foo:1234/", + Location{Scheme: "rest", + Config: rest.Config{ + URL: parseURL("http://hostname.foo:1234/"), + }, + }, }, } func TestParse(t *testing.T) { for i, test := range parseTests { - u, err := Parse(test.s) - if err != nil { - t.Errorf("unexpected error: %v", err) - continue - } + t.Run(test.s, func(t *testing.T) { + u, err := Parse(test.s) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } - if test.u.Scheme != u.Scheme { - t.Errorf("test %d: scheme does not match, want %q, got %q", - i, test.u.Scheme, u.Scheme) - } + if test.u.Scheme != u.Scheme { + t.Errorf("test %d: scheme does not match, want %q, got %q", + i, test.u.Scheme, u.Scheme) + } - if !reflect.DeepEqual(test.u.Config, u.Config) { - t.Errorf("test %d: cfg map does not match, want:\n %#v\ngot: \n %#v", - i, test.u.Config, u.Config) - } + if !reflect.DeepEqual(test.u.Config, u.Config) { + t.Errorf("test %d: cfg map does not match, want:\n %#v\ngot: \n %#v", + i, test.u.Config, u.Config) + } + }) } } From 2e0b19f63fce05f779b0afa8efe90ffa596a8af2 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 25 Mar 2017 15:25:58 +0100 Subject: [PATCH 3/9] location: Move to backend/location --- src/cmds/restic/global.go | 2 +- src/restic/{ => backend}/location/location.go | 0 src/restic/{ => backend}/location/location_test.go | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename src/restic/{ => backend}/location/location.go (100%) rename src/restic/{ => backend}/location/location_test.go (100%) diff --git a/src/cmds/restic/global.go b/src/cmds/restic/global.go index a41194a1c..8cfcedce7 100644 --- a/src/cmds/restic/global.go +++ b/src/cmds/restic/global.go @@ -12,11 +12,11 @@ import ( "syscall" "restic/backend/local" + "restic/backend/location" "restic/backend/rest" "restic/backend/s3" "restic/backend/sftp" "restic/debug" - "restic/location" "restic/repository" "restic/errors" diff --git a/src/restic/location/location.go b/src/restic/backend/location/location.go similarity index 100% rename from src/restic/location/location.go rename to src/restic/backend/location/location.go diff --git a/src/restic/location/location_test.go b/src/restic/backend/location/location_test.go similarity index 100% rename from src/restic/location/location_test.go rename to src/restic/backend/location/location_test.go From d0a5e86da13091886e0b4b223f4284f1d1097fa5 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 25 Mar 2017 14:36:39 +0100 Subject: [PATCH 4/9] Add free-form options parsing --- src/restic/options/options.go | 61 ++++++++++++++++ src/restic/options/options_test.go | 107 +++++++++++++++++++++++++++++ 2 files changed, 168 insertions(+) create mode 100644 src/restic/options/options.go create mode 100644 src/restic/options/options_test.go diff --git a/src/restic/options/options.go b/src/restic/options/options.go new file mode 100644 index 000000000..635216041 --- /dev/null +++ b/src/restic/options/options.go @@ -0,0 +1,61 @@ +package options + +import ( + "restic/errors" + "strings" +) + +// Options holds options in the form key=value. +type Options map[string]string + +// splitKeyValue splits at the first equals (=) sign. +func splitKeyValue(s string) (key string, value string) { + data := strings.SplitN(s, "=", 2) + key = strings.ToLower(strings.TrimSpace(data[0])) + if len(data) == 1 { + // no equals sign is treated as the empty value + return key, "" + } + + return key, strings.TrimSpace(data[1]) +} + +// Parse takes a slice of key=value pairs and returns an Options type. +// The key may include namespaces, separated by dots. Example: "foo.bar=value". +// Keys are converted to lower-case. +func Parse(in []string) (Options, error) { + opts := make(Options, len(in)) + + for _, opt := range in { + key, value := splitKeyValue(opt) + + if key == "" { + return Options{}, errors.Fatalf("empty key is not a valid option") + } + opts[key] = value + } + + return opts, nil +} + +// Extract returns an Options type with all keys in namespace ns, which is +// also stripped from the keys. ns must end with a dot. +func (o Options) Extract(ns string) Options { + l := len(ns) + if ns[l-1] != '.' { + ns += "." + l++ + } + + opts := make(Options) + + for k, v := range o { + if !strings.HasPrefix(k, ns) { + continue + } + + opts[k[l:]] = v + } + + return opts +} diff --git a/src/restic/options/options_test.go b/src/restic/options/options_test.go new file mode 100644 index 000000000..feead1c89 --- /dev/null +++ b/src/restic/options/options_test.go @@ -0,0 +1,107 @@ +package options + +import ( + "fmt" + "reflect" + "testing" +) + +var optsTests = []struct { + input []string + output Options +}{ + { + []string{"foo=bar", "bar=baz ", "k="}, + Options{ + "foo": "bar", + "bar": "baz", + "k": "", + }, + }, + { + []string{"Foo=23", "baR", "k=thing with spaces"}, + Options{ + "foo": "23", + "bar": "", + "k": "thing with spaces", + }, + }, + { + []string{"k=thing with spaces", "k2=more spaces = not evil"}, + Options{ + "k": "thing with spaces", + "k2": "more spaces = not evil", + }, + }, +} + +func TestParseOptions(t *testing.T) { + for i, test := range optsTests { + t.Run(fmt.Sprintf("test-%v", i), func(t *testing.T) { + opts, err := Parse(test.input) + if err != nil { + t.Fatalf("unable to parse options: %v", err) + } + + if !reflect.DeepEqual(opts, test.output) { + t.Fatalf("wrong result, want:\n %#v\ngot:\n %#v", test.output, opts) + } + }) + } +} + +var invalidOptsTests = []struct { + input []string + err string +}{ + { + []string{"=bar", "bar=baz", "k="}, + "empty key is not a valid option", + }, +} + +func TestParseInvalidOptions(t *testing.T) { + for _, test := range invalidOptsTests { + t.Run(test.err, func(t *testing.T) { + _, err := Parse(test.input) + if err == nil { + t.Fatalf("expected error (%v) not found, err is nil", test.err) + } + + if err.Error() != test.err { + t.Fatalf("expected error %q, got %q", test.err, err.Error()) + } + }) + } +} + +var extractTests = []struct { + input Options + ns string + output Options +}{ + { + input: Options{ + "foo.bar:": "baz", + "s3.timeout": "10s", + "sftp.timeout": "5s", + "global": "foobar", + }, + ns: "s3", + output: Options{ + "timeout": "10s", + }, + }, +} + +func TestOptionsExtract(t *testing.T) { + for _, test := range extractTests { + t.Run(test.ns, func(t *testing.T) { + opts := test.input.Extract(test.ns) + + if !reflect.DeepEqual(opts, test.output) { + t.Fatalf("wrong result, want:\n %#v\ngot:\n %#v", test.output, opts) + } + }) + } +} From 946b4f4b86319b802d2c4f68df88400253a6fa59 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 25 Mar 2017 15:33:52 +0100 Subject: [PATCH 5/9] Add extended options via -o/--option --- src/cmds/restic/cmd_init.go | 2 +- src/cmds/restic/global.go | 13 ++++++++++--- src/cmds/restic/main.go | 18 +++++++++++++++--- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/src/cmds/restic/cmd_init.go b/src/cmds/restic/cmd_init.go index d134e9376..ffa9cf272 100644 --- a/src/cmds/restic/cmd_init.go +++ b/src/cmds/restic/cmd_init.go @@ -27,7 +27,7 @@ func runInit(gopts GlobalOptions, args []string) error { return errors.Fatal("Please specify repository location (-r)") } - be, err := create(gopts.Repo) + be, err := create(gopts.Repo, gopts.extended) if err != nil { return errors.Fatalf("create backend at %s failed: %v\n", gopts.Repo, err) } diff --git a/src/cmds/restic/global.go b/src/cmds/restic/global.go index 8cfcedce7..d4ccfb5e7 100644 --- a/src/cmds/restic/global.go +++ b/src/cmds/restic/global.go @@ -17,6 +17,7 @@ import ( "restic/backend/s3" "restic/backend/sftp" "restic/debug" + "restic/options" "restic/repository" "restic/errors" @@ -38,6 +39,10 @@ type GlobalOptions struct { password string stdout io.Writer stderr io.Writer + + Options []string + + extended options.Options } var globalOptions = GlobalOptions{ @@ -65,6 +70,8 @@ func init() { f.BoolVar(&globalOptions.NoLock, "no-lock", false, "do not lock the repo, this allows some operations on read-only repos") f.BoolVarP(&globalOptions.JSON, "json", "", false, "set output mode to JSON for commands that support it") + f.StringSliceVarP(&globalOptions.Options, "option", "o", []string{}, "set extended option (`key=value`, can be specified multiple times)") + restoreTerminal() } @@ -287,7 +294,7 @@ func OpenRepository(opts GlobalOptions) (*repository.Repository, error) { return nil, errors.Fatal("Please specify repository location (-r)") } - be, err := open(opts.Repo) + be, err := open(opts.Repo, opts.extended) if err != nil { return nil, err } @@ -310,7 +317,7 @@ func OpenRepository(opts GlobalOptions) (*repository.Repository, error) { } // Open the backend specified by a location config. -func open(s string) (restic.Backend, error) { +func open(s string, opts options.Options) (restic.Backend, error) { debug.Log("parsing location %v", s) loc, err := location.Parse(s) if err != nil { @@ -352,7 +359,7 @@ func open(s string) (restic.Backend, error) { } // Create the backend specified by URI. -func create(s string) (restic.Backend, error) { +func create(s string, opts options.Options) (restic.Backend, error) { debug.Log("parsing location %v", s) loc, err := location.Parse(s) if err != nil { diff --git a/src/cmds/restic/main.go b/src/cmds/restic/main.go index 03f7f9035..96a508da9 100644 --- a/src/cmds/restic/main.go +++ b/src/cmds/restic/main.go @@ -5,6 +5,7 @@ import ( "os" "restic" "restic/debug" + "restic/options" "github.com/spf13/cobra" @@ -22,10 +23,21 @@ directories in an encrypted repository stored on different backends. SilenceErrors: true, SilenceUsage: true, - // run the debug functions for all subcommands (if build tag "debug" is - // enabled) PersistentPreRunE: func(*cobra.Command, []string) error { - return runDebug() + // parse extended options + opts, err := options.Parse(globalOptions.Options) + if err != nil { + return err + } + globalOptions.extended = opts + + // run the debug functions for all subcommands (if build tag "debug" is + // enabled) + if err := runDebug(); err != nil { + return err + } + + return nil }, PersistentPostRun: func(*cobra.Command, []string) { shutdownDebug() From f587a5f4f0f37f0b3a911dba7560cd0f5a6ff6cc Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 25 Mar 2017 15:39:38 +0100 Subject: [PATCH 6/9] options: fail when key is configured twice --- src/restic/options/options.go | 5 +++++ src/restic/options/options_test.go | 12 ++++++++++++ 2 files changed, 17 insertions(+) diff --git a/src/restic/options/options.go b/src/restic/options/options.go index 635216041..0c620c8a3 100644 --- a/src/restic/options/options.go +++ b/src/restic/options/options.go @@ -32,6 +32,11 @@ func Parse(in []string) (Options, error) { if key == "" { return Options{}, errors.Fatalf("empty key is not a valid option") } + + if v, ok := opts[key]; ok && v != value { + return Options{}, errors.Fatalf("key %q present more than once", key) + } + opts[key] = value } diff --git a/src/restic/options/options_test.go b/src/restic/options/options_test.go index feead1c89..2717bc9c3 100644 --- a/src/restic/options/options_test.go +++ b/src/restic/options/options_test.go @@ -33,6 +33,14 @@ var optsTests = []struct { "k2": "more spaces = not evil", }, }, + { + []string{"x=1", "foo=bar", "y=2", "foo=bar"}, + Options{ + "x": "1", + "y": "2", + "foo": "bar", + }, + }, } func TestParseOptions(t *testing.T) { @@ -58,6 +66,10 @@ var invalidOptsTests = []struct { []string{"=bar", "bar=baz", "k="}, "empty key is not a valid option", }, + { + []string{"x=1", "foo=bar", "y=2", "foo=baz"}, + `key "foo" present more than once`, + }, } func TestParseInvalidOptions(t *testing.T) { From 2924ebc124f8085fe856e780e2085ff4eda72e01 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 25 Mar 2017 16:53:30 +0100 Subject: [PATCH 7/9] options: Add Apply() --- src/restic/options/options.go | 59 ++++++++++++++++++ src/restic/options/options_test.go | 97 ++++++++++++++++++++++++++++++ 2 files changed, 156 insertions(+) diff --git a/src/restic/options/options.go b/src/restic/options/options.go index 0c620c8a3..49d3fd309 100644 --- a/src/restic/options/options.go +++ b/src/restic/options/options.go @@ -1,8 +1,11 @@ package options import ( + "reflect" "restic/errors" + "strconv" "strings" + "time" ) // Options holds options in the form key=value. @@ -64,3 +67,59 @@ func (o Options) Extract(ns string) Options { return opts } + +// Apply sets the options on dst via reflection, using the struct tag `option`. +func (o Options) Apply(dst interface{}) error { + v := reflect.ValueOf(dst).Elem() + + fields := make(map[string]reflect.StructField) + + for i := 0; i < v.NumField(); i++ { + f := v.Type().Field(i) + tag := f.Tag.Get("option") + + if tag == "" { + continue + } + + if _, ok := fields[tag]; ok { + panic("option tag " + tag + " is not unique in " + v.Type().Name()) + } + + fields[tag] = f + } + + for key, value := range o { + field, ok := fields[key] + if !ok { + return errors.Fatalf("option %v is not known", key) + } + + i := field.Index[0] + switch v.Type().Field(i).Type.Name() { + case "string": + v.Field(i).SetString(value) + + case "int": + vi, err := strconv.ParseInt(value, 0, 32) + if err != nil { + return err + } + + v.Field(i).SetInt(vi) + + case "Duration": + d, err := time.ParseDuration(value) + if err != nil { + return err + } + + v.Field(i).SetInt(int64(d)) + + default: + panic("type " + v.Type().Field(i).Type.Name() + " not handled") + } + } + + return nil +} diff --git a/src/restic/options/options_test.go b/src/restic/options/options_test.go index 2717bc9c3..5a255591c 100644 --- a/src/restic/options/options_test.go +++ b/src/restic/options/options_test.go @@ -4,6 +4,7 @@ import ( "fmt" "reflect" "testing" + "time" ) var optsTests = []struct { @@ -117,3 +118,99 @@ func TestOptionsExtract(t *testing.T) { }) } } + +// Target is used for Apply() tests +type Target struct { + Name string `option:"name"` + ID int `option:"id"` + Timeout time.Duration `option:"timeout"` + Other string +} + +var setTests = []struct { + input Options + output Target +}{ + { + Options{ + "name": "foobar", + }, + Target{ + Name: "foobar", + }, + }, + { + Options{ + "name": "foobar", + "id": "1234", + }, + Target{ + Name: "foobar", + ID: 1234, + }, + }, + { + Options{ + "timeout": "10m3s", + }, + Target{ + Timeout: time.Duration(10*time.Minute + 3*time.Second), + }, + }, +} + +func TestOptionsApply(t *testing.T) { + for i, test := range setTests { + t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) { + var dst Target + err := test.input.Apply(&dst) + if err != nil { + t.Fatal(err) + } + + if dst != test.output { + t.Fatalf("wrong result, want:\n %#v\ngot:\n %#v", test.output, dst) + } + }) + } +} + +var invalidSetTests = []struct { + input Options + err string +}{ + { + Options{ + "first_name": "foobar", + }, + "option first_name is not known", + }, + { + Options{ + "id": "foobar", + }, + `strconv.ParseInt: parsing "foobar": invalid syntax`, + }, + { + Options{ + "timeout": "2134", + }, + `time: missing unit in duration 2134`, + }, +} + +func TestOptionsApplyInvalid(t *testing.T) { + for i, test := range invalidSetTests { + t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) { + var dst Target + err := test.input.Apply(&dst) + if err == nil { + t.Fatalf("expected error %v not found", test.err) + } + + if err.Error() != test.err { + t.Fatalf("expected error %q, got %q", test.err, err.Error()) + } + }) + } +} From a8a7701f6059c16d14422b924a6981c4cc7f90e5 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 25 Mar 2017 17:20:03 +0100 Subject: [PATCH 8/9] options: Add namespace to Apply() --- src/restic/options/options.go | 6 +++++- src/restic/options/options_test.go | 14 +++++++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/restic/options/options.go b/src/restic/options/options.go index 49d3fd309..c5d9ff3e3 100644 --- a/src/restic/options/options.go +++ b/src/restic/options/options.go @@ -69,7 +69,8 @@ func (o Options) Extract(ns string) Options { } // Apply sets the options on dst via reflection, using the struct tag `option`. -func (o Options) Apply(dst interface{}) error { +// The namespace argument (ns) is only used for error messages. +func (o Options) Apply(ns string, dst interface{}) error { v := reflect.ValueOf(dst).Elem() fields := make(map[string]reflect.StructField) @@ -92,6 +93,9 @@ func (o Options) Apply(dst interface{}) error { for key, value := range o { field, ok := fields[key] if !ok { + if ns != "" { + key = ns + "." + key + } return errors.Fatalf("option %v is not known", key) } diff --git a/src/restic/options/options_test.go b/src/restic/options/options_test.go index 5a255591c..a5ab83952 100644 --- a/src/restic/options/options_test.go +++ b/src/restic/options/options_test.go @@ -163,7 +163,7 @@ func TestOptionsApply(t *testing.T) { for i, test := range setTests { t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) { var dst Target - err := test.input.Apply(&dst) + err := test.input.Apply("", &dst) if err != nil { t.Fatal(err) } @@ -176,25 +176,29 @@ func TestOptionsApply(t *testing.T) { } var invalidSetTests = []struct { - input Options - err string + input Options + namespace string + err string }{ { Options{ "first_name": "foobar", }, - "option first_name is not known", + "ns", + "option ns.first_name is not known", }, { Options{ "id": "foobar", }, + "ns", `strconv.ParseInt: parsing "foobar": invalid syntax`, }, { Options{ "timeout": "2134", }, + "ns", `time: missing unit in duration 2134`, }, } @@ -203,7 +207,7 @@ func TestOptionsApplyInvalid(t *testing.T) { for i, test := range invalidSetTests { t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) { var dst Target - err := test.input.Apply(&dst) + err := test.input.Apply(test.namespace, &dst) if err == nil { t.Fatalf("expected error %v not found", test.err) } From 719bb183161bf2adf4a1771f90ea5ca02f014d0a Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 25 Mar 2017 17:31:59 +0100 Subject: [PATCH 9/9] Parse extended options --- src/cmds/restic/global.go | 104 ++++++++++++++++++++++++++------------ 1 file changed, 72 insertions(+), 32 deletions(-) diff --git a/src/cmds/restic/global.go b/src/cmds/restic/global.go index d4ccfb5e7..45b41cf33 100644 --- a/src/cmds/restic/global.go +++ b/src/cmds/restic/global.go @@ -316,6 +316,59 @@ func OpenRepository(opts GlobalOptions) (*repository.Repository, error) { return s, nil } +func parseConfig(loc location.Location, opts options.Options) (interface{}, error) { + // only apply options for a particular backend here + opts = opts.Extract(loc.Scheme) + + switch loc.Scheme { + case "local": + cfg := loc.Config.(local.Config) + if err := opts.Apply(loc.Scheme, &cfg); err != nil { + return nil, err + } + + debug.Log("opening local repository at %#v", cfg) + return cfg, nil + + case "sftp": + cfg := loc.Config.(sftp.Config) + if err := opts.Apply(loc.Scheme, &cfg); err != nil { + return nil, err + } + + debug.Log("opening sftp repository at %#v", cfg) + return cfg, nil + + case "s3": + cfg := loc.Config.(s3.Config) + if cfg.KeyID == "" { + cfg.KeyID = os.Getenv("AWS_ACCESS_KEY_ID") + } + + if cfg.Secret == "" { + cfg.Secret = os.Getenv("AWS_SECRET_ACCESS_KEY") + } + + if err := opts.Apply(loc.Scheme, &cfg); err != nil { + return nil, err + } + + debug.Log("opening s3 repository at %#v", cfg) + return cfg, nil + + case "rest": + cfg := loc.Config.(rest.Config) + if err := opts.Apply(loc.Scheme, &cfg); err != nil { + return nil, err + } + + debug.Log("opening rest repository at %#v", cfg) + return cfg, nil + } + + return nil, errors.Fatalf("invalid backend: %q", loc.Scheme) +} + // Open the backend specified by a location config. func open(s string, opts options.Options) (restic.Backend, error) { debug.Log("parsing location %v", s) @@ -326,27 +379,21 @@ func open(s string, opts options.Options) (restic.Backend, error) { var be restic.Backend + cfg, err := parseConfig(loc, opts) + if err != nil { + return nil, err + } + switch loc.Scheme { case "local": - debug.Log("opening local repository at %#v", loc.Config) - be, err = local.Open(loc.Config.(local.Config)) + be, err = local.Open(cfg.(local.Config)) case "sftp": - debug.Log("opening sftp repository at %#v", loc.Config) - be, err = sftp.OpenWithConfig(loc.Config.(sftp.Config)) + be, err = sftp.OpenWithConfig(cfg.(sftp.Config)) case "s3": - cfg := loc.Config.(s3.Config) - if cfg.KeyID == "" { - cfg.KeyID = os.Getenv("AWS_ACCESS_KEY_ID") - - } - if cfg.Secret == "" { - cfg.Secret = os.Getenv("AWS_SECRET_ACCESS_KEY") - } - - debug.Log("opening s3 repository at %#v", cfg) - be, err = s3.Open(cfg) + be, err = s3.Open(cfg.(s3.Config)) case "rest": - be, err = rest.Open(loc.Config.(rest.Config)) + be, err = rest.Open(cfg.(rest.Config)) + default: return nil, errors.Fatalf("invalid backend: %q", loc.Scheme) } @@ -366,27 +413,20 @@ func create(s string, opts options.Options) (restic.Backend, error) { return nil, err } + cfg, err := parseConfig(loc, opts) + if err != nil { + return nil, err + } + switch loc.Scheme { case "local": - debug.Log("create local repository at %#v", loc.Config) - return local.Create(loc.Config.(local.Config)) + return local.Create(cfg.(local.Config)) case "sftp": - debug.Log("create sftp repository at %#v", loc.Config) - return sftp.CreateWithConfig(loc.Config.(sftp.Config)) + return sftp.CreateWithConfig(cfg.(sftp.Config)) case "s3": - cfg := loc.Config.(s3.Config) - if cfg.KeyID == "" { - cfg.KeyID = os.Getenv("AWS_ACCESS_KEY_ID") - - } - if cfg.Secret == "" { - cfg.Secret = os.Getenv("AWS_SECRET_ACCESS_KEY") - } - - debug.Log("create s3 repository at %#v", loc.Config) - return s3.Open(cfg) + return s3.Open(cfg.(s3.Config)) case "rest": - return rest.Create(loc.Config.(rest.Config)) + return rest.Create(cfg.(rest.Config)) } debug.Log("invalid repository scheme: %v", s)