Introduced a configurable object path prefix for s3 repositories.

Prepends the object path prefix to all s3 paths and allows to have multiple independent
restic backup repositories in a single s3 bucket.

Removed the hardcoded "restic" prefix from s3 paths.

Use "restic" as the default object path prefix for s3 if no other prefix gets specified.
This will retain backward compatibility with existing s3 repository configurations.

Simplified the parse flow to have a single point where we parse the bucket name and the prefix within the bucket.

Added tests for s3 object path prefix and the new default prefix to config_test and location_test.

backend/s3/config.go

@@ -13,53 +13,28 @@ type Config struct {
 	UseHTTP       bool
 	KeyID, Secret string
 	Bucket        string
+	Prefix        string
 }
 
+const defaultPrefix = "restic"
+
 // ParseConfig parses the string s and extracts the s3 config. The two
-// supported configuration formats are s3://host/bucketname and
-// s3:host:bucketname. The host can also be a valid s3 region name.
+// supported configuration formats are s3://host/bucketname/prefix and
+// s3:host:bucketname/prefix. The host can also be a valid s3 region
+// name. If no prefix is given the prefix "restic" will be used.
 func ParseConfig(s string) (interface{}, error) {
+	var path []string
+	cfg := Config{}
 	if strings.HasPrefix(s, "s3://") {
 		s = s[5:]
-
-		data := strings.SplitN(s, "/", 2)
-		if len(data) != 2 {
-			return nil, errors.New("s3: invalid format, host/region or bucket name not found")
-		}
-
-		cfg := Config{
-			Endpoint: data[0],
-			Bucket:   data[1],
-		}
-
-		return cfg, nil
-	}
-
-	data := strings.SplitN(s, ":", 2)
-	if len(data) != 2 {
-		return nil, errors.New("s3: invalid format")
-	}
-
-	if data[0] != "s3" {
-		return nil, errors.New(`s3: config does not start with "s3"`)
-	}
-
-	s = data[1]
-
-	cfg := Config{}
-	rest := strings.Split(s, "/")
-	if len(rest) < 2 {
-		return nil, errors.New("s3: region or bucket not found")
-	}
-
-	if len(rest) == 2 {
-		// assume that just a region name and a bucket has been specified, in
-		// the format region/bucket
-		cfg.Endpoint = rest[0]
-		cfg.Bucket = rest[1]
-	} else {
-		// assume that a URL has been specified, parse it and use the path as
-		// the bucket name.
+		path = strings.SplitN(s, "/", 3)
+		cfg.Endpoint = path[0]
+		path = path[1:]
+	} else if strings.HasPrefix(s, "s3:http") {
+		s = s[3:]
+		// assume that a URL has been specified, parse it and
+		// use the host as the endpoint and the path as the
+		// bucket name and prefix
 		url, err := url.Parse(s)
 		if err != nil {
 			return nil, err
@@ -73,8 +48,23 @@ func ParseConfig(s string) (interface{}, error) {
 		if url.Scheme == "http" {
 			cfg.UseHTTP = true
 		}
-
-		cfg.Bucket = url.Path[1:]
+		path = strings.SplitN(url.Path[1:], "/", 2)
+	} else if strings.HasPrefix(s, "s3:") {
+		s = s[3:]
+		path = strings.SplitN(s, "/", 3)
+		cfg.Endpoint = path[0]
+		path = path[1:]
+	} else {
+		return nil, errors.New("s3: invalid format")
+	}
+	if len(path) < 1 {
+		return nil, errors.New("s3: invalid format, host/region or bucket name not found")
+	}
+	cfg.Bucket = path[0]
+	if len(path) > 1 {
+		cfg.Prefix = path[1]
+	} else {
+		cfg.Prefix = defaultPrefix
 	}
 
 	return cfg, nil

backend/s3/config_test.go

@@ -9,18 +9,38 @@ var configTests = []struct {
 	{"s3://eu-central-1/bucketname", Config{
 		Endpoint: "eu-central-1",
 		Bucket:   "bucketname",
+		Prefix:   "restic",
+	}},
+	{"s3://eu-central-1/bucketname/prefix/directory", Config{
+		Endpoint: "eu-central-1",
+		Bucket:   "bucketname",
+		Prefix:   "prefix/directory",
 	}},
 	{"s3:eu-central-1/foobar", Config{
 		Endpoint: "eu-central-1",
 		Bucket:   "foobar",
+		Prefix:   "restic",
+	}},
+	{"s3:eu-central-1/foobar/prefix/directory", Config{
+		Endpoint: "eu-central-1",
+		Bucket:   "foobar",
+		Prefix:   "prefix/directory",
 	}},
 	{"s3:https://hostname:9999/foobar", Config{
 		Endpoint: "hostname:9999",
 		Bucket:   "foobar",
+		Prefix:   "restic",
 	}},
 	{"s3:http://hostname:9999/foobar", Config{
 		Endpoint: "hostname:9999",
 		Bucket:   "foobar",
+		Prefix:   "restic",
+		UseHTTP:  true,
+	}},
+	{"s3:http://hostname:9999/bucket/prefix/directory", Config{
+		Endpoint: "hostname:9999",
+		Bucket:   "bucket",
+		Prefix:   "prefix/directory",
 		UseHTTP:  true,
 	}},
 }

backend/s3/s3.go

@@ -13,13 +13,12 @@ import (
 )
 
 const connLimit = 10
-const backendPrefix = "restic"
 
-func s3path(t backend.Type, name string) string {
+func s3path(prefix string, t backend.Type, name string) string {
 	if t == backend.Config {
-		return backendPrefix + "/" + string(t)
+		return prefix + "/" + string(t)
 	}
-	return backendPrefix + "/" + string(t) + "/" + name
+	return prefix + "/" + string(t) + "/" + name
 }
 
 // s3 is a backend which stores the data on an S3 endpoint.
@@ -27,6 +26,7 @@ type s3 struct {
 	client     minio.CloudStorageClient
 	connChan   chan struct{}
 	bucketname string
+	prefix     string
 }
 
 // Open opens the S3 backend at bucket and region. The bucket is created if it
@@ -39,7 +39,7 @@ func Open(cfg Config) (backend.Backend, error) {
 		return nil, err
 	}
 
-	be := &s3{client: client, bucketname: cfg.Bucket}
+	be := &s3{client: client, bucketname: cfg.Bucket, prefix: cfg.Prefix}
 	be.createConnections()
 
 	if err := client.BucketExists(cfg.Bucket); err != nil {
@@ -72,7 +72,7 @@ func (be *s3) Location() string {
 // and saves it in p. Load has the same semantics as io.ReaderAt.
 func (be s3) Load(h backend.Handle, p []byte, off int64) (int, error) {
 	debug.Log("s3.Load", "%v, offset %v, len %v", h, off, len(p))
-	path := s3path(h.Type, h.Name)
+	path := s3path(be.prefix, h.Type, h.Name)
 	obj, err := be.client.GetObject(be.bucketname, path)
 	if err != nil {
 		debug.Log("s3.GetReader", "  err %v", err)
@@ -101,7 +101,7 @@ func (be s3) Save(h backend.Handle, p []byte) (err error) {
 
 	debug.Log("s3.Save", "%v bytes at %d", len(p), h)
 
-	path := s3path(h.Type, h.Name)
+	path := s3path(be.prefix, h.Type, h.Name)
 
 	// Check key does not already exist
 	_, err = be.client.StatObject(be.bucketname, path)
@@ -126,7 +126,7 @@ func (be s3) Save(h backend.Handle, p []byte) (err error) {
 // Stat returns information about a blob.
 func (be s3) Stat(h backend.Handle) (backend.BlobInfo, error) {
 	debug.Log("s3.Stat", "%v")
-	path := s3path(h.Type, h.Name)
+	path := s3path(be.prefix, h.Type, h.Name)
 	obj, err := be.client.GetObject(be.bucketname, path)
 	if err != nil {
 		debug.Log("s3.Stat", "GetObject() err %v", err)
@@ -145,7 +145,7 @@ func (be s3) Stat(h backend.Handle) (backend.BlobInfo, error) {
 // Test returns true if a blob of the given type and name exists in the backend.
 func (be *s3) Test(t backend.Type, name string) (bool, error) {
 	found := false
-	path := s3path(t, name)
+	path := s3path(be.prefix, t, name)
 	_, err := be.client.StatObject(be.bucketname, path)
 	if err == nil {
 		found = true
@@ -157,7 +157,7 @@ func (be *s3) Test(t backend.Type, name string) (bool, error) {
 
 // Remove removes the blob with the given name and type.
 func (be *s3) Remove(t backend.Type, name string) error {
-	path := s3path(t, name)
+	path := s3path(be.prefix, t, name)
 	err := be.client.RemoveObject(be.bucketname, path)
 	debug.Log("s3.Remove", "%v %v -> err %v", t, name, err)
 	return err
@@ -170,7 +170,7 @@ func (be *s3) List(t backend.Type, done <-chan struct{}) <-chan string {
 	debug.Log("s3.List", "listing %v", t)
 	ch := make(chan string)
 
-	prefix := s3path(t, "")
+	prefix := s3path(be.prefix, t, "")
 
 	listresp := be.client.ListObjects(be.bucketname, prefix, true, done)
 

location/location_test.go

@@ -48,30 +48,56 @@ var parseTests = []struct {
 		Config: s3.Config{
 			Endpoint: "eu-central-1",
 			Bucket:   "bucketname",
+			Prefix:   "restic",
 		}},
 	},
 	{"s3://hostname.foo/bucketname", Location{Scheme: "s3",
 		Config: s3.Config{
 			Endpoint: "hostname.foo",
 			Bucket:   "bucketname",
+			Prefix:   "restic",
+		}},
+	},
+	{"s3://hostname.foo/bucketname/prefix/directory", Location{Scheme: "s3",
+		Config: s3.Config{
+			Endpoint: "hostname.foo",
+			Bucket:   "bucketname",
+			Prefix:   "prefix/directory",
 		}},
 	},
 	{"s3:eu-central-1/repo", Location{Scheme: "s3",
 		Config: s3.Config{
 			Endpoint: "eu-central-1",
 			Bucket:   "repo",
+			Prefix:   "restic",
+		}},
+	},
+	{"s3:eu-central-1/repo/prefix/directory", Location{Scheme: "s3",
+		Config: s3.Config{
+			Endpoint: "eu-central-1",
+			Bucket:   "repo",
+			Prefix:   "prefix/directory",
 		}},
 	},
 	{"s3:https://hostname.foo/repo", Location{Scheme: "s3",
 		Config: s3.Config{
 			Endpoint: "hostname.foo",
 			Bucket:   "repo",
+			Prefix:   "restic",
+		}},
+	},
+	{"s3:https://hostname.foo/repo/prefix/directory", Location{Scheme: "s3",
+		Config: s3.Config{
+			Endpoint: "hostname.foo",
+			Bucket:   "repo",
+			Prefix:   "prefix/directory",
 		}},
 	},
 	{"s3:http://hostname.foo/repo", Location{Scheme: "s3",
 		Config: s3.Config{
 			Endpoint: "hostname.foo",
 			Bucket:   "repo",
+			Prefix:   "restic",
 			UseHTTP:  true,
 		}},
 	},