From 4f97492d28126d80311736301fbbe1a98694f6b0 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 7 Aug 2021 22:20:49 +0200 Subject: [PATCH] Backend: Expose connections parameter --- internal/backend/azure/azure.go | 6 +++++ internal/backend/b2/b2.go | 4 ++++ internal/backend/dryrun/dry_backend.go | 4 ++++ internal/backend/gs/gs.go | 18 ++++++++++----- internal/backend/mem/mem_backend.go | 4 ++++ internal/backend/rest/rest.go | 20 +++++++++++------ internal/backend/s3/s3.go | 4 ++++ internal/backend/swift/swift.go | 20 +++++++++++------ internal/mock/backend.go | 31 +++++++++++++++++--------- internal/restic/backend.go | 3 +++ 10 files changed, 83 insertions(+), 31 deletions(-) diff --git a/internal/backend/azure/azure.go b/internal/backend/azure/azure.go index b20579f2c..243a1eaef 100644 --- a/internal/backend/azure/azure.go +++ b/internal/backend/azure/azure.go @@ -24,6 +24,7 @@ import ( type Backend struct { accountName string container *storage.Container + connections uint sem *backend.Semaphore prefix string listMaxItems int @@ -55,6 +56,7 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) { be := &Backend{ container: service.GetContainerReference(cfg.Container), accountName: cfg.AccountName, + connections: cfg.Connections, sem: sem, prefix: cfg.Prefix, Layout: &backend.DefaultLayout{ @@ -109,6 +111,10 @@ func (be *Backend) Join(p ...string) string { return path.Join(p...) } +func (be *Backend) Connections() uint { + return be.connections +} + // Location returns this backend's location (the container name). func (be *Backend) Location() string { return be.Join(be.container.Name, be.prefix) diff --git a/internal/backend/b2/b2.go b/internal/backend/b2/b2.go index 90aeca3b2..6108aaf5c 100644 --- a/internal/backend/b2/b2.go +++ b/internal/backend/b2/b2.go @@ -133,6 +133,10 @@ func (be *b2Backend) SetListMaxItems(i int) { be.listMaxItems = i } +func (be *b2Backend) Connections() uint { + return be.cfg.Connections +} + // Location returns the location for the backend. func (be *b2Backend) Location() string { return be.cfg.Bucket diff --git a/internal/backend/dryrun/dry_backend.go b/internal/backend/dryrun/dry_backend.go index 8412bd26a..44eee9a45 100644 --- a/internal/backend/dryrun/dry_backend.go +++ b/internal/backend/dryrun/dry_backend.go @@ -45,6 +45,10 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { return nil } +func (be *Backend) Connections() uint { + return be.b.Connections() +} + // Location returns the location of the backend. func (be *Backend) Location() string { return "DRY:" + be.b.Location() diff --git a/internal/backend/gs/gs.go b/internal/backend/gs/gs.go index 443de70e5..c87211be3 100644 --- a/internal/backend/gs/gs.go +++ b/internal/backend/gs/gs.go @@ -34,6 +34,7 @@ import ( type Backend struct { gcsClient *storage.Client projectID string + connections uint sem *backend.Semaphore bucketName string bucket *storage.BucketHandle @@ -102,12 +103,13 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) { } be := &Backend{ - gcsClient: gcsClient, - projectID: cfg.ProjectID, - sem: sem, - bucketName: cfg.Bucket, - bucket: gcsClient.Bucket(cfg.Bucket), - prefix: cfg.Prefix, + gcsClient: gcsClient, + projectID: cfg.ProjectID, + connections: cfg.Connections, + sem: sem, + bucketName: cfg.Bucket, + bucket: gcsClient.Bucket(cfg.Bucket), + prefix: cfg.Prefix, Layout: &backend.DefaultLayout{ Path: cfg.Prefix, Join: path.Join, @@ -185,6 +187,10 @@ func (be *Backend) Join(p ...string) string { return path.Join(p...) } +func (be *Backend) Connections() uint { + return be.connections +} + // Location returns this backend's location (the bucket name). func (be *Backend) Location() string { return be.Join(be.bucketName, be.prefix) diff --git a/internal/backend/mem/mem_backend.go b/internal/backend/mem/mem_backend.go index 9e3cd0e74..8f3a52d02 100644 --- a/internal/backend/mem/mem_backend.go +++ b/internal/backend/mem/mem_backend.go @@ -229,6 +229,10 @@ func (be *MemoryBackend) List(ctx context.Context, t restic.FileType, fn func(re return ctx.Err() } +func (be *MemoryBackend) Connections() uint { + return 2 +} + // Location returns the location of the backend (RAM). func (be *MemoryBackend) Location() string { return "RAM" diff --git a/internal/backend/rest/rest.go b/internal/backend/rest/rest.go index c7675cba1..1e372229a 100644 --- a/internal/backend/rest/rest.go +++ b/internal/backend/rest/rest.go @@ -29,9 +29,10 @@ var _ restic.Backend = &Backend{} // Backend uses the REST protocol to access data stored on a server. type Backend struct { - url *url.URL - sem *backend.Semaphore - client *http.Client + url *url.URL + connections uint + sem *backend.Semaphore + client *http.Client backend.Layout } @@ -57,10 +58,11 @@ func Open(cfg Config, rt http.RoundTripper) (*Backend, error) { } be := &Backend{ - url: cfg.URL, - client: client, - Layout: &backend.RESTLayout{URL: url, Join: path.Join}, - sem: sem, + url: cfg.URL, + client: client, + Layout: &backend.RESTLayout{URL: url, Join: path.Join}, + connections: cfg.Connections, + sem: sem, } return be, nil @@ -105,6 +107,10 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, er return be, nil } +func (b *Backend) Connections() uint { + return b.connections +} + // Location returns this backend's location (the server's URL). func (b *Backend) Location() string { return b.url.String() diff --git a/internal/backend/s3/s3.go b/internal/backend/s3/s3.go index 0d7c74bf4..1bdf2d795 100644 --- a/internal/backend/s3/s3.go +++ b/internal/backend/s3/s3.go @@ -255,6 +255,10 @@ func (be *Backend) ReadDir(ctx context.Context, dir string) (list []os.FileInfo, return list, nil } +func (be *Backend) Connections() uint { + return be.cfg.Connections +} + // Location returns this backend's location (the bucket name). func (be *Backend) Location() string { return be.Join(be.cfg.Bucket, be.cfg.Prefix) diff --git a/internal/backend/swift/swift.go b/internal/backend/swift/swift.go index 8d82d90cb..6157002b5 100644 --- a/internal/backend/swift/swift.go +++ b/internal/backend/swift/swift.go @@ -24,10 +24,11 @@ import ( // beSwift is a backend which stores the data on a swift endpoint. type beSwift struct { - conn *swift.Connection - sem *backend.Semaphore - container string // Container name - prefix string // Prefix of object names in the container + conn *swift.Connection + connections uint + sem *backend.Semaphore + container string // Container name + prefix string // Prefix of object names in the container backend.Layout } @@ -68,9 +69,10 @@ func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend Transport: rt, }, - sem: sem, - container: cfg.Container, - prefix: cfg.Prefix, + connections: cfg.Connections, + sem: sem, + container: cfg.Container, + prefix: cfg.Prefix, Layout: &backend.DefaultLayout{ Path: cfg.Prefix, Join: path.Join, @@ -113,6 +115,10 @@ func (be *beSwift) createContainer(ctx context.Context, policy string) error { return be.conn.ContainerCreate(ctx, be.container, h) } +func (be *beSwift) Connections() uint { + return be.connections +} + // Location returns this backend's location (the container name). func (be *beSwift) Location() string { return be.container diff --git a/internal/mock/backend.go b/internal/mock/backend.go index 9f6036fdb..05fe1dc6e 100644 --- a/internal/mock/backend.go +++ b/internal/mock/backend.go @@ -11,17 +11,18 @@ import ( // Backend implements a mock backend. type Backend struct { - CloseFn func() error - IsNotExistFn func(err error) bool - SaveFn func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error - OpenReaderFn func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) - StatFn func(ctx context.Context, h restic.Handle) (restic.FileInfo, error) - ListFn func(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error - RemoveFn func(ctx context.Context, h restic.Handle) error - TestFn func(ctx context.Context, h restic.Handle) (bool, error) - DeleteFn func(ctx context.Context) error - LocationFn func() string - HasherFn func() hash.Hash + CloseFn func() error + IsNotExistFn func(err error) bool + SaveFn func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error + OpenReaderFn func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) + StatFn func(ctx context.Context, h restic.Handle) (restic.FileInfo, error) + ListFn func(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error + RemoveFn func(ctx context.Context, h restic.Handle) error + TestFn func(ctx context.Context, h restic.Handle) (bool, error) + DeleteFn func(ctx context.Context) error + ConnectionsFn func() uint + LocationFn func() string + HasherFn func() hash.Hash } // NewBackend returns new mock Backend instance @@ -39,6 +40,14 @@ func (m *Backend) Close() error { return m.CloseFn() } +func (m *Backend) Connections() uint { + if m.ConnectionsFn == nil { + return 2 + } + + return m.ConnectionsFn() +} + // Location returns a location string. func (m *Backend) Location() string { if m.LocationFn == nil { diff --git a/internal/restic/backend.go b/internal/restic/backend.go index 41292470a..1203bf3d3 100644 --- a/internal/restic/backend.go +++ b/internal/restic/backend.go @@ -18,6 +18,9 @@ type Backend interface { // repository. Location() string + // Connections returns the maxmimum number of concurrent backend operations. + Connections() uint + // Hasher may return a hash function for calculating a content hash for the backend Hasher() hash.Hash