diff --git a/internal/backend/local/config.go b/internal/backend/local/config.go index 13b7f67aa..e59d1f693 100644 --- a/internal/backend/local/config.go +++ b/internal/backend/local/config.go @@ -11,6 +11,15 @@ import ( type Config struct { Path string Layout string `option:"layout" help:"use this backend directory layout (default: auto-detect)"` + + Connections uint `option:"connections" help:"set a limit for the number of concurrent operations (default: 2)"` +} + +// NewConfig returns a new config with default options applied. +func NewConfig() Config { + return Config{ + Connections: 2, + } } func init() { @@ -18,10 +27,12 @@ func init() { } // ParseConfig parses a local backend config. -func ParseConfig(cfg string) (interface{}, error) { - if !strings.HasPrefix(cfg, "local:") { +func ParseConfig(s string) (interface{}, error) { + if !strings.HasPrefix(s, "local:") { return nil, errors.New(`invalid format, prefix "local" not found`) } - return Config{Path: cfg[6:]}, nil + cfg := NewConfig() + cfg.Path = s[6:] + return cfg, nil } diff --git a/internal/backend/local/layout_test.go b/internal/backend/local/layout_test.go index 5b1135253..9da702877 100644 --- a/internal/backend/local/layout_test.go +++ b/internal/backend/local/layout_test.go @@ -37,8 +37,9 @@ func TestLayout(t *testing.T) { repo := filepath.Join(path, "repo") be, err := Open(context.TODO(), Config{ - Path: repo, - Layout: test.layout, + Path: repo, + Layout: test.layout, + Connections: 2, }) if err != nil { t.Fatal(err) diff --git a/internal/backend/local/local.go b/internal/backend/local/local.go index 833bde26f..0ae023b8e 100644 --- a/internal/backend/local/local.go +++ b/internal/backend/local/local.go @@ -22,6 +22,7 @@ import ( // Local is a backend in a local directory. type Local struct { Config + sem *backend.Semaphore backend.Layout } @@ -30,15 +31,28 @@ var _ restic.Backend = &Local{} const defaultLayout = "default" -// Open opens the local backend as specified by config. -func Open(ctx context.Context, cfg Config) (*Local, error) { - debug.Log("open local backend at %v (layout %q)", cfg.Path, cfg.Layout) +func open(ctx context.Context, cfg Config) (*Local, error) { l, err := backend.ParseLayout(ctx, &backend.LocalFilesystem{}, cfg.Layout, defaultLayout, cfg.Path) if err != nil { return nil, err } - return &Local{Config: cfg, Layout: l}, nil + sem, err := backend.NewSemaphore(cfg.Connections) + if err != nil { + return nil, err + } + + return &Local{ + Config: cfg, + Layout: l, + sem: sem, + }, nil +} + +// Open opens the local backend as specified by config. +func Open(ctx context.Context, cfg Config) (*Local, error) { + debug.Log("open local backend at %v (layout %q)", cfg.Path, cfg.Layout) + return open(ctx, cfg) } // Create creates all the necessary files and directories for a new local @@ -46,16 +60,11 @@ func Open(ctx context.Context, cfg Config) (*Local, error) { func Create(ctx context.Context, cfg Config) (*Local, error) { debug.Log("create local backend at %v (layout %q)", cfg.Path, cfg.Layout) - l, err := backend.ParseLayout(ctx, &backend.LocalFilesystem{}, cfg.Layout, defaultLayout, cfg.Path) + be, err := open(ctx, cfg) if err != nil { return nil, err } - be := &Local{ - Config: cfg, - Layout: l, - } - // test if config file already exists _, err = fs.Lstat(be.Filename(restic.Handle{Type: restic.ConfigFile})) if err == nil { @@ -73,6 +82,10 @@ func Create(ctx context.Context, cfg Config) (*Local, error) { return be, nil } +func (b *Local) Connections() uint { + return b.Config.Connections +} + // Location returns this backend's location (the directory name). func (b *Local) Location() string { return b.Path @@ -105,6 +118,9 @@ func (b *Local) Save(ctx context.Context, h restic.Handle, rd restic.RewindReade } }() + b.sem.GetToken() + defer b.sem.ReleaseToken() + // Create new file with a temporary name. tmpname := filepath.Base(finalname) + "-tmp-" f, err := tempFile(dir, tmpname) @@ -199,24 +215,29 @@ func (b *Local) openReader(ctx context.Context, h restic.Handle, length int, off return nil, errors.New("offset is negative") } + b.sem.GetToken() f, err := fs.Open(b.Filename(h)) if err != nil { + b.sem.ReleaseToken() return nil, err } if offset > 0 { _, err = f.Seek(offset, 0) if err != nil { + b.sem.ReleaseToken() _ = f.Close() return nil, err } } + r := b.sem.ReleaseTokenOnClose(f, nil) + if length > 0 { - return backend.LimitReadCloser(f, int64(length)), nil + return backend.LimitReadCloser(r, int64(length)), nil } - return f, nil + return r, nil } // Stat returns information about a blob. @@ -226,6 +247,9 @@ func (b *Local) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, err return restic.FileInfo{}, backoff.Permanent(err) } + b.sem.GetToken() + defer b.sem.ReleaseToken() + fi, err := fs.Stat(b.Filename(h)) if err != nil { return restic.FileInfo{}, errors.WithStack(err) @@ -237,6 +261,10 @@ func (b *Local) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, err // Test returns true if a blob of the given type and name exists in the backend. func (b *Local) Test(ctx context.Context, h restic.Handle) (bool, error) { debug.Log("Test %v", h) + + b.sem.GetToken() + defer b.sem.ReleaseToken() + _, err := fs.Stat(b.Filename(h)) if err != nil { if b.IsNotExist(err) { @@ -253,6 +281,9 @@ func (b *Local) Remove(ctx context.Context, h restic.Handle) error { debug.Log("Remove %v", h) fn := b.Filename(h) + b.sem.GetToken() + defer b.sem.ReleaseToken() + // reset read-only flag err := fs.Chmod(fn, 0666) if err != nil && !os.IsPermission(err) { diff --git a/internal/backend/local/local_internal_test.go b/internal/backend/local/local_internal_test.go index 8d2ec08c3..8de3d3c2f 100644 --- a/internal/backend/local/local_internal_test.go +++ b/internal/backend/local/local_internal_test.go @@ -27,7 +27,7 @@ func TestNoSpacePermanent(t *testing.T) { dir, cleanup := rtest.TempDir(t) defer cleanup() - be, err := Open(context.Background(), Config{Path: dir}) + be, err := Open(context.Background(), Config{Path: dir, Connections: 2}) rtest.OK(t, err) defer func() { rtest.OK(t, be.Close()) diff --git a/internal/backend/local/local_test.go b/internal/backend/local/local_test.go index 70b5e771e..75c3b8ed7 100644 --- a/internal/backend/local/local_test.go +++ b/internal/backend/local/local_test.go @@ -25,7 +25,8 @@ func newTestSuite(t testing.TB) *test.Suite { t.Logf("create new backend at %v", dir) cfg := local.Config{ - Path: dir, + Path: dir, + Connections: 2, } return cfg, nil }, diff --git a/internal/backend/location/location_test.go b/internal/backend/location/location_test.go index 3160a2af7..ded9450e9 100644 --- a/internal/backend/location/location_test.go +++ b/internal/backend/location/location_test.go @@ -30,7 +30,8 @@ var parseTests = []struct { "local:/srv/repo", Location{Scheme: "local", Config: local.Config{ - Path: "/srv/repo", + Path: "/srv/repo", + Connections: 2, }, }, }, @@ -38,7 +39,8 @@ var parseTests = []struct { "local:dir1/dir2", Location{Scheme: "local", Config: local.Config{ - Path: "dir1/dir2", + Path: "dir1/dir2", + Connections: 2, }, }, }, @@ -46,7 +48,8 @@ var parseTests = []struct { "local:dir1/dir2", Location{Scheme: "local", Config: local.Config{ - Path: "dir1/dir2", + Path: "dir1/dir2", + Connections: 2, }, }, }, @@ -54,7 +57,8 @@ var parseTests = []struct { "dir1/dir2", Location{Scheme: "local", Config: local.Config{ - Path: "dir1/dir2", + Path: "dir1/dir2", + Connections: 2, }, }, }, @@ -62,7 +66,8 @@ var parseTests = []struct { "/dir1/dir2", Location{Scheme: "local", Config: local.Config{ - Path: "/dir1/dir2", + Path: "/dir1/dir2", + Connections: 2, }, }, }, @@ -70,7 +75,8 @@ var parseTests = []struct { "local:../dir1/dir2", Location{Scheme: "local", Config: local.Config{ - Path: "../dir1/dir2", + Path: "../dir1/dir2", + Connections: 2, }, }, }, @@ -78,7 +84,8 @@ var parseTests = []struct { "/dir1/dir2", Location{Scheme: "local", Config: local.Config{ - Path: "/dir1/dir2", + Path: "/dir1/dir2", + Connections: 2, }, }, }, @@ -86,7 +93,8 @@ var parseTests = []struct { "/dir1:foobar/dir2", Location{Scheme: "local", Config: local.Config{ - Path: "/dir1:foobar/dir2", + Path: "/dir1:foobar/dir2", + Connections: 2, }, }, }, @@ -94,7 +102,8 @@ var parseTests = []struct { `\dir1\foobar\dir2`, Location{Scheme: "local", Config: local.Config{ - Path: `\dir1\foobar\dir2`, + Path: `\dir1\foobar\dir2`, + Connections: 2, }, }, }, @@ -102,7 +111,8 @@ var parseTests = []struct { `c:\dir1\foobar\dir2`, Location{Scheme: "local", Config: local.Config{ - Path: `c:\dir1\foobar\dir2`, + Path: `c:\dir1\foobar\dir2`, + Connections: 2, }, }, }, @@ -110,7 +120,8 @@ var parseTests = []struct { `C:\Users\appveyor\AppData\Local\Temp\1\restic-test-879453535\repo`, Location{Scheme: "local", Config: local.Config{ - Path: `C:\Users\appveyor\AppData\Local\Temp\1\restic-test-879453535\repo`, + Path: `C:\Users\appveyor\AppData\Local\Temp\1\restic-test-879453535\repo`, + Connections: 2, }, }, }, @@ -118,7 +129,8 @@ var parseTests = []struct { `c:/dir1/foobar/dir2`, Location{Scheme: "local", Config: local.Config{ - Path: `c:/dir1/foobar/dir2`, + Path: `c:/dir1/foobar/dir2`, + Connections: 2, }, }, }, diff --git a/internal/repository/testing.go b/internal/repository/testing.go index 899b8a7e3..d752e107e 100644 --- a/internal/repository/testing.go +++ b/internal/repository/testing.go @@ -93,7 +93,7 @@ func TestRepository(t testing.TB) (r restic.Repository, cleanup func()) { // TestOpenLocal opens a local repository. func TestOpenLocal(t testing.TB, dir string) (r restic.Repository) { - be, err := local.Open(context.TODO(), local.Config{Path: dir}) + be, err := local.Open(context.TODO(), local.Config{Path: dir, Connections: 2}) if err != nil { t.Fatal(err) }