octoleo/restic
2
2
Fork 0
mirror of https://github.com/octoleo/restic.git synced 2024-12-28 21:02:47 +00:00
Code Issues Releases Wiki Activity

Merge pull request #4067 from MichaelEischer/remove-backend-test-method

Browse Source 
Remove `Test()` method from Backend
This commit is contained in:
Michael Eischer 2022-12-03 17:40:55 +01:00 committed by GitHub
commit 8bf6b2b80d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
24 changed files with 77 additions and 230 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
internal/archiver/archiver_test.go
View File

@ -1873,10 +1873,6 @@ type noCancelBackend struct {
restic.Backend
}
func (c *noCancelBackend) Test(ctx context.Context, h restic.Handle) (bool, error) {
return c.Backend.Test(context.Background(), h)
}
func (c *noCancelBackend) Remove(ctx context.Context, h restic.Handle) error {
return c.Backend.Remove(context.Background(), h)
}

14
internal/backend/azure/azure.go
View File

@ -330,20 +330,6 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo,
return fi, nil
}
// Test returns true if a blob of the given type and name exists in the backend.
func (be *Backend) Test(ctx context.Context, h restic.Handle) (bool, error) {
objName := be.Filename(h)
be.sem.GetToken()
found, err := be.container.GetBlobReference(objName).Exists()
be.sem.ReleaseToken()
if err != nil {
return false, err
}
return found, nil
}
// Remove removes the blob with the given name and type.
func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
objName := be.Filename(h)

6
internal/backend/azure/azure_test.go
View File

@ -51,12 +51,12 @@ func newAzureTestSuite(t testing.TB) *test.Suite {
return nil, err
}
exists, err := be.Test(context.TODO(), restic.Handle{Type: restic.ConfigFile})
if err != nil {
_, err = be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile})
if err != nil && !be.IsNotExist(err) {
return nil, err
}
if exists {
if err == nil {
return nil, errors.New("config already exists")
}

23
internal/backend/b2/b2.go
View File

@ -147,12 +147,12 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backe
sem: sem,
}
present, err := be.Test(ctx, restic.Handle{Type: restic.ConfigFile})
if err != nil {
_, err = be.Stat(ctx, restic.Handle{Type: restic.ConfigFile})
if err != nil && !be.IsNotExist(err) {
return nil, err
}
if present {
if err == nil {
return nil, errors.New("config already exists")
}
@ -287,23 +287,6 @@ func (be *b2Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileI
return restic.FileInfo{Size: info.Size, Name: h.Name}, nil
}
// Test returns true if a blob of the given type and name exists in the backend.
func (be *b2Backend) Test(ctx context.Context, h restic.Handle) (bool, error) {
debug.Log("Test %v", h)
be.sem.GetToken()
defer be.sem.ReleaseToken()
found := false
name := be.Filename(h)
obj := be.bucket.Object(name)
info, err := obj.Attrs(ctx)
if err == nil && info != nil && info.Status == b2.Uploaded {
found = true
}
return found, nil
}
// Remove removes the blob with the given name and type.
func (be *b2Backend) Remove(ctx context.Context, h restic.Handle) error {
debug.Log("Remove %v", h)

4
internal/backend/dryrun/dry_backend.go
View File

@ -86,7 +86,3 @@ func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset
func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
return be.b.Stat(ctx, h)
}
func (be *Backend) Test(ctx context.Context, h restic.Handle) (bool, error) {
return be.b.Test(ctx, h)
}

9
internal/backend/dryrun/dry_backend_test.go
View File

@ -41,12 +41,9 @@ func TestDry(t *testing.T) {
{d, "stat", "a", "", "not found"},
{d, "list", "", "", ""},
{d, "save", "", "", "invalid"},
{d, "test", "a", "", ""},
{m, "save", "a", "baz", ""}, // save a directly to the mem backend
{d, "save", "b", "foob", ""}, // b is not saved
{d, "save", "b", "xxx", ""}, // no error as b is not saved
{d, "test", "a", "1", ""},
{d, "test", "b", "", ""},
{d, "stat", "", "", "invalid"},
{d, "stat", "a", "a 3", ""},
{d, "load", "a", "baz", ""},
@ -65,17 +62,11 @@ func TestDry(t *testing.T) {
for i, step := range steps {
var err error
var boolRes bool
handle := restic.Handle{Type: restic.PackFile, Name: step.fname}
switch step.op {
case "save":
err = step.be.Save(ctx, handle, restic.NewByteReader([]byte(step.content), step.be.Hasher()))
case "test":
boolRes, err = step.be.Test(ctx, handle)
if boolRes != (step.content != "") {
t.Errorf("%d. Test(%q) = %v, want %v", i, step.fname, boolRes, step.content != "")
}
case "list":
fileList := []string{}
err = step.be.List(ctx, restic.PackFile, func(fi restic.FileInfo) error {

16
internal/backend/gs/gs.go
View File

@ -329,22 +329,6 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInf
return restic.FileInfo{Size: attr.Size, Name: h.Name}, nil
}
// Test returns true if a blob of the given type and name exists in the backend.
func (be *Backend) Test(ctx context.Context, h restic.Handle) (bool, error) {
found := false
objName := be.Filename(h)
be.sem.GetToken()
_, err := be.bucket.Object(objName).Attrs(ctx)
be.sem.ReleaseToken()
if err == nil {
found = true
}
// If error, then not found
return found, nil
}
// Remove removes the blob with the given name and type.
func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
objName := be.Filename(h)

6
internal/backend/gs/gs_test.go
View File

@ -47,12 +47,12 @@ func newGSTestSuite(t testing.TB) *test.Suite {
return nil, err
}
exists, err := be.Test(context.TODO(), restic.Handle{Type: restic.ConfigFile})
if err != nil {
_, err = be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile})
if err != nil && !be.IsNotExist(err) {
return nil, err
}
if exists {
if err == nil {
return nil, errors.New("config already exists")
}

18
internal/backend/local/local.go
View File

@ -269,24 +269,6 @@ func (b *Local) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, err
return restic.FileInfo{Size: fi.Size(), Name: h.Name}, nil
}
// Test returns true if a blob of the given type and name exists in the backend.
func (b *Local) Test(ctx context.Context, h restic.Handle) (bool, error) {
debug.Log("Test %v", h)
b.sem.GetToken()
defer b.sem.ReleaseToken()
_, err := fs.Stat(b.Filename(h))
if err != nil {
if b.IsNotExist(err) {
return false, nil
}
return false, errors.WithStack(err)
}
return true, nil
}
// Remove removes the blob with the given name and type.
func (b *Local) Remove(ctx context.Context, h restic.Handle) error {
debug.Log("Remove %v", h)

17
internal/backend/mem/mem_backend.go
View File

@ -52,23 +52,6 @@ func New() *MemoryBackend {
return be
}
// Test returns whether a file exists.
func (be *MemoryBackend) Test(ctx context.Context, h restic.Handle) (bool, error) {
be.sem.GetToken()
defer be.sem.ReleaseToken()
be.m.Lock()
defer be.m.Unlock()
debug.Log("Test %v", h)
if _, ok := be.data[h]; ok {
return true, ctx.Err()
}
return false, ctx.Err()
}
// IsNotExist returns true if the file does not exist.
func (be *MemoryBackend) IsNotExist(err error) bool {
return errors.Is(err, errNotFound)

6
internal/backend/mem/mem_backend_test.go
View File

@ -26,12 +26,12 @@ func newTestSuite() *test.Suite {
Create: func(cfg interface{}) (restic.Backend, error) {
c := cfg.(*memConfig)
if c.be != nil {
ok, err := c.be.Test(context.TODO(), restic.Handle{Type: restic.ConfigFile})
if err != nil {
_, err := c.be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile})
if err != nil && !c.be.IsNotExist(err) {
return nil, err
}
if ok {
if err == nil {
return nil, errors.New("config already exists")
}
}

10
internal/backend/mock/backend.go
View File

@ -18,7 +18,6 @@ type Backend struct {
StatFn func(ctx context.Context, h restic.Handle) (restic.FileInfo, error)
ListFn func(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error
RemoveFn func(ctx context.Context, h restic.Handle) error
TestFn func(ctx context.Context, h restic.Handle) (bool, error)
DeleteFn func(ctx context.Context) error
ConnectionsFn func() uint
LocationFn func() string
@ -143,15 +142,6 @@ func (m *Backend) Remove(ctx context.Context, h restic.Handle) error {
return m.RemoveFn(ctx, h)
}
// Test for the existence of a specific item.
func (m *Backend) Test(ctx context.Context, h restic.Handle) (bool, error) {
if m.TestFn == nil {
return false, errors.New("not implemented")
}
return m.TestFn(ctx, h)
}
// Delete all data.
func (m *Backend) Delete(ctx context.Context) error {
if m.DeleteFn == nil {

10
internal/backend/rest/rest.go
View File

@ -308,16 +308,6 @@ func (b *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, e
return bi, nil
}
// Test returns true if a blob of the given type and name exists in the backend.
func (b *Backend) Test(ctx context.Context, h restic.Handle) (bool, error) {
_, err := b.Stat(ctx, h)
if err != nil {
return false, nil
}
return true, nil
}
// Remove removes the blob with the given name and type.
func (b *Backend) Remove(ctx context.Context, h restic.Handle) error {
if err := h.Valid(); err != nil {

15
internal/backend/retry/backend_retry.go
View File

@ -139,6 +139,10 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (fi restic.FileInf
var innerError error
fi, innerError = be.Backend.Stat(ctx, h)
if be.Backend.IsNotExist(innerError) {
// do not retry if file is not found, as stat is usually used to check whether a file exists
return backoff.Permanent(innerError)
}
return innerError
})
return fi, err
@ -151,17 +155,6 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) (err error) {
})
}
// Test a boolean value whether a File with the name and type exists.
func (be *Backend) Test(ctx context.Context, h restic.Handle) (exists bool, err error) {
err = be.retry(ctx, fmt.Sprintf("Test(%v)", h), func() error {
var innerError error
exists, innerError = be.Backend.Test(ctx, h)
return innerError
})
return exists, err
}
// List runs fn for each file in the backend which has the type t. When an
// error is returned by the underlying backend, the request is retried. When fn
// returns an error, the operation is aborted and the error is returned to the

30
internal/backend/retry/backend_retry_test.go
View File

@ -274,6 +274,32 @@ func TestBackendLoadRetry(t *testing.T) {
test.Equals(t, 2, attempt)
}
func TestBackendStatNotExists(t *testing.T) {
// stat should not retry if the error matches IsNotExist
notFound := errors.New("not found")
attempt := 0
be := mock.NewBackend()
be.StatFn = func(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
attempt++
if attempt > 1 {
t.Fail()
return restic.FileInfo{}, errors.New("must not retry")
}
return restic.FileInfo{}, notFound
}
be.IsNotExistFn = func(err error) bool {
return errors.Is(err, notFound)
}
TestFastRetries(t)
retryBackend := New(be, 10, nil, nil)
_, err := retryBackend.Stat(context.TODO(), restic.Handle{})
test.Assert(t, be.IsNotExistFn(err), "unexpected error %v", err)
test.Equals(t, 1, attempt)
}
func assertIsCanceled(t *testing.T, err error) {
test.Assert(t, err == context.Canceled, "got unexpected err %v", err)
}
@ -289,9 +315,7 @@ func TestBackendCanceledContext(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
_, err := retryBackend.Test(ctx, h)
assertIsCanceled(t, err)
_, err = retryBackend.Stat(ctx, h)
_, err := retryBackend.Stat(ctx, h)
assertIsCanceled(t, err)
err = retryBackend.Save(ctx, h, restic.NewByteReader([]byte{}, nil))

17
internal/backend/s3/s3.go
View File

@ -392,23 +392,6 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInf
return restic.FileInfo{Size: fi.Size, Name: h.Name}, nil
}
// Test returns true if a blob of the given type and name exists in the backend.
func (be *Backend) Test(ctx context.Context, h restic.Handle) (bool, error) {
found := false
objName := be.Filename(h)
be.sem.GetToken()
_, err := be.client.StatObject(ctx, be.cfg.Bucket, objName, minio.StatObjectOptions{})
be.sem.ReleaseToken()
if err == nil {
found = true
}
// If error, then not found
return found, nil
}
// Remove removes the blob with the given name and type.
func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
objName := be.Filename(h)

12
internal/backend/s3/s3_test.go
View File

@ -155,12 +155,12 @@ func newMinioTestSuite(ctx context.Context, t testing.TB) *test.Suite {
return nil, err
}
exists, err := be.Test(ctx, restic.Handle{Type: restic.ConfigFile})
if err != nil {
_, err = be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile})
if err != nil && !be.IsNotExist(err) {
return nil, err
}
if exists {
if err == nil {
return nil, errors.New("config already exists")
}
@ -254,12 +254,12 @@ func newS3TestSuite(t testing.TB) *test.Suite {
return nil, err
}
exists, err := be.Test(context.TODO(), restic.Handle{Type: restic.ConfigFile})
if err != nil {
_, err = be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile})
if err != nil && !be.IsNotExist(err) {
return nil, err
}
if exists {
if err == nil {
return nil, errors.New("config already exists")
}

22
internal/backend/sftp/sftp.go
View File

@ -495,28 +495,6 @@ func (r *SFTP) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, erro
return restic.FileInfo{Size: fi.Size(), Name: h.Name}, nil
}
// Test returns true if a blob of the given type and name exists in the backend.
func (r *SFTP) Test(ctx context.Context, h restic.Handle) (bool, error) {
debug.Log("Test(%v)", h)
if err := r.clientError(); err != nil {
return false, err
}
r.sem.GetToken()
defer r.sem.ReleaseToken()
_, err := r.c.Lstat(r.Filename(h))
if errors.Is(err, os.ErrNotExist) {
return false, nil
}
if err != nil {
return false, errors.Wrap(err, "Lstat")
}
return true, nil
}
// Remove removes the content stored at name.
func (r *SFTP) Remove(ctx context.Context, h restic.Handle) error {
debug.Log("Remove(%v)", h)

19
internal/backend/swift/swift.go
View File

@ -226,25 +226,6 @@ func (be *beSwift) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInf
return restic.FileInfo{Size: obj.Bytes, Name: h.Name}, nil
}
// Test returns true if a blob of the given type and name exists in the backend.
func (be *beSwift) Test(ctx context.Context, h restic.Handle) (bool, error) {
objName := be.Filename(h)
be.sem.GetToken()
defer be.sem.ReleaseToken()
switch _, _, err := be.conn.Object(ctx, be.container, objName); err {
case nil:
return true, nil
case swift.ObjectNotFound:
return false, nil
default:
return false, errors.Wrap(err, "conn.Object")
}
}
// Remove removes the blob with the given name and type.
func (be *beSwift) Remove(ctx context.Context, h restic.Handle) error {
objName := be.Filename(h)

6
internal/backend/swift/swift_test.go
View File

@ -66,12 +66,12 @@ func newSwiftTestSuite(t testing.TB) *test.Suite {
return nil, err
}
exists, err := be.Test(context.TODO(), restic.Handle{Type: restic.ConfigFile})
if err != nil {
_, err = be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile})
if err != nil && !be.IsNotExist(err) {
return nil, err
}
if exists {
if err == nil {
return nil, errors.New("config already exists")
}

25
internal/backend/test/tests.go
View File

@ -27,6 +27,15 @@ func seedRand(t testing.TB) {
t.Logf("rand initialized with seed %d", seed)
}
func beTest(ctx context.Context, be restic.Backend, h restic.Handle) (bool, error) {
_, err := be.Stat(ctx, h)
if err != nil && be.IsNotExist(err) {
return false, nil
}
return err == nil, err
}
// TestCreateWithConfig tests that creating a backend in a location which already
// has a config file fails.
func (s *Suite) TestCreateWithConfig(t *testing.T) {
@ -35,7 +44,7 @@ func (s *Suite) TestCreateWithConfig(t *testing.T) {
// remove a config if present
cfgHandle := restic.Handle{Type: restic.ConfigFile}
cfgPresent, err := b.Test(context.TODO(), cfgHandle)
cfgPresent, err := beTest(context.TODO(), b, cfgHandle)
if err != nil {
t.Fatalf("unable to test for config: %+v", err)
}
@ -642,7 +651,7 @@ func (s *Suite) TestSaveWrongHash(t *testing.T) {
// test that upload with hash mismatch fails
h := restic.Handle{Type: restic.PackFile, Name: id.String()}
err := b.Save(context.TODO(), h, &wrongByteReader{ByteReader: *restic.NewByteReader(data, b.Hasher())})
exists, err2 := b.Test(context.TODO(), h)
exists, err2 := beTest(context.TODO(), b, h)
if err2 != nil {
t.Fatal(err2)
}
@ -702,7 +711,7 @@ func (s *Suite) delayedRemove(t testing.TB, be restic.Backend, handles ...restic
var found bool
var err error
for time.Since(start) <= s.WaitForDelayedRemoval {
found, err = be.Test(context.TODO(), h)
found, err = beTest(context.TODO(), be, h)
if s.ErrorHandler != nil {
err = s.ErrorHandler(t, be, err)
}
@ -764,7 +773,7 @@ func (s *Suite) TestBackend(t *testing.T) {
// test if blob is already in repository
h := restic.Handle{Type: tpe, Name: id.String()}
ret, err := b.Test(context.TODO(), h)
ret, err := beTest(context.TODO(), b, h)
test.OK(t, err)
test.Assert(t, !ret, "blob was found to exist before creating")
@ -777,7 +786,7 @@ func (s *Suite) TestBackend(t *testing.T) {
test.Assert(t, err != nil, "blob could be read before creation")
// try to get string out, should fail
ret, err = b.Test(context.TODO(), h)
ret, err = beTest(context.TODO(), b, h)
test.OK(t, err)
test.Assert(t, !ret, "id %q was found (but should not have)", ts.id)
}
@ -818,7 +827,7 @@ func (s *Suite) TestBackend(t *testing.T) {
test.OK(t, err)
// test that the blob is gone
ok, err := b.Test(context.TODO(), h)
ok, err := beTest(context.TODO(), b, h)
test.OK(t, err)
test.Assert(t, !ok, "removed blob still present")
@ -854,9 +863,9 @@ func (s *Suite) TestBackend(t *testing.T) {
h := restic.Handle{Type: tpe, Name: id.String()}
found, err := b.Test(context.TODO(), h)
found, err := beTest(context.TODO(), b, h)
test.OK(t, err)
test.Assert(t, found, fmt.Sprintf("id %q not found", id))
test.Assert(t, found, fmt.Sprintf("id %v/%q not found", tpe, id))
handles = append(handles, h)
}

6
internal/repository/repository.go
View File

@ -738,11 +738,11 @@ func (r *Repository) Init(ctx context.Context, version uint, password string, ch
return fmt.Errorf("repository version %v too low", version)
}
has, err := r.be.Test(ctx, restic.Handle{Type: restic.ConfigFile})
if err != nil {
_, err := r.be.Stat(ctx, restic.Handle{Type: restic.ConfigFile})
if err != nil && !r.be.IsNotExist(err) {
return err
}
if has {
if err == nil {
return errors.New("repository master key and config already initialized")
}

3
internal/restic/backend.go
View File

@ -27,9 +27,6 @@ type Backend interface {
// HasAtomicReplace returns whether Save() can atomically replace files
HasAtomicReplace() bool
// Test a boolean value whether a File with the name and type exists.
Test(ctx context.Context, h Handle) (bool, error)
// Remove removes a File described by h.
Remove(ctx context.Context, h Handle) error

9
internal/restic/lock_test.go
View File

@ -193,10 +193,11 @@ func TestLockStale(t *testing.T) {
func lockExists(repo restic.Repository, t testing.TB, id restic.ID) bool {
h := restic.Handle{Type: restic.LockFile, Name: id.String()}
exists, err := repo.Backend().Test(context.TODO(), h)
rtest.OK(t, err)
return exists
_, err := repo.Backend().Stat(context.TODO(), h)
if err != nil && !repo.Backend().IsNotExist(err) {
t.Fatal(err)
}
return err == nil
}
func TestLockWithStaleLock(t *testing.T) {