From 54f88606121d504cccf493ea789478570ea65e42 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 24 Jan 2016 16:59:38 +0100 Subject: [PATCH] backends: Add Save() --- backend/handle.go | 2 +- backend/local/backend_test.go | 14 ++++++ backend/local/local.go | 58 +++++++++++++++++++++-- backend/mem/backend_test.go | 14 ++++++ backend/mem/mem_backend.go | 4 ++ backend/s3/backend_test.go | 14 ++++++ backend/sftp/backend_test.go | 14 ++++++ backend/sftp/sftp.go | 23 ++++++--- backend/test/backend_test.go | 14 ++++++ backend/test/tests.go | 87 +++++++++++++++++++++++++++++++++++ 10 files changed, 233 insertions(+), 11 deletions(-) diff --git a/backend/handle.go b/backend/handle.go index 9c7a443ff..6bdf6af23 100644 --- a/backend/handle.go +++ b/backend/handle.go @@ -33,7 +33,7 @@ func (h Handle) Valid() error { case Index: case Config: default: - return fmt.Errorf("invalid config %q", h.Type) + return fmt.Errorf("invalid Type %q", h.Type) } if h.Type == Config { diff --git a/backend/local/backend_test.go b/backend/local/backend_test.go index c6032be6b..5617ab74a 100644 --- a/backend/local/backend_test.go +++ b/backend/local/backend_test.go @@ -58,6 +58,20 @@ func TestLocalBackendWrite(t *testing.T) { test.TestWrite(t) } +func TestLocalBackendSave(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestSave(t) +} + +func TestLocalBackendSaveFilenames(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestSaveFilenames(t) +} + func TestLocalBackendBackend(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) diff --git a/backend/local/local.go b/backend/local/local.go index 4183ff474..5f836a8a5 100644 --- a/backend/local/local.go +++ b/backend/local/local.go @@ -11,6 +11,7 @@ import ( "sync" "github.com/restic/restic/backend" + "github.com/restic/restic/debug" ) var ErrWrongData = errors.New("wrong data returned by backend, checksum does not match") @@ -231,12 +232,14 @@ func (b *Local) Save(h backend.Handle, p []byte) (err error) { return err } - f, err := os.Create(filename(b.p, h.Type, h.Name)) + tmpfile, err := ioutil.TempFile(filepath.Join(b.p, backend.Paths.Temp), "temp-") if err != nil { return err } - n, err := f.Write(p) + debug.Log("local.Save", "save %v (%d bytes) to %v", h, len(p), tmpfile.Name()) + + n, err := tmpfile.Write(p) if err != nil { return err } @@ -245,11 +248,58 @@ func (b *Local) Save(h backend.Handle, p []byte) (err error) { return errors.New("not all bytes writen") } - if err = f.Sync(); err != nil { + if err = tmpfile.Sync(); err != nil { return err } - return f.Close() + err = tmpfile.Close() + if err != nil { + return err + } + + f := filename(b.p, h.Type, h.Name) + + // create directories if necessary, ignore errors + if h.Type == backend.Data { + os.MkdirAll(filepath.Dir(f), backend.Modes.Dir) + } + + // test if new path already exists + if _, err := os.Stat(f); err == nil { + return fmt.Errorf("Rename(): file %v already exists", f) + } + + err = os.Rename(tmpfile.Name(), f) + debug.Log("local.Save", "save %v: rename %v -> %v: %v", + h, filepath.Base(tmpfile.Name()), filepath.Base(f), err) + + if err != nil { + return err + } + + // set mode to read-only + fi, err := os.Stat(f) + if err != nil { + return err + } + + err = setNewFileMode(f, fi) + if err != nil { + return err + } + + // try to flush directory + d, err := os.Open(filepath.Dir(f)) + if err != nil { + return err + } + + err = d.Sync() + if err != nil { + return err + } + + return d.Close() } // Stat returns information about a blob. diff --git a/backend/mem/backend_test.go b/backend/mem/backend_test.go index 681d0a680..9b1a602a1 100644 --- a/backend/mem/backend_test.go +++ b/backend/mem/backend_test.go @@ -58,6 +58,20 @@ func TestMemBackendWrite(t *testing.T) { test.TestWrite(t) } +func TestMemBackendSave(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestSave(t) +} + +func TestMemBackendSaveFilenames(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestSaveFilenames(t) +} + func TestMemBackendBackend(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) diff --git a/backend/mem/mem_backend.go b/backend/mem/mem_backend.go index 0de0c0837..91831dc5e 100644 --- a/backend/mem/mem_backend.go +++ b/backend/mem/mem_backend.go @@ -45,6 +45,10 @@ func New() *MemoryBackend { return memLoad(be, h, p, off) } + be.MockBackend.SaveFn = func(h backend.Handle, p []byte) error { + return memSave(be, h, p) + } + be.MockBackend.StatFn = func(h backend.Handle) (backend.BlobInfo, error) { return memStat(be, h) } diff --git a/backend/s3/backend_test.go b/backend/s3/backend_test.go index cc981cf18..4bd4d6bee 100644 --- a/backend/s3/backend_test.go +++ b/backend/s3/backend_test.go @@ -58,6 +58,20 @@ func TestS3BackendWrite(t *testing.T) { test.TestWrite(t) } +func TestS3BackendSave(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestSave(t) +} + +func TestS3BackendSaveFilenames(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestSaveFilenames(t) +} + func TestS3BackendBackend(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) diff --git a/backend/sftp/backend_test.go b/backend/sftp/backend_test.go index 217f34b11..ca268b3a3 100644 --- a/backend/sftp/backend_test.go +++ b/backend/sftp/backend_test.go @@ -58,6 +58,20 @@ func TestSftpBackendWrite(t *testing.T) { test.TestWrite(t) } +func TestSftpBackendSave(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestSave(t) +} + +func TestSftpBackendSaveFilenames(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestSaveFilenames(t) +} + func TestSftpBackendBackend(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go index c5017bdfc..a7ad4955e 100644 --- a/backend/sftp/sftp.go +++ b/backend/sftp/sftp.go @@ -15,6 +15,7 @@ import ( "github.com/juju/errors" "github.com/pkg/sftp" "github.com/restic/restic/backend" + "github.com/restic/restic/debug" ) const ( @@ -379,12 +380,10 @@ func (r *SFTP) Save(h backend.Handle, p []byte) (err error) { return err } - f, err := r.c.Create(r.filename(h.Type, h.Name)) - if err != nil { - return err - } + filename, tmpfile, err := r.tempFile() + debug.Log("sftp.Save", "save %v (%d bytes) to %v", h, len(p), filename) - n, err := f.Write(p) + n, err := tmpfile.Write(p) if err != nil { return err } @@ -393,7 +392,19 @@ func (r *SFTP) Save(h backend.Handle, p []byte) (err error) { return errors.New("not all bytes writen") } - return f.Close() + err = tmpfile.Close() + if err != nil { + return err + } + + err = r.renameFile(filename, h.Type, h.Name) + debug.Log("sftp.Save", "save %v: rename %v: %v", + h, filepath.Base(filename), err) + if err != nil { + return fmt.Errorf("sftp: renameFile: %v", err) + } + + return nil } // Stat returns information about a blob. diff --git a/backend/test/backend_test.go b/backend/test/backend_test.go index 733fecaaa..e9c19680d 100644 --- a/backend/test/backend_test.go +++ b/backend/test/backend_test.go @@ -58,6 +58,20 @@ func TestTestBackendWrite(t *testing.T) { test.TestWrite(t) } +func TestTestBackendSave(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestSave(t) +} + +func TestTestBackendSaveFilenames(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestSaveFilenames(t) +} + func TestTestBackendBackend(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) diff --git a/backend/test/tests.go b/backend/test/tests.go index 009ef1172..f91b59da1 100644 --- a/backend/test/tests.go +++ b/backend/test/tests.go @@ -325,6 +325,93 @@ func TestWrite(t testing.TB) { } } +// TestSave tests saving data in the backend. +func TestSave(t testing.TB) { + b := open(t) + defer close(t) + + length := rand.Intn(1<<23) + 2000 + data := make([]byte, length) + + for i := 0; i < 10; i++ { + _, err := io.ReadFull(crand.Reader, data) + OK(t, err) + id := backend.Hash(data) + + h := backend.Handle{ + Type: backend.Data, + Name: fmt.Sprintf("%s-%d", id, i), + } + err = b.Save(h, data) + OK(t, err) + + buf, err := backend.LoadAll(b, h, nil) + OK(t, err) + if len(buf) != len(data) { + t.Fatalf("number of bytes does not match, want %v, got %v", len(data), len(buf)) + } + + if !bytes.Equal(buf, data) { + t.Fatalf("data not equal") + } + + fi, err := b.Stat(h) + OK(t, err) + + if fi.Size != int64(len(data)) { + t.Fatalf("Stat() returned different size, want %q, got %d", len(data), fi.Size) + } + + err = b.Remove(h.Type, h.Name) + if err != nil { + t.Fatalf("error removing item: %v", err) + } + } +} + +var filenameTests = []struct { + name string + data string +}{ + {"1dfc6bc0f06cb255889e9ea7860a5753e8eb9665c9a96627971171b444e3113e", "x"}, + {"foobar", "foobar"}, + { + "1dfc6bc0f06cb255889e9ea7860a5753e8eb9665c9a96627971171b444e3113e4bf8f2d9144cc5420a80f04a4880ad6155fc58903a4fb6457c476c43541dcaa6-5", + "foobar content of data blob", + }, +} + +// TestSaveFilenames tests saving data with various file names in the backend. +func TestSaveFilenames(t testing.TB) { + b := open(t) + defer close(t) + + for i, test := range filenameTests { + h := backend.Handle{Name: test.name, Type: backend.Data} + err := b.Save(h, []byte(test.data)) + if err != nil { + t.Errorf("test %d failed: Save() returned %v", i, err) + continue + } + + buf, err := backend.LoadAll(b, h, nil) + if err != nil { + t.Errorf("test %d failed: Load() returned %v", i, err) + continue + } + + if !bytes.Equal(buf, []byte(test.data)) { + t.Errorf("test %d: returned wrong bytes", i) + } + + err = b.Remove(h.Type, h.Name) + if err != nil { + t.Errorf("test %d failed: Remove() returned %v", i, err) + continue + } + } +} + var testStrings = []struct { id string data string