diff --git a/backend/handle.go b/backend/handle.go new file mode 100644 index 000000000..9c7a443ff --- /dev/null +++ b/backend/handle.go @@ -0,0 +1,48 @@ +package backend + +import ( + "errors" + "fmt" +) + +// Handle is used to store and access data in a backend. +type Handle struct { + Type Type + Name string +} + +func (h Handle) String() string { + name := h.Name + if len(name) > 10 { + name = name[:10] + } + return fmt.Sprintf("<%s/%s>", h.Type, name) +} + +// Valid returns an error if h is not valid. +func (h Handle) Valid() error { + if h.Type == "" { + return errors.New("type is empty") + } + + switch h.Type { + case Data: + case Key: + case Lock: + case Snapshot: + case Index: + case Config: + default: + return fmt.Errorf("invalid config %q", h.Type) + } + + if h.Type == Config { + return nil + } + + if h.Name == "" { + return errors.New("invalid Name") + } + + return nil +} diff --git a/backend/handle_test.go b/backend/handle_test.go new file mode 100644 index 000000000..a477c0aec --- /dev/null +++ b/backend/handle_test.go @@ -0,0 +1,28 @@ +package backend + +import "testing" + +var handleTests = []struct { + h Handle + valid bool +}{ + {Handle{Name: "foo"}, false}, + {Handle{Type: "foobar"}, false}, + {Handle{Type: Config, Name: ""}, true}, + {Handle{Type: Data, Name: ""}, false}, + {Handle{Type: "", Name: "x"}, false}, + {Handle{Type: Lock, Name: "010203040506"}, true}, +} + +func TestHandleValid(t *testing.T) { + for i, test := range handleTests { + err := test.h.Valid() + if err != nil && test.valid { + t.Errorf("test %v failed: error returned for valid handle: %v", i, err) + } + + if !test.valid && err == nil { + t.Errorf("test %v failed: expected error for invalid handle not found", i) + } + } +} diff --git a/backend/interface.go b/backend/interface.go index 73c3e82b6..1fdacb4d7 100644 --- a/backend/interface.go +++ b/backend/interface.go @@ -1,9 +1,6 @@ package backend -import ( - "fmt" - "io" -) +import "io" // Type is the type of a Blob. type Type string @@ -18,20 +15,6 @@ const ( Config = "config" ) -// Handle is used to store and access data in a backend. -type Handle struct { - Type Type - Name string -} - -func (h Handle) String() string { - name := h.Name - if len(name) > 10 { - name = name[:10] - } - return fmt.Sprintf("<%s/%s>", h.Type, name) -} - // Backend is used to store and access data. type Backend interface { // Location returns a string that describes the type and location of the diff --git a/backend/local/backend_test.go b/backend/local/backend_test.go new file mode 100644 index 000000000..8dea61892 --- /dev/null +++ b/backend/local/backend_test.go @@ -0,0 +1,20 @@ +// DO NOT EDIT! +// generated at 2016-23-01 17:06:38 +0100 CET +package local_test + +import ( + "testing" + + "github.com/restic/restic/backend/test" +) + +func TestCreate(t *testing.T) { test.Create(t) } +func TestOpen(t *testing.T) { test.Open(t) } +func TestLocation(t *testing.T) { test.Location(t) } +func TestConfig(t *testing.T) { test.Config(t) } +func TestGetReader(t *testing.T) { test.GetReader(t) } +func TestLoad(t *testing.T) { test.Load(t) } +func TestWrite(t *testing.T) { test.Write(t) } +func TestGeneric(t *testing.T) { test.Generic(t) } +func TestDelete(t *testing.T) { test.Delete(t) } +func TestCleanup(t *testing.T) { test.Cleanup(t) } diff --git a/backend/local/local.go b/backend/local/local.go index 9946ddc4a..25ba26779 100644 --- a/backend/local/local.go +++ b/backend/local/local.go @@ -226,6 +226,10 @@ func (b *Local) GetReader(t backend.Type, name string, offset, length uint) (io. // Load returns the data stored in the backend for h at the given offset // and saves it in p. Load has the same semantics as io.ReaderAt. func (b *Local) Load(h backend.Handle, p []byte, off int64) (n int, err error) { + if err := h.Valid(); err != nil { + return 0, err + } + f, err := os.Open(filename(b.p, h.Type, h.Name)) if err != nil { return 0, err diff --git a/backend/local/local_test.go b/backend/local/local_test.go new file mode 100644 index 000000000..d574b015c --- /dev/null +++ b/backend/local/local_test.go @@ -0,0 +1,53 @@ +package local_test + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + + "github.com/restic/restic/backend" + "github.com/restic/restic/backend/local" + "github.com/restic/restic/backend/test" +) + +var tempBackendDir string + +//go:generate go run ../test/generate_backend_tests.go + +func init() { + test.CreateFn = func() (backend.Backend, error) { + if tempBackendDir != "" { + return nil, errors.New("temporary local backend dir already exists") + } + + tempdir, err := ioutil.TempDir("", "restic-local-test-") + if err != nil { + return nil, err + } + + fmt.Printf("created new test backend at %v\n", tempdir) + tempBackendDir = tempdir + + return local.Create(tempdir) + } + + test.OpenFn = func() (backend.Backend, error) { + if tempBackendDir == "" { + return nil, errors.New("repository not initialized") + } + + return local.Open(tempBackendDir) + } + + test.CleanupFn = func() error { + if tempBackendDir == "" { + return nil + } + + fmt.Printf("removing test backend at %v\n", tempBackendDir) + err := os.RemoveAll(tempBackendDir) + tempBackendDir = "" + return err + } +} diff --git a/backend/test/generate_backend_tests.go b/backend/test/generate_backend_tests.go new file mode 100644 index 000000000..64c89159d --- /dev/null +++ b/backend/test/generate_backend_tests.go @@ -0,0 +1,111 @@ +// +build ignore + +package main + +import ( + "bufio" + "flag" + "fmt" + "io" + "log" + "os" + "os/exec" + "path/filepath" + "regexp" + "text/template" + "time" +) + +var data struct { + Package string + Funcs []string + Timestamp string +} + +var testTemplate = ` +// DO NOT EDIT! +// generated at {{ .Timestamp }} +package {{ .Package }} + +import ( + "testing" + + "github.com/restic/restic/backend/test" +) + +{{ range $f := .Funcs }}func Test{{ $f }}(t *testing.T){ test.{{ $f }}(t) } +{{ end }} +` + +var testFile = flag.String("testfile", "../test/tests.go", "file to search test functions in") +var outputFile = flag.String("output", "backend_test.go", "output file to write generated code to") + +func errx(err error) { + if err == nil { + return + } + + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) +} + +var funcRegex = regexp.MustCompile(`^func\s+([A-Z].*)\s*\(`) + +func findTestFunctions() (funcs []string) { + f, err := os.Open(*testFile) + errx(err) + + sc := bufio.NewScanner(f) + for sc.Scan() { + match := funcRegex.FindStringSubmatch(sc.Text()) + if len(match) > 0 { + funcs = append(funcs, match[1]) + } + } + + if err := sc.Err(); err != nil { + log.Fatalf("Error scanning file: %v", err) + } + + errx(f.Close()) + return funcs +} + +func generateOutput(wr io.Writer, data interface{}) { + t := template.Must(template.New("backendtest").Parse(testTemplate)) + + cmd := exec.Command("gofmt") + cmd.Stdout = wr + in, err := cmd.StdinPipe() + errx(err) + errx(cmd.Start()) + errx(t.Execute(in, data)) + errx(in.Close()) + errx(cmd.Wait()) +} + +func init() { + flag.Parse() +} + +func main() { + dir, err := os.Getwd() + if err != nil { + fmt.Fprintf(os.Stderr, "Getwd() %v\n", err) + os.Exit(1) + } + + packageName := filepath.Base(dir) + + f, err := os.Create(*outputFile) + errx(err) + + data.Package = packageName + "_test" + data.Funcs = findTestFunctions() + data.Timestamp = time.Now().Format("2006-02-01 15:04:05 -0700 MST") + generateOutput(f, data) + + errx(f.Close()) + + fmt.Printf("wrote backend tests for package %v\n", packageName) +} diff --git a/backend/test/tests.go b/backend/test/tests.go new file mode 100644 index 000000000..65a3e723a --- /dev/null +++ b/backend/test/tests.go @@ -0,0 +1,550 @@ +package test + +import ( + "bytes" + crand "crypto/rand" + "fmt" + "io" + "io/ioutil" + "math/rand" + "reflect" + "sort" + "testing" + + "github.com/restic/restic/backend" + . "github.com/restic/restic/test" +) + +// CreateFn is a function that creates a temporary repository for the tests. +var CreateFn func() (backend.Backend, error) + +// OpenFn is a function that opens a previously created temporary repository. +var OpenFn func() (backend.Backend, error) + +// CleanupFn removes temporary files and directories created during the tests. +var CleanupFn func() error + +var but backend.Backend // backendUnderTest +var butInitialized bool + +func open(t testing.TB) backend.Backend { + if OpenFn == nil { + t.Fatal("OpenFn not set") + } + + if CreateFn == nil { + t.Fatalf("CreateFn not set") + } + + if !butInitialized { + be, err := CreateFn() + if err != nil { + t.Fatalf("Create returned unexpected error: %v", err) + } + + but = be + butInitialized = true + } + + if but == nil { + var err error + but, err = OpenFn() + if err != nil { + t.Fatalf("Open returned unexpected error: %v", err) + } + } + + return but +} + +func close(t testing.TB) { + if but == nil { + t.Fatalf("trying to close non-existing backend") + } + + err := but.Close() + if err != nil { + t.Fatalf("Close returned unexpected error: %v", err) + } + + but = nil +} + +// Create creates a backend. +func Create(t testing.TB) { + if CreateFn == nil { + t.Fatalf("CreateFn not set!") + } + + be, err := CreateFn() + if err != nil { + fmt.Printf("foo\n") + t.Fatalf("Create returned error: %v", err) + } + + butInitialized = true + + err = be.Close() + if err != nil { + t.Fatalf("Close returned error: %v", err) + } +} + +// Open opens a previously created backend. +func Open(t testing.TB) { + if OpenFn == nil { + t.Fatalf("OpenFn not set!") + } + + be, err := OpenFn() + if err != nil { + t.Fatalf("Open returned error: %v", err) + } + + err = be.Close() + if err != nil { + t.Fatalf("Close returned error: %v", err) + } +} + +// Location tests that a location string is returned. +func Location(t testing.TB) { + b := open(t) + defer close(t) + + l := b.Location() + if l == "" { + t.Fatalf("invalid location string %q", l) + } +} + +// Config saves and loads a config from the backend. +func Config(t testing.TB) { + b := open(t) + defer close(t) + + var testString = "Config" + + // create config and read it back + _, err := b.GetReader(backend.Config, "", 0, 0) + if err == nil { + t.Fatalf("did not get expected error for non-existing config") + } + + blob, err := b.Create() + if err != nil { + t.Fatalf("Create() error: %v", err) + } + + _, err = blob.Write([]byte(testString)) + if err != nil { + t.Fatalf("Write() error: %v", err) + } + + err = blob.Finalize(backend.Config, "") + if err != nil { + t.Fatalf("Finalize() error: %v", err) + } + + // try accessing the config with different names, should all return the + // same config + for _, name := range []string{"", "foo", "bar", "0000000000000000000000000000000000000000000000000000000000000000"} { + rd, err := b.GetReader(backend.Config, name, 0, 0) + if err != nil { + t.Fatalf("unable to read config with name %q: %v", name, err) + } + + buf, err := ioutil.ReadAll(rd) + if err != nil { + t.Fatalf("read config error: %v", err) + } + + err = rd.Close() + if err != nil { + t.Fatalf("close error: %v", err) + } + + if string(buf) != testString { + t.Fatalf("wrong data returned, want %q, got %q", testString, string(buf)) + } + } +} + +// GetReader tests various ways the GetReader function can be called. +func GetReader(t testing.TB) { + b := open(t) + defer close(t) + + length := rand.Intn(1<<24) + 2000 + + data := make([]byte, length) + _, err := io.ReadFull(crand.Reader, data) + OK(t, err) + + blob, err := b.Create() + OK(t, err) + + id := backend.Hash(data) + + _, err = blob.Write([]byte(data)) + OK(t, err) + OK(t, blob.Finalize(backend.Data, id.String())) + + for i := 0; i < 500; i++ { + l := rand.Intn(length + 2000) + o := rand.Intn(length + 2000) + + d := data + if o < len(d) { + d = d[o:] + } else { + o = len(d) + d = d[:0] + } + + if l > 0 && l < len(d) { + d = d[:l] + } + + rd, err := b.GetReader(backend.Data, id.String(), uint(o), uint(l)) + OK(t, err) + buf, err := ioutil.ReadAll(rd) + OK(t, err) + + if !bytes.Equal(buf, d) { + t.Fatalf("data not equal") + } + } + + OK(t, b.Remove(backend.Data, id.String())) +} + +// Load tests the backend's Load function. +func Load(t testing.TB) { + b := open(t) + defer close(t) + + _, err := b.Load(backend.Handle{}, nil, 0) + if err == nil { + t.Fatalf("Load() did not return an error for invalid handle") + } + + _, err = b.Load(backend.Handle{Type: backend.Data, Name: "foobar"}, nil, 0) + if err == nil { + t.Fatalf("Load() did not return an error for non-existing blob") + } + + length := rand.Intn(1<<24) + 2000 + + data := make([]byte, length) + _, err = io.ReadFull(crand.Reader, data) + if err != nil { + t.Fatalf("reading random data failed: %v", err) + } + + id := backend.Hash(data) + + blob, err := b.Create() + OK(t, err) + + _, err = blob.Write([]byte(data)) + OK(t, err) + OK(t, blob.Finalize(backend.Data, id.String())) + + for i := 0; i < 500; i++ { + l := rand.Intn(length + 2000) + o := rand.Intn(length + 2000) + + d := data + if o < len(d) { + d = d[o:] + } else { + o = len(d) + d = d[:0] + } + + if l > 0 && l < len(d) { + d = d[:l] + } + + buf := make([]byte, l) + h := backend.Handle{Type: backend.Data, Name: id.String()} + n, err := b.Load(h, buf, int64(o)) + + // if we requested data beyond the end of the file, ignore + // ErrUnexpectedEOF error + if l > len(d) && err == io.ErrUnexpectedEOF { + err = nil + buf = buf[:len(d)] + } + + if err != nil { + t.Errorf("Load(%d, %d): unexpected error: %v", len(buf), int64(o), err) + continue + } + + if n != len(buf) { + t.Errorf("Load(%d, %d): wrong length returned, want %d, got %d", + len(buf), int64(o), len(buf), n) + continue + } + + buf = buf[:n] + if !bytes.Equal(buf, d) { + t.Errorf("Load(%d, %d) returned wrong bytes", len(buf), int64(o)) + continue + } + } + + OK(t, b.Remove(backend.Data, id.String())) +} + +// Write tests writing data to the backend. +func Write(t testing.TB) { + b := open(t) + defer close(t) + + length := rand.Intn(1<<23) + 2000 + + data := make([]byte, length) + _, err := io.ReadFull(crand.Reader, data) + OK(t, err) + id := backend.Hash(data) + + for i := 0; i < 10; i++ { + blob, err := b.Create() + OK(t, err) + + o := 0 + for o < len(data) { + l := rand.Intn(len(data) - o) + if len(data)-o < 20 { + l = len(data) - o + } + + n, err := blob.Write(data[o : o+l]) + OK(t, err) + if n != l { + t.Fatalf("wrong number of bytes written, want %v, got %v", l, n) + } + + o += l + } + + name := fmt.Sprintf("%s-%d", id, i) + OK(t, blob.Finalize(backend.Data, name)) + + rd, err := b.GetReader(backend.Data, name, 0, 0) + OK(t, err) + + buf, err := ioutil.ReadAll(rd) + OK(t, err) + + if len(buf) != len(data) { + t.Fatalf("number of bytes does not match, want %v, got %v", len(data), len(buf)) + } + + if !bytes.Equal(buf, data) { + t.Fatalf("data not equal") + } + + err = b.Remove(backend.Data, name) + if err != nil { + t.Fatalf("error removing item: %v", err) + } + } +} + +var testStrings = []struct { + id string + data string +}{ + {"c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2", "foobar"}, + {"248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, + {"cc5d46bdb4991c6eae3eb739c9c8a7a46fe9654fab79c47b4fe48383b5b25e1c", "foo/bar"}, + {"4e54d2c721cbdb730f01b10b62dec622962b36966ec685880effa63d71c808f2", "foo/../../baz"}, +} + +func store(t testing.TB, b backend.Backend, tpe backend.Type, data []byte) { + id := backend.Hash(data) + + blob, err := b.Create() + OK(t, err) + + _, err = blob.Write([]byte(data)) + OK(t, err) + OK(t, blob.Finalize(tpe, id.String())) +} + +func read(t testing.TB, rd io.Reader, expectedData []byte) { + buf, err := ioutil.ReadAll(rd) + OK(t, err) + if expectedData != nil { + Equals(t, expectedData, buf) + } +} + +// Generic tests all functions of the backend. +func Generic(t testing.TB) { + b := open(t) + defer close(t) + + for _, tpe := range []backend.Type{ + backend.Data, backend.Key, backend.Lock, + backend.Snapshot, backend.Index, + } { + // detect non-existing files + for _, test := range testStrings { + id, err := backend.ParseID(test.id) + OK(t, err) + + // test if blob is already in repository + ret, err := b.Test(tpe, id.String()) + OK(t, err) + Assert(t, !ret, "blob was found to exist before creating") + + // try to open not existing blob + _, err = b.GetReader(tpe, id.String(), 0, 0) + Assert(t, err != nil, "blob data could be extracted before creation") + + // try to read not existing blob + _, err = b.GetReader(tpe, id.String(), 0, 1) + Assert(t, err != nil, "blob reader could be obtained before creation") + + // try to get string out, should fail + ret, err = b.Test(tpe, id.String()) + OK(t, err) + Assert(t, !ret, "id %q was found (but should not have)", test.id) + } + + // add files + for _, test := range testStrings { + store(t, b, tpe, []byte(test.data)) + + // test GetReader() + rd, err := b.GetReader(tpe, test.id, 0, uint(len(test.data))) + OK(t, err) + Assert(t, rd != nil, "GetReader() returned nil") + + read(t, rd, []byte(test.data)) + OK(t, rd.Close()) + + // try to read it out with an offset and a length + start := 1 + end := len(test.data) - 2 + length := end - start + rd, err = b.GetReader(tpe, test.id, uint(start), uint(length)) + OK(t, err) + Assert(t, rd != nil, "GetReader() returned nil") + + read(t, rd, []byte(test.data[start:end])) + OK(t, rd.Close()) + } + + // test adding the first file again + test := testStrings[0] + + // create blob + blob, err := b.Create() + OK(t, err) + + _, err = blob.Write([]byte(test.data)) + OK(t, err) + err = blob.Finalize(tpe, test.id) + Assert(t, err != nil, "expected error, got %v", err) + + // remove and recreate + err = b.Remove(tpe, test.id) + OK(t, err) + + // test that the blob is gone + ok, err := b.Test(tpe, test.id) + OK(t, err) + Assert(t, ok == false, "removed blob still present") + + // create blob + blob, err = b.Create() + OK(t, err) + + _, err = io.Copy(blob, bytes.NewReader([]byte(test.data))) + OK(t, err) + OK(t, blob.Finalize(tpe, test.id)) + + // list items + IDs := backend.IDs{} + + for _, test := range testStrings { + id, err := backend.ParseID(test.id) + OK(t, err) + IDs = append(IDs, id) + } + + list := backend.IDs{} + + for s := range b.List(tpe, nil) { + list = append(list, ParseID(s)) + } + + if len(IDs) != len(list) { + t.Fatalf("wrong number of IDs returned: want %d, got %d", len(IDs), len(list)) + } + + sort.Sort(IDs) + sort.Sort(list) + + if !reflect.DeepEqual(IDs, list) { + t.Fatalf("lists aren't equal, want:\n %v\n got:\n%v\n", IDs, list) + } + + // remove content if requested + if TestCleanup { + for _, test := range testStrings { + id, err := backend.ParseID(test.id) + OK(t, err) + + found, err := b.Test(tpe, id.String()) + OK(t, err) + + OK(t, b.Remove(tpe, id.String())) + + found, err = b.Test(tpe, id.String()) + OK(t, err) + Assert(t, !found, fmt.Sprintf("id %q not found after removal", id)) + } + } + } +} + +// Delete tests the Delete function. +func Delete(t testing.TB) { + b := open(t) + defer close(t) + + be, ok := b.(backend.Deleter) + if !ok { + return + } + + err := be.Delete() + if err != nil { + t.Fatalf("error deleting backend: %v", err) + } +} + +// Cleanup runs the cleanup function after all tests are run. +func Cleanup(t testing.TB) { + if CleanupFn == nil { + t.Log("CleanupFn function not set") + return + } + + if !TestCleanup { + t.Logf("not cleaning up backend") + return + } + + err := CleanupFn() + if err != nil { + t.Fatalf("Cleanup returned error: %v", err) + } +} diff --git a/backend/test/tests_test.go b/backend/test/tests_test.go new file mode 100644 index 000000000..fbdba9534 --- /dev/null +++ b/backend/test/tests_test.go @@ -0,0 +1,3 @@ +package test + +// func Test