diff --git a/run_integration_tests.go b/run_integration_tests.go index 2f2c71284..709b39a2f 100644 --- a/run_integration_tests.go +++ b/run_integration_tests.go @@ -45,9 +45,8 @@ type CIEnvironment interface { // TravisEnvironment is the environment in which Travis tests run. type TravisEnvironment struct { - goxArch []string - goxOS []string - minio string + goxOSArch []string + minio string minioSrv *Background minioTempdir string @@ -175,24 +174,27 @@ func (env *TravisEnvironment) Prepare() error { return err } if runtime.GOOS == "linux" { - env.goxArch = []string{"386", "amd64"} - if !strings.HasPrefix(runtime.Version(), "go1.3") { - env.goxArch = append(env.goxArch, "arm") + env.goxOSArch = []string{ + "linux/386", "linux/amd64", + "windows/386", "windows/amd64", + "darwin/386", "darwin/amd64", + "freebsd/386", "freebsd/amd64", + "opendbsd/386", "opendbsd/amd64", + } + if !strings.HasPrefix(runtime.Version(), "go1.3") { + env.goxOSArch = append(env.goxOSArch, + "linux/arm", "freebsd/arm") } - - env.goxOS = []string{"linux", "darwin", "freebsd", "openbsd", "windows"} } else { - env.goxArch = []string{runtime.GOARCH} - env.goxOS = []string{runtime.GOOS} + env.goxOSArch = []string{runtime.GOOS + "/" + runtime.GOARCH} } - msg("gox: OS %v, ARCH %v\n", env.goxOS, env.goxArch) + msg("gox: OS/ARCH %v\n", env.goxOSArch) v := runtime.Version() if !strings.HasPrefix(v, "go1.5") && !strings.HasPrefix(v, "go1.6") { err := run("gox", "-build-toolchain", - "-os", strings.Join(env.goxOS, " "), - "-arch", strings.Join(env.goxArch, " ")) + "-osarch", strings.Join(env.goxOSArch, " ")) if err != nil { return err @@ -320,8 +322,7 @@ func (env *TravisEnvironment) RunTests() error { // compile for all target architectures with tags for _, tags := range []string{"release", "debug"} { runWithEnv(env.env, "gox", "-verbose", - "-os", strings.Join(env.goxOS, " "), - "-arch", strings.Join(env.goxArch, " "), + "-osarch", strings.Join(env.goxOSArch, " "), "-tags", tags, "-output", "/tmp/{{.Dir}}_{{.OS}}_{{.Arch}}", "cmds/restic") diff --git a/src/cmds/restic/cmd_rebuild_index.go b/src/cmds/restic/cmd_rebuild_index.go index e3e82684a..8ac54d80b 100644 --- a/src/cmds/restic/cmd_rebuild_index.go +++ b/src/cmds/restic/cmd_rebuild_index.go @@ -33,17 +33,7 @@ func loadBlobsFromPacks(repo *repository.Repository) (packs map[backend.ID][]pac defer close(done) f := func(job worker.Job, done <-chan struct{}) (interface{}, error) { - id := job.Data.(backend.ID) - - h := backend.Handle{Type: backend.Data, Name: id.String()} - rd := backend.NewReadSeeker(repo.Backend(), h) - - unpacker, err := pack.NewUnpacker(repo.Key(), rd) - if err != nil { - return nil, err - } - - return unpacker.Entries, nil + return repo.ListPack(job.Data.(backend.ID)) } jobCh := make(chan worker.Job) diff --git a/src/restic/archiver.go b/src/restic/archiver.go index 9f9298f83..007bfa993 100644 --- a/src/restic/archiver.go +++ b/src/restic/archiver.go @@ -1,7 +1,6 @@ package restic import ( - "bytes" "encoding/json" "fmt" "io" @@ -92,7 +91,7 @@ func (arch *Archiver) isKnownBlob(id backend.ID) bool { } // Save stores a blob read from rd in the repository. -func (arch *Archiver) Save(t pack.BlobType, id backend.ID, length uint, rd io.Reader) error { +func (arch *Archiver) Save(t pack.BlobType, data []byte, id backend.ID) error { debug.Log("Archiver.Save", "Save(%v, %v)\n", t, id.Str()) if arch.isKnownBlob(id) { @@ -100,7 +99,7 @@ func (arch *Archiver) Save(t pack.BlobType, id backend.ID, length uint, rd io.Re return nil } - err := arch.repo.SaveFrom(t, &id, length, rd) + _, err := arch.repo.SaveAndEncrypt(t, data, &id) if err != nil { debug.Log("Archiver.Save", "Save(%v, %v): error %v\n", t, id.Str(), err) return err @@ -160,7 +159,7 @@ func (arch *Archiver) saveChunk(chunk chunker.Chunk, p *Progress, token struct{} defer freeBuf(chunk.Data) id := backend.Hash(chunk.Data) - err := arch.Save(pack.Data, id, chunk.Length, bytes.NewReader(chunk.Data)) + err := arch.Save(pack.Data, chunk.Data, id) // TODO handle error if err != nil { panic(err) diff --git a/src/restic/archiver_duplication_test.go b/src/restic/archiver_duplication_test.go index ffcbacee4..1c0193eab 100644 --- a/src/restic/archiver_duplication_test.go +++ b/src/restic/archiver_duplication_test.go @@ -1,7 +1,6 @@ package restic_test import ( - "bytes" "crypto/rand" "errors" "io" @@ -108,7 +107,7 @@ func testArchiverDuplication(t *testing.T) { buf := make([]byte, 50) - err := arch.Save(pack.Data, id, uint(len(buf)), bytes.NewReader(buf)) + err := arch.Save(pack.Data, buf, id) if err != nil { t.Fatal(err) } diff --git a/src/restic/archiver_test.go b/src/restic/archiver_test.go index 813cc3362..e42151a27 100644 --- a/src/restic/archiver_test.go +++ b/src/restic/archiver_test.go @@ -267,7 +267,7 @@ func testParallelSaveWithDuplication(t *testing.T, seed int) { id := backend.Hash(c.Data) time.Sleep(time.Duration(id[0])) - err := arch.Save(pack.Data, id, c.Length, bytes.NewReader(c.Data)) + err := arch.Save(pack.Data, c.Data, id) <-barrier errChan <- err }(c, errChan) diff --git a/src/restic/checker/checker.go b/src/restic/checker/checker.go index 4b147d442..4bd3303ba 100644 --- a/src/restic/checker/checker.go +++ b/src/restic/checker/checker.go @@ -534,6 +534,7 @@ func filterTrees(backlog backend.IDs, loaderChan chan<- backend.ID, in <-chan tr inCh = nil case outCh <- job: + debug.Log("checker.FilterTrees", "tree sent to check: %v", job.ID.Str()) outCh = nil inCh = in } @@ -581,6 +582,14 @@ func (c *Checker) checkTree(id backend.ID, tree *restic.Tree) (errs []error) { for _, node := range tree.Nodes { switch node.Type { case "file": + if node.Content == nil { + errs = append(errs, Error{TreeID: id, Err: fmt.Errorf("file %q has nil blob list", node.Name)}) + } + + if node.Mode == 0 { + errs = append(errs, Error{TreeID: id, Err: fmt.Errorf("file %q has invalid mode: %v", node.Name, node.Mode)}) + } + for b, blobID := range node.Content { if blobID.IsNull() { errs = append(errs, Error{TreeID: id, Err: fmt.Errorf("file %q blob %d has null ID", node.Name, b)}) @@ -598,6 +607,16 @@ func (c *Checker) checkTree(id backend.ID, tree *restic.Tree) (errs []error) { errs = append(errs, Error{TreeID: id, Err: fmt.Errorf("dir node %q subtree id is null", node.Name)}) continue } + + case "symlink": + // nothing to check + + default: + errs = append(errs, Error{TreeID: id, Err: fmt.Errorf("node %q with invalid type %q", node.Name, node.Type)}) + } + + if node.Name == "" { + errs = append(errs, Error{TreeID: id, Err: errors.New("node with empty name")}) } } diff --git a/src/restic/progress.go b/src/restic/progress.go index 7215a3206..cca1a4e47 100644 --- a/src/restic/progress.go +++ b/src/restic/progress.go @@ -19,7 +19,7 @@ type Progress struct { start time.Time c *time.Ticker cancel chan struct{} - o sync.Once + o *sync.Once d time.Duration lastUpdate time.Time @@ -52,7 +52,7 @@ func (p *Progress) Start() { return } - p.o = sync.Once{} + p.o = &sync.Once{} p.cancel = make(chan struct{}) p.running = true p.Reset() diff --git a/src/restic/repository/repository.go b/src/restic/repository/repository.go index e7e2e5ada..aa75d322e 100644 --- a/src/restic/repository/repository.go +++ b/src/restic/repository/repository.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "os" "restic/backend" @@ -212,26 +211,6 @@ func (r *Repository) SaveAndEncrypt(t pack.BlobType, data []byte, id *backend.ID return *id, r.savePacker(packer) } -// SaveFrom encrypts data read from rd and stores it in a pack in the backend as type t. -func (r *Repository) SaveFrom(t pack.BlobType, id *backend.ID, length uint, rd io.Reader) error { - debug.Log("Repo.SaveFrom", "save id %v (%v, %d bytes)", id.Str(), t, length) - if id == nil { - return errors.New("id is nil") - } - - buf, err := ioutil.ReadAll(rd) - if err != nil { - return err - } - - _, err = r.SaveAndEncrypt(t, buf, id) - if err != nil { - return err - } - - return nil -} - // SaveJSON serialises item as JSON and encrypts and saves it in a pack in the // backend as type t. func (r *Repository) SaveJSON(t pack.BlobType, item interface{}) (backend.ID, error) { @@ -539,6 +518,19 @@ func (r *Repository) List(t backend.Type, done <-chan struct{}) <-chan backend.I return outCh } +// ListPack returns the list of blobs saved in the pack id. +func (r *Repository) ListPack(id backend.ID) ([]pack.Blob, error) { + h := backend.Handle{Type: backend.Data, Name: id.String()} + rd := backend.NewReadSeeker(r.Backend(), h) + + unpacker, err := pack.NewUnpacker(r.Key(), rd) + if err != nil { + return nil, err + } + + return unpacker.Entries, nil +} + // Delete calls backend.Delete() if implemented, and returns an error // otherwise. func (r *Repository) Delete() error { diff --git a/src/restic/repository/repository_test.go b/src/restic/repository/repository_test.go index 5bd595086..3df824771 100644 --- a/src/restic/repository/repository_test.go +++ b/src/restic/repository/repository_test.go @@ -117,8 +117,9 @@ func TestSaveFrom(t *testing.T) { id := backend.Hash(data) // save - err = repo.SaveFrom(pack.Data, &id, uint(size), bytes.NewReader(data)) + id2, err := repo.SaveAndEncrypt(pack.Data, data, &id) OK(t, err) + Equals(t, id, id2) OK(t, repo.Flush()) @@ -136,7 +137,7 @@ func TestSaveFrom(t *testing.T) { } } -func BenchmarkSaveFrom(t *testing.B) { +func BenchmarkSaveAndEncrypt(t *testing.B) { repo := SetupRepo() defer TeardownRepo(repo) @@ -153,7 +154,7 @@ func BenchmarkSaveFrom(t *testing.B) { for i := 0; i < t.N; i++ { // save - err = repo.SaveFrom(pack.Data, &id, uint(size), bytes.NewReader(data)) + _, err = repo.SaveAndEncrypt(pack.Data, data, &id) OK(t, err) } } diff --git a/src/restic/repository/testing.go b/src/restic/repository/testing.go new file mode 100644 index 000000000..f45714d80 --- /dev/null +++ b/src/restic/repository/testing.go @@ -0,0 +1,63 @@ +package repository + +import ( + "os" + "restic/backend" + "restic/backend/local" + "restic/backend/mem" + "testing" +) + +// TestBackend returns a fully configured in-memory backend. +func TestBackend(t testing.TB) (be backend.Backend, cleanup func()) { + return mem.New(), func() {} +} + +// TestPassword is used for all repositories created by the Test* functions. +const TestPassword = "geheim" + +// TestRepositoryWithBackend returns a repository initialized with a test +// password. If be is nil, an in-memory backend is used. +func TestRepositoryWithBackend(t testing.TB, be backend.Backend) (r *Repository, cleanup func()) { + var beCleanup func() + if be == nil { + be, beCleanup = TestBackend(t) + } + + r = New(be) + + err := r.Init(TestPassword) + if err != nil { + t.Fatalf("TestRepopository(): initialize repo failed: %v", err) + } + + return r, func() { + if beCleanup != nil { + beCleanup() + } + } +} + +// TestRepository returns a repository initialized with a test password on an +// in-memory backend. When the environment variable RESTIC_TEST_REPO is set to +// a non-existing directory, a local backend is created there and this is used +// instead. The directory is not removed. +func TestRepository(t testing.TB) (r *Repository, cleanup func()) { + dir := os.Getenv("RESTIC_TEST_REPO") + if dir != "" { + _, err := os.Stat(dir) + if err != nil { + be, err := local.Create(dir) + if err != nil { + t.Fatalf("error creating local backend at %v: %v", dir, err) + } + return TestRepositoryWithBackend(t, be) + } + + if err == nil { + t.Logf("directory at %v already exists, using mem backend", dir) + } + } + + return TestRepositoryWithBackend(t, nil) +} diff --git a/src/restic/snapshot.go b/src/restic/snapshot.go index 6b4e87298..18aa9cad0 100644 --- a/src/restic/snapshot.go +++ b/src/restic/snapshot.go @@ -11,6 +11,7 @@ import ( "restic/repository" ) +// Snapshot is the state of a resource at one point in time. type Snapshot struct { Time time.Time `json:"time"` Parent *backend.ID `json:"parent,omitempty"` @@ -25,6 +26,8 @@ type Snapshot struct { id *backend.ID // plaintext ID, used during restore } +// NewSnapshot returns an initialized snapshot struct for the current user and +// time. func NewSnapshot(paths []string) (*Snapshot, error) { for i, path := range paths { if p, err := filepath.Abs(path); err != nil { @@ -50,6 +53,7 @@ func NewSnapshot(paths []string) (*Snapshot, error) { return sn, nil } +// LoadSnapshot loads the snapshot with the id and returns it. func LoadSnapshot(repo *repository.Repository, id backend.ID) (*Snapshot, error) { sn := &Snapshot{id: &id} err := repo.LoadJSONUnpacked(backend.Snapshot, id, sn) @@ -60,10 +64,28 @@ func LoadSnapshot(repo *repository.Repository, id backend.ID) (*Snapshot, error) return sn, nil } +// LoadAllSnapshots returns a list of all snapshots in the repo. +func LoadAllSnapshots(repo *repository.Repository) (snapshots []*Snapshot, err error) { + done := make(chan struct{}) + defer close(done) + + for id := range repo.List(backend.Snapshot, done) { + sn, err := LoadSnapshot(repo, id) + if err != nil { + return nil, err + } + + snapshots = append(snapshots, sn) + } + + return snapshots, nil +} + func (sn Snapshot) String() string { return fmt.Sprintf("", sn.Paths, sn.Time) } +// ID retuns the snapshot's ID. func (sn Snapshot) ID() *backend.ID { return sn.id } diff --git a/src/restic/testing.go b/src/restic/testing.go new file mode 100644 index 000000000..fa2ab649d --- /dev/null +++ b/src/restic/testing.go @@ -0,0 +1,182 @@ +package restic + +import ( + "fmt" + "io" + "math/rand" + "restic/backend" + "restic/pack" + "restic/repository" + "testing" + "time" + + "github.com/restic/chunker" +) + +type randReader struct { + rnd *rand.Rand + buf []byte +} + +func newRandReader(rnd *rand.Rand) io.Reader { + return &randReader{rnd: rnd, buf: make([]byte, 0, 7)} +} + +func (rd *randReader) read(p []byte) (n int, err error) { + if len(p)%7 != 0 { + panic("invalid buffer length, not multiple of 7") + } + + rnd := rd.rnd + for i := 0; i < len(p); i += 7 { + val := rnd.Int63() + + p[i+0] = byte(val >> 0) + p[i+1] = byte(val >> 8) + p[i+2] = byte(val >> 16) + p[i+3] = byte(val >> 24) + p[i+4] = byte(val >> 32) + p[i+5] = byte(val >> 40) + p[i+6] = byte(val >> 48) + } + + return len(p), nil +} + +func (rd *randReader) Read(p []byte) (int, error) { + // first, copy buffer to p + pos := copy(p, rd.buf) + copy(rd.buf, rd.buf[pos:]) + + // shorten buf and p accordingly + rd.buf = rd.buf[:len(rd.buf)-pos] + p = p[pos:] + + // if this is enough to fill p, return + if len(p) == 0 { + return pos, nil + } + + // load multiple of 7 byte + l := (len(p) / 7) * 7 + n, err := rd.read(p[:l]) + pos += n + if err != nil { + return pos, err + } + p = p[n:] + + // load 7 byte to temp buffer + rd.buf = rd.buf[:7] + n, err = rd.read(rd.buf) + if err != nil { + return pos, err + } + + // copy the remaining bytes from the buffer to p + n = copy(p, rd.buf) + pos += n + + // save the remaining bytes in rd.buf + n = copy(rd.buf, rd.buf[n:]) + rd.buf = rd.buf[:n] + + return pos, nil +} + +// fakeFile returns a reader which yields deterministic pseudo-random data. +func fakeFile(t testing.TB, seed, size int64) io.Reader { + return io.LimitReader(newRandReader(rand.New(rand.NewSource(seed))), size) +} + +// saveFile reads from rd and saves the blobs in the repository. The list of +// IDs is returned. +func saveFile(t testing.TB, repo *repository.Repository, rd io.Reader) (blobs backend.IDs) { + ch := chunker.New(rd, repo.Config.ChunkerPolynomial) + + for { + chunk, err := ch.Next(getBuf()) + if err == io.EOF { + break + } + + if err != nil { + t.Fatalf("unabel to save chunk in repo: %v", err) + } + + id, err := repo.SaveAndEncrypt(pack.Data, chunk.Data, nil) + if err != nil { + t.Fatalf("error saving chunk: %v", err) + } + blobs = append(blobs, id) + } + + return blobs +} + +const maxFileSize = 1500000 +const maxSeed = 100 + +// saveTree saves a tree of fake files in the repo and returns the ID. +func saveTree(t testing.TB, repo *repository.Repository, seed int64) backend.ID { + rnd := rand.NewSource(seed) + numNodes := int(rnd.Int63() % 64) + t.Logf("create %v nodes", numNodes) + + var tree Tree + for i := 0; i < numNodes; i++ { + seed := rnd.Int63() % maxSeed + size := rnd.Int63() % maxFileSize + + node := &Node{ + Name: fmt.Sprintf("file-%v", seed), + Type: "file", + Mode: 0644, + Size: uint64(size), + } + + node.Content = saveFile(t, repo, fakeFile(t, seed, size)) + tree.Nodes = append(tree.Nodes, node) + } + + id, err := repo.SaveJSON(pack.Tree, tree) + if err != nil { + t.Fatal(err) + } + + return id +} + +// TestCreateSnapshot creates a snapshot filled with fake data. The +// fake data is generated deterministically from the timestamp `at`, which is +// also used as the snapshot's timestamp. +func TestCreateSnapshot(t testing.TB, repo *repository.Repository, at time.Time) backend.ID { + fakedir := fmt.Sprintf("fakedir-at-%v", at.Format("2006-01-02 15:04:05")) + snapshot, err := NewSnapshot([]string{fakedir}) + if err != nil { + t.Fatal(err) + } + snapshot.Time = at + + treeID := saveTree(t, repo, at.UnixNano()) + snapshot.Tree = &treeID + + id, err := repo.SaveJSONUnpacked(backend.Snapshot, snapshot) + if err != nil { + t.Fatal(err) + } + + t.Logf("saved snapshot %v", id.Str()) + + err = repo.Flush() + if err != nil { + t.Fatal(err) + } + + err = repo.SaveIndex() + if err != nil { + t.Fatal(err) + } + + return id +} diff --git a/src/restic/testing_test.go b/src/restic/testing_test.go new file mode 100644 index 000000000..8243a01a8 --- /dev/null +++ b/src/restic/testing_test.go @@ -0,0 +1,71 @@ +package restic_test + +import ( + "restic" + "restic/checker" + "restic/repository" + "testing" + "time" +) + +var testSnapshotTime = time.Unix(1460289341, 207401672) + +const testCreateSnapshots = 3 + +func TestCreateSnapshot(t *testing.T) { + repo, cleanup := repository.TestRepository(t) + defer cleanup() + + for i := 0; i < testCreateSnapshots; i++ { + restic.TestCreateSnapshot(t, repo, testSnapshotTime.Add(time.Duration(i)*time.Second)) + } + + snapshots, err := restic.LoadAllSnapshots(repo) + if err != nil { + t.Fatal(err) + } + + if len(snapshots) != testCreateSnapshots { + t.Fatalf("got %d snapshots, expected %d", len(snapshots), 1) + } + + sn := snapshots[0] + if sn.Time.Before(testSnapshotTime) || sn.Time.After(testSnapshotTime.Add(testCreateSnapshots*time.Second)) { + t.Fatalf("timestamp %v is outside of the allowed time range", sn.Time) + } + + if sn.Tree == nil { + t.Fatalf("tree id is nil") + } + + if sn.Tree.IsNull() { + t.Fatalf("snapshot has zero tree ID") + } + + chkr := checker.New(repo) + + hints, errs := chkr.LoadIndex() + if len(errs) != 0 { + t.Fatalf("errors loading index: %v", errs) + } + + if len(hints) != 0 { + t.Fatalf("errors loading index: %v", hints) + } + + done := make(chan struct{}) + defer close(done) + errChan := make(chan error) + go chkr.Structure(errChan, done) + + for err := range errChan { + t.Error(err) + } + + errChan = make(chan error) + go chkr.ReadData(nil, errChan, done) + + for err := range errChan { + t.Error(err) + } +}