Merge pull request #511 from restic/cleanups

Cleanups and test functions

run_integration_tests.go

@@ -45,9 +45,8 @@ type CIEnvironment interface {
 
 // TravisEnvironment is the environment in which Travis tests run.
 type TravisEnvironment struct {
-	goxArch []string
-	goxOS   []string
-	minio   string
+	goxOSArch []string
+	minio     string
 
 	minioSrv     *Background
 	minioTempdir string
@@ -175,24 +174,27 @@ func (env *TravisEnvironment) Prepare() error {
 			return err
 		}
 		if runtime.GOOS == "linux" {
-			env.goxArch = []string{"386", "amd64"}
-			if !strings.HasPrefix(runtime.Version(), "go1.3") {
-				env.goxArch = append(env.goxArch, "arm")
+			env.goxOSArch = []string{
+				"linux/386", "linux/amd64",
+				"windows/386", "windows/amd64",
+				"darwin/386", "darwin/amd64",
+				"freebsd/386", "freebsd/amd64",
+				"opendbsd/386", "opendbsd/amd64",
+			}
+			if !strings.HasPrefix(runtime.Version(), "go1.3") {
+				env.goxOSArch = append(env.goxOSArch,
+					"linux/arm", "freebsd/arm")
 			}
-
-			env.goxOS = []string{"linux", "darwin", "freebsd", "openbsd", "windows"}
 		} else {
-			env.goxArch = []string{runtime.GOARCH}
-			env.goxOS = []string{runtime.GOOS}
+			env.goxOSArch = []string{runtime.GOOS + "/" + runtime.GOARCH}
 		}
 
-		msg("gox: OS %v, ARCH %v\n", env.goxOS, env.goxArch)
+		msg("gox: OS/ARCH %v\n", env.goxOSArch)
 
 		v := runtime.Version()
 		if !strings.HasPrefix(v, "go1.5") && !strings.HasPrefix(v, "go1.6") {
 			err := run("gox", "-build-toolchain",
-				"-os", strings.Join(env.goxOS, " "),
-				"-arch", strings.Join(env.goxArch, " "))
+				"-osarch", strings.Join(env.goxOSArch, " "))
 
 			if err != nil {
 				return err
@@ -320,8 +322,7 @@ func (env *TravisEnvironment) RunTests() error {
 		// compile for all target architectures with tags
 		for _, tags := range []string{"release", "debug"} {
 			runWithEnv(env.env, "gox", "-verbose",
-				"-os", strings.Join(env.goxOS, " "),
-				"-arch", strings.Join(env.goxArch, " "),
+				"-osarch", strings.Join(env.goxOSArch, " "),
 				"-tags", tags,
 				"-output", "/tmp/{{.Dir}}_{{.OS}}_{{.Arch}}",
 				"cmds/restic")

src/cmds/restic/cmd_rebuild_index.go

@@ -33,17 +33,7 @@ func loadBlobsFromPacks(repo *repository.Repository) (packs map[backend.ID][]pac
 	defer close(done)
 
 	f := func(job worker.Job, done <-chan struct{}) (interface{}, error) {
-		id := job.Data.(backend.ID)
-
-		h := backend.Handle{Type: backend.Data, Name: id.String()}
-		rd := backend.NewReadSeeker(repo.Backend(), h)
-
-		unpacker, err := pack.NewUnpacker(repo.Key(), rd)
-		if err != nil {
-			return nil, err
-		}
-
-		return unpacker.Entries, nil
+		return repo.ListPack(job.Data.(backend.ID))
 	}
 
 	jobCh := make(chan worker.Job)

src/restic/archiver.go

@@ -1,7 +1,6 @@
 package restic
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -92,7 +91,7 @@ func (arch *Archiver) isKnownBlob(id backend.ID) bool {
 }
 
 // Save stores a blob read from rd in the repository.
-func (arch *Archiver) Save(t pack.BlobType, id backend.ID, length uint, rd io.Reader) error {
+func (arch *Archiver) Save(t pack.BlobType, data []byte, id backend.ID) error {
 	debug.Log("Archiver.Save", "Save(%v, %v)\n", t, id.Str())
 
 	if arch.isKnownBlob(id) {
@@ -100,7 +99,7 @@ func (arch *Archiver) Save(t pack.BlobType, id backend.ID, length uint, rd io.Re
 		return nil
 	}
 
-	err := arch.repo.SaveFrom(t, &id, length, rd)
+	_, err := arch.repo.SaveAndEncrypt(t, data, &id)
 	if err != nil {
 		debug.Log("Archiver.Save", "Save(%v, %v): error %v\n", t, id.Str(), err)
 		return err
@@ -160,7 +159,7 @@ func (arch *Archiver) saveChunk(chunk chunker.Chunk, p *Progress, token struct{}
 	defer freeBuf(chunk.Data)
 
 	id := backend.Hash(chunk.Data)
-	err := arch.Save(pack.Data, id, chunk.Length, bytes.NewReader(chunk.Data))
+	err := arch.Save(pack.Data, chunk.Data, id)
 	// TODO handle error
 	if err != nil {
 		panic(err)

src/restic/archiver_duplication_test.go

@@ -1,7 +1,6 @@
 package restic_test
 
 import (
-	"bytes"
 	"crypto/rand"
 	"errors"
 	"io"
@@ -108,7 +107,7 @@ func testArchiverDuplication(t *testing.T) {
 
 				buf := make([]byte, 50)
 
-				err := arch.Save(pack.Data, id, uint(len(buf)), bytes.NewReader(buf))
+				err := arch.Save(pack.Data, buf, id)
 				if err != nil {
 					t.Fatal(err)
 				}

src/restic/archiver_test.go

@@ -267,7 +267,7 @@ func testParallelSaveWithDuplication(t *testing.T, seed int) {
 
 				id := backend.Hash(c.Data)
 				time.Sleep(time.Duration(id[0]))
-				err := arch.Save(pack.Data, id, c.Length, bytes.NewReader(c.Data))
+				err := arch.Save(pack.Data, c.Data, id)
 				<-barrier
 				errChan <- err
 			}(c, errChan)

src/restic/checker/checker.go

@@ -534,6 +534,7 @@ func filterTrees(backlog backend.IDs, loaderChan chan<- backend.ID, in <-chan tr
 			inCh = nil
 
 		case outCh <- job:
+			debug.Log("checker.FilterTrees", "tree sent to check: %v", job.ID.Str())
 			outCh = nil
 			inCh = in
 		}
@@ -581,6 +582,14 @@ func (c *Checker) checkTree(id backend.ID, tree *restic.Tree) (errs []error) {
 	for _, node := range tree.Nodes {
 		switch node.Type {
 		case "file":
+			if node.Content == nil {
+				errs = append(errs, Error{TreeID: id, Err: fmt.Errorf("file %q has nil blob list", node.Name)})
+			}
+
+			if node.Mode == 0 {
+				errs = append(errs, Error{TreeID: id, Err: fmt.Errorf("file %q has invalid mode: %v", node.Name, node.Mode)})
+			}
+
 			for b, blobID := range node.Content {
 				if blobID.IsNull() {
 					errs = append(errs, Error{TreeID: id, Err: fmt.Errorf("file %q blob %d has null ID", node.Name, b)})
@@ -598,6 +607,16 @@ func (c *Checker) checkTree(id backend.ID, tree *restic.Tree) (errs []error) {
 				errs = append(errs, Error{TreeID: id, Err: fmt.Errorf("dir node %q subtree id is null", node.Name)})
 				continue
 			}
+
+		case "symlink":
+			// nothing to check
+
+		default:
+			errs = append(errs, Error{TreeID: id, Err: fmt.Errorf("node %q with invalid type %q", node.Name, node.Type)})
+		}
+
+		if node.Name == "" {
+			errs = append(errs, Error{TreeID: id, Err: errors.New("node with empty name")})
 		}
 	}
 

src/restic/progress.go

@@ -19,7 +19,7 @@ type Progress struct {
 	start      time.Time
 	c          *time.Ticker
 	cancel     chan struct{}
-	o          sync.Once
+	o          *sync.Once
 	d          time.Duration
 	lastUpdate time.Time
 
@@ -52,7 +52,7 @@ func (p *Progress) Start() {
 		return
 	}
 
-	p.o = sync.Once{}
+	p.o = &sync.Once{}
 	p.cancel = make(chan struct{})
 	p.running = true
 	p.Reset()

src/restic/repository/repository.go

@@ -6,7 +6,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 
 	"restic/backend"
@@ -212,26 +211,6 @@ func (r *Repository) SaveAndEncrypt(t pack.BlobType, data []byte, id *backend.ID
 	return *id, r.savePacker(packer)
 }
 
-// SaveFrom encrypts data read from rd and stores it in a pack in the backend as type t.
-func (r *Repository) SaveFrom(t pack.BlobType, id *backend.ID, length uint, rd io.Reader) error {
-	debug.Log("Repo.SaveFrom", "save id %v (%v, %d bytes)", id.Str(), t, length)
-	if id == nil {
-		return errors.New("id is nil")
-	}
-
-	buf, err := ioutil.ReadAll(rd)
-	if err != nil {
-		return err
-	}
-
-	_, err = r.SaveAndEncrypt(t, buf, id)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
 // SaveJSON serialises item as JSON and encrypts and saves it in a pack in the
 // backend as type t.
 func (r *Repository) SaveJSON(t pack.BlobType, item interface{}) (backend.ID, error) {
@@ -539,6 +518,19 @@ func (r *Repository) List(t backend.Type, done <-chan struct{}) <-chan backend.I
 	return outCh
 }
 
+// ListPack returns the list of blobs saved in the pack id.
+func (r *Repository) ListPack(id backend.ID) ([]pack.Blob, error) {
+	h := backend.Handle{Type: backend.Data, Name: id.String()}
+	rd := backend.NewReadSeeker(r.Backend(), h)
+
+	unpacker, err := pack.NewUnpacker(r.Key(), rd)
+	if err != nil {
+		return nil, err
+	}
+
+	return unpacker.Entries, nil
+}
+
 // Delete calls backend.Delete() if implemented, and returns an error
 // otherwise.
 func (r *Repository) Delete() error {

src/restic/repository/repository_test.go

@@ -117,8 +117,9 @@ func TestSaveFrom(t *testing.T) {
 		id := backend.Hash(data)
 
 		// save
-		err = repo.SaveFrom(pack.Data, &id, uint(size), bytes.NewReader(data))
+		id2, err := repo.SaveAndEncrypt(pack.Data, data, &id)
 		OK(t, err)
+		Equals(t, id, id2)
 
 		OK(t, repo.Flush())
 
@@ -136,7 +137,7 @@ func TestSaveFrom(t *testing.T) {
 	}
 }
 
-func BenchmarkSaveFrom(t *testing.B) {
+func BenchmarkSaveAndEncrypt(t *testing.B) {
 	repo := SetupRepo()
 	defer TeardownRepo(repo)
 
@@ -153,7 +154,7 @@ func BenchmarkSaveFrom(t *testing.B) {
 
 	for i := 0; i < t.N; i++ {
 		// save
-		err = repo.SaveFrom(pack.Data, &id, uint(size), bytes.NewReader(data))
+		_, err = repo.SaveAndEncrypt(pack.Data, data, &id)
 		OK(t, err)
 	}
 }

src/restic/repository/testing.go

@@ -0,0 +1,63 @@
+package repository
+
+import (
+	"os"
+	"restic/backend"
+	"restic/backend/local"
+	"restic/backend/mem"
+	"testing"
+)
+
+// TestBackend returns a fully configured in-memory backend.
+func TestBackend(t testing.TB) (be backend.Backend, cleanup func()) {
+	return mem.New(), func() {}
+}
+
+// TestPassword is used for all repositories created by the Test* functions.
+const TestPassword = "geheim"
+
+// TestRepositoryWithBackend returns a repository initialized with a test
+// password. If be is nil, an in-memory backend is used.
+func TestRepositoryWithBackend(t testing.TB, be backend.Backend) (r *Repository, cleanup func()) {
+	var beCleanup func()
+	if be == nil {
+		be, beCleanup = TestBackend(t)
+	}
+
+	r = New(be)
+
+	err := r.Init(TestPassword)
+	if err != nil {
+		t.Fatalf("TestRepopository(): initialize repo failed: %v", err)
+	}
+
+	return r, func() {
+		if beCleanup != nil {
+			beCleanup()
+		}
+	}
+}
+
+// TestRepository returns a repository initialized with a test password on an
+// in-memory backend. When the environment variable RESTIC_TEST_REPO is set to
+// a non-existing directory, a local backend is created there and this is used
+// instead. The directory is not removed.
+func TestRepository(t testing.TB) (r *Repository, cleanup func()) {
+	dir := os.Getenv("RESTIC_TEST_REPO")
+	if dir != "" {
+		_, err := os.Stat(dir)
+		if err != nil {
+			be, err := local.Create(dir)
+			if err != nil {
+				t.Fatalf("error creating local backend at %v: %v", dir, err)
+			}
+			return TestRepositoryWithBackend(t, be)
+		}
+
+		if err == nil {
+			t.Logf("directory at %v already exists, using mem backend", dir)
+		}
+	}
+
+	return TestRepositoryWithBackend(t, nil)
+}

src/restic/snapshot.go

@@ -11,6 +11,7 @@ import (
 	"restic/repository"
 )
 
+// Snapshot is the state of a resource at one point in time.
 type Snapshot struct {
 	Time     time.Time   `json:"time"`
 	Parent   *backend.ID `json:"parent,omitempty"`
@@ -25,6 +26,8 @@ type Snapshot struct {
 	id *backend.ID // plaintext ID, used during restore
 }
 
+// NewSnapshot returns an initialized snapshot struct for the current user and
+// time.
 func NewSnapshot(paths []string) (*Snapshot, error) {
 	for i, path := range paths {
 		if p, err := filepath.Abs(path); err != nil {
@@ -50,6 +53,7 @@ func NewSnapshot(paths []string) (*Snapshot, error) {
 	return sn, nil
 }
 
+// LoadSnapshot loads the snapshot with the id and returns it.
 func LoadSnapshot(repo *repository.Repository, id backend.ID) (*Snapshot, error) {
 	sn := &Snapshot{id: &id}
 	err := repo.LoadJSONUnpacked(backend.Snapshot, id, sn)
@@ -60,10 +64,28 @@ func LoadSnapshot(repo *repository.Repository, id backend.ID) (*Snapshot, error)
 	return sn, nil
 }
 
+// LoadAllSnapshots returns a list of all snapshots in the repo.
+func LoadAllSnapshots(repo *repository.Repository) (snapshots []*Snapshot, err error) {
+	done := make(chan struct{})
+	defer close(done)
+
+	for id := range repo.List(backend.Snapshot, done) {
+		sn, err := LoadSnapshot(repo, id)
+		if err != nil {
+			return nil, err
+		}
+
+		snapshots = append(snapshots, sn)
+	}
+
+	return snapshots, nil
+}
+
 func (sn Snapshot) String() string {
 	return fmt.Sprintf("<Snapshot of %v at %s>", sn.Paths, sn.Time)
 }
 
+// ID retuns the snapshot's ID.
 func (sn Snapshot) ID() *backend.ID {
 	return sn.id
 }

src/restic/testing.go

@@ -0,0 +1,182 @@
+package restic
+
+import (
+	"fmt"
+	"io"
+	"math/rand"
+	"restic/backend"
+	"restic/pack"
+	"restic/repository"
+	"testing"
+	"time"
+
+	"github.com/restic/chunker"
+)
+
+type randReader struct {
+	rnd *rand.Rand
+	buf []byte
+}
+
+func newRandReader(rnd *rand.Rand) io.Reader {
+	return &randReader{rnd: rnd, buf: make([]byte, 0, 7)}
+}
+
+func (rd *randReader) read(p []byte) (n int, err error) {
+	if len(p)%7 != 0 {
+		panic("invalid buffer length, not multiple of 7")
+	}
+
+	rnd := rd.rnd
+	for i := 0; i < len(p); i += 7 {
+		val := rnd.Int63()
+
+		p[i+0] = byte(val >> 0)
+		p[i+1] = byte(val >> 8)
+		p[i+2] = byte(val >> 16)
+		p[i+3] = byte(val >> 24)
+		p[i+4] = byte(val >> 32)
+		p[i+5] = byte(val >> 40)
+		p[i+6] = byte(val >> 48)
+	}
+
+	return len(p), nil
+}
+
+func (rd *randReader) Read(p []byte) (int, error) {
+	// first, copy buffer to p
+	pos := copy(p, rd.buf)
+	copy(rd.buf, rd.buf[pos:])
+
+	// shorten buf and p accordingly
+	rd.buf = rd.buf[:len(rd.buf)-pos]
+	p = p[pos:]
+
+	// if this is enough to fill p, return
+	if len(p) == 0 {
+		return pos, nil
+	}
+
+	// load multiple of 7 byte
+	l := (len(p) / 7) * 7
+	n, err := rd.read(p[:l])
+	pos += n
+	if err != nil {
+		return pos, err
+	}
+	p = p[n:]
+
+	// load 7 byte to temp buffer
+	rd.buf = rd.buf[:7]
+	n, err = rd.read(rd.buf)
+	if err != nil {
+		return pos, err
+	}
+
+	// copy the remaining bytes from the buffer to p
+	n = copy(p, rd.buf)
+	pos += n
+
+	// save the remaining bytes in rd.buf
+	n = copy(rd.buf, rd.buf[n:])
+	rd.buf = rd.buf[:n]
+
+	return pos, nil
+}
+
+// fakeFile returns a reader which yields deterministic pseudo-random data.
+func fakeFile(t testing.TB, seed, size int64) io.Reader {
+	return io.LimitReader(newRandReader(rand.New(rand.NewSource(seed))), size)
+}
+
+// saveFile reads from rd and saves the blobs in the repository. The list of
+// IDs is returned.
+func saveFile(t testing.TB, repo *repository.Repository, rd io.Reader) (blobs backend.IDs) {
+	ch := chunker.New(rd, repo.Config.ChunkerPolynomial)
+
+	for {
+		chunk, err := ch.Next(getBuf())
+		if err == io.EOF {
+			break
+		}
+
+		if err != nil {
+			t.Fatalf("unabel to save chunk in repo: %v", err)
+		}
+
+		id, err := repo.SaveAndEncrypt(pack.Data, chunk.Data, nil)
+		if err != nil {
+			t.Fatalf("error saving chunk: %v", err)
+		}
+		blobs = append(blobs, id)
+	}
+
+	return blobs
+}
+
+const maxFileSize = 1500000
+const maxSeed = 100
+
+// saveTree saves a tree of fake files in the repo and returns the ID.
+func saveTree(t testing.TB, repo *repository.Repository, seed int64) backend.ID {
+	rnd := rand.NewSource(seed)
+	numNodes := int(rnd.Int63() % 64)
+	t.Logf("create %v nodes", numNodes)
+
+	var tree Tree
+	for i := 0; i < numNodes; i++ {
+		seed := rnd.Int63() % maxSeed
+		size := rnd.Int63() % maxFileSize
+
+		node := &Node{
+			Name: fmt.Sprintf("file-%v", seed),
+			Type: "file",
+			Mode: 0644,
+			Size: uint64(size),
+		}
+
+		node.Content = saveFile(t, repo, fakeFile(t, seed, size))
+		tree.Nodes = append(tree.Nodes, node)
+	}
+
+	id, err := repo.SaveJSON(pack.Tree, tree)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	return id
+}
+
+// TestCreateSnapshot creates a snapshot filled with fake data. The
+// fake data is generated deterministically from the timestamp `at`, which is
+// also used as the snapshot's timestamp.
+func TestCreateSnapshot(t testing.TB, repo *repository.Repository, at time.Time) backend.ID {
+	fakedir := fmt.Sprintf("fakedir-at-%v", at.Format("2006-01-02 15:04:05"))
+	snapshot, err := NewSnapshot([]string{fakedir})
+	if err != nil {
+		t.Fatal(err)
+	}
+	snapshot.Time = at
+
+	treeID := saveTree(t, repo, at.UnixNano())
+	snapshot.Tree = &treeID
+
+	id, err := repo.SaveJSONUnpacked(backend.Snapshot, snapshot)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Logf("saved snapshot %v", id.Str())
+
+	err = repo.Flush()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = repo.SaveIndex()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	return id
+}

src/restic/testing_test.go

@@ -0,0 +1,71 @@
+package restic_test
+
+import (
+	"restic"
+	"restic/checker"
+	"restic/repository"
+	"testing"
+	"time"
+)
+
+var testSnapshotTime = time.Unix(1460289341, 207401672)
+
+const testCreateSnapshots = 3
+
+func TestCreateSnapshot(t *testing.T) {
+	repo, cleanup := repository.TestRepository(t)
+	defer cleanup()
+
+	for i := 0; i < testCreateSnapshots; i++ {
+		restic.TestCreateSnapshot(t, repo, testSnapshotTime.Add(time.Duration(i)*time.Second))
+	}
+
+	snapshots, err := restic.LoadAllSnapshots(repo)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(snapshots) != testCreateSnapshots {
+		t.Fatalf("got %d snapshots, expected %d", len(snapshots), 1)
+	}
+
+	sn := snapshots[0]
+	if sn.Time.Before(testSnapshotTime) || sn.Time.After(testSnapshotTime.Add(testCreateSnapshots*time.Second)) {
+		t.Fatalf("timestamp %v is outside of the allowed time range", sn.Time)
+	}
+
+	if sn.Tree == nil {
+		t.Fatalf("tree id is nil")
+	}
+
+	if sn.Tree.IsNull() {
+		t.Fatalf("snapshot has zero tree ID")
+	}
+
+	chkr := checker.New(repo)
+
+	hints, errs := chkr.LoadIndex()
+	if len(errs) != 0 {
+		t.Fatalf("errors loading index: %v", errs)
+	}
+
+	if len(hints) != 0 {
+		t.Fatalf("errors loading index: %v", hints)
+	}
+
+	done := make(chan struct{})
+	defer close(done)
+	errChan := make(chan error)
+	go chkr.Structure(errChan, done)
+
+	for err := range errChan {
+		t.Error(err)
+	}
+
+	errChan = make(chan error)
+	go chkr.ReadData(nil, errChan, done)
+
+	for err := range errChan {
+		t.Error(err)
+	}
+}