Merge pull request #179 from restic/refactor-integration-tests

Add integration test with the go testing framework

.travis.yml

@@ -8,7 +8,7 @@ os:
   - linux
   - osx
 
-env: SFTP_PATH="/usr/lib/openssh/sftp-server" GOX_OS="linux darwin openbsd freebsd"
+env: GOX_OS="linux darwin openbsd freebsd"
 
 notifications:
   irc:
@@ -19,23 +19,19 @@ notifications:
     skip_join: true
 
 install:
+  - go version
+  - export GOBIN="$GOPATH/bin"
+  - export PATH="$PATH:$GOBIN"
+  - export GOPATH="$GOPATH:${TRAVIS_BUILD_DIR}/Godeps/_workspace"
+  - go env
   - go get github.com/mattn/goveralls
   - go get github.com/mitchellh/gox
   - gox -build-toolchain -os "$GOX_OS"
-  - go version
-  - go env
-  - make env
-  - make goenv
-  - make list
 
 script:
-  - make restic
-  - make gox
-  - make test
-  - GOARCH=386 make test
-  - make test-integration
-  - GOARCH=386 make test-integration
-  - make all.cov
-  - goveralls -coverprofile=all.cov -service=travis-ci -repotoken "$COVERALLS_TOKEN"
+  - gox -verbose -os "${GOX_OS}" -tags "release" ./cmd/restic
+  - go run run_tests.go all.cov
+  - GOARCH=386 RESTIC_TEST_INTEGRATION=0 go test ./...
+  - goveralls -coverprofile=all.cov -service=travis-ci -repotoken "$COVERALLS_TOKEN" || true
   - gofmt -l *.go */*.go */*/*.go
   - test -z "$(gofmt -l *.go */*.go */*/*.go)"

Makefile

@@ -26,7 +26,7 @@ all: restic
 
 %: cmd/% .gopath $(SOURCE)
 	cd $(BASEPATH) && \
-		go build -a -ldflags "-s" -o $@ ./$<
+		go build -a -tags release -ldflags "-s" -o $@ ./$<
 
 %.debug: cmd/% .gopath $(SOURCE)
 	cd $(BASEPATH) && \
@@ -48,17 +48,8 @@ gox: .gopath $(SOURCE)
 	cd $(BASEPATH) && \
 		gox -verbose -os "$(GOX_OS)" ./cmd/restic
 
-test-integration: .gopath restic restic.debug dirdiff
-	# run testsuite
-	PATH=.:$(PATH) ./testsuite.sh
-
-	# run sftp integration tests
-	cd $(BASEPATH)/backend && \
-		go test $(GOTESTFLAGS) -test.sftppath $(SFTP_PATH) ./...
-
 all.cov: .gopath $(SOURCE)
-	cd $(BASEPATH) && \
-		./coverage_all.sh all.cov
+	cd $(BASEPATH) && go run run_tests.go all.cov
 
 env:
 	@echo export GOPATH=\"$(GOPATH)\"

archiver.go

@@ -534,7 +534,7 @@ func (j archiveJob) Copy() pipe.Job {
 func (arch *Archiver) Snapshot(p *Progress, paths []string, parentID backend.ID) (*Snapshot, backend.ID, error) {
 	debug.Log("Archiver.Snapshot", "start for %v", paths)
 
-	debug.Break("Archiver.Snapshot")
+	debug.RunHook("Archiver.Snapshot", nil)
 	sort.Strings(paths)
 
 	// signal the whole pipeline to stop

backend/backend_test.go

@@ -102,7 +102,7 @@ func testBackend(b backend.Backend, t *testing.T) {
 		}
 
 		// remove content if requested
-		if *TestCleanup {
+		if TestCleanup {
 			for _, test := range TestStrings {
 				id, err := backend.ParseID(test.id)
 				OK(t, err)

backend/local_test.go

@@ -22,7 +22,7 @@ func setupLocalBackend(t *testing.T) *local.Local {
 }
 
 func teardownLocalBackend(t *testing.T, b *local.Local) {
-	if !*TestCleanup {
+	if !TestCleanup {
 		t.Logf("leaving local backend at %s\n", b.Location())
 		return
 	}

backend/sftp_test.go

@@ -1,22 +1,37 @@
 package backend_test
 
 import (
-	"flag"
 	"io/ioutil"
 	"os"
+	"path/filepath"
+	"strings"
 	"testing"
 
 	"github.com/restic/restic/backend/sftp"
 	. "github.com/restic/restic/test"
 )
 
-var sftpPath = flag.String("test.sftppath", "", "sftp binary path (default: empty)")
-
 func setupSFTPBackend(t *testing.T) *sftp.SFTP {
+	sftpserver := ""
+
+	for _, dir := range strings.Split(TestSFTPPath, ":") {
+		testpath := filepath.Join(dir, "sftp-server")
+		fd, err := os.Open(testpath)
+		fd.Close()
+		if !os.IsNotExist(err) {
+			sftpserver = testpath
+			break
+		}
+	}
+
+	if sftpserver == "" {
+		return nil
+	}
+
 	tempdir, err := ioutil.TempDir("", "restic-test-")
 	OK(t, err)
 
-	b, err := sftp.Create(tempdir, *sftpPath)
+	b, err := sftp.Create(tempdir, sftpserver)
 	OK(t, err)
 
 	t.Logf("created sftp backend locally at %s", tempdir)
@@ -25,7 +40,7 @@ func setupSFTPBackend(t *testing.T) *sftp.SFTP {
 }
 
 func teardownSFTPBackend(t *testing.T, b *sftp.SFTP) {
-	if !*TestCleanup {
+	if !TestCleanup {
 		t.Logf("leaving backend at %s\n", b.Location())
 		return
 	}
@@ -35,11 +50,15 @@ func teardownSFTPBackend(t *testing.T, b *sftp.SFTP) {
 }
 
 func TestSFTPBackend(t *testing.T) {
-	if *sftpPath == "" {
-		t.Skipf("sftppath not set, skipping TestSFTPBackend")
+	if !RunIntegrationTest {
+		t.Skip("integration tests disabled")
 	}
 
 	s := setupSFTPBackend(t)
+	if s == nil {
+		t.Skip("unable to find sftp-server binary")
+		return
+	}
 	defer teardownSFTPBackend(t, s)
 
 	testBackend(s, t)

cmd/dirdiff/main.go

@@ -1,142 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-	"syscall"
-)
-
-type entry struct {
-	path string
-	fi   os.FileInfo
-}
-
-func walk(dir string) <-chan *entry {
-	ch := make(chan *entry, 100)
-
-	go func() {
-		err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
-			if err != nil {
-				fmt.Fprintf(os.Stderr, "error: %v\n", err)
-				return nil
-			}
-
-			name, err := filepath.Rel(dir, path)
-			if err != nil {
-				fmt.Fprintf(os.Stderr, "error: %v\n", err)
-				return nil
-			}
-
-			ch <- &entry{
-				path: name,
-				fi:   info,
-			}
-
-			return nil
-		})
-
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "Walk() error: %v\n", err)
-		}
-
-		close(ch)
-	}()
-
-	// first element is root
-	_ = <-ch
-
-	return ch
-}
-
-func (e *entry) equals(other *entry) bool {
-	if e.path != other.path {
-		fmt.Printf("path does not match\n")
-		return false
-	}
-
-	if e.fi.Mode() != other.fi.Mode() {
-		fmt.Printf("mode does not match\n")
-		return false
-	}
-
-	if e.fi.ModTime() != other.fi.ModTime() {
-		fmt.Printf("%s: ModTime does not match\n", e.path)
-		// TODO: Fix ModTime for symlinks, return false
-		// see http://grokbase.com/t/gg/golang-nuts/154wnph4y8/go-nuts-no-way-to-utimes-a-symlink
-		return true
-	}
-
-	stat, _ := e.fi.Sys().(*syscall.Stat_t)
-	stat2, _ := other.fi.Sys().(*syscall.Stat_t)
-
-	if stat.Uid != stat2.Uid || stat2.Gid != stat2.Gid {
-		return false
-	}
-
-	return true
-}
-
-func main() {
-	if len(os.Args) != 3 {
-		fmt.Fprintf(os.Stderr, "USAGE: %s DIR1 DIR2\n", os.Args[0])
-		os.Exit(1)
-	}
-
-	ch1 := walk(os.Args[1])
-	ch2 := walk(os.Args[2])
-
-	changes := false
-
-	var a, b *entry
-	for {
-		var ok bool
-
-		if ch1 != nil && a == nil {
-			a, ok = <-ch1
-			if !ok {
-				ch1 = nil
-			}
-		}
-
-		if ch2 != nil && b == nil {
-			b, ok = <-ch2
-			if !ok {
-				ch2 = nil
-			}
-		}
-
-		if ch1 == nil && ch2 == nil {
-			break
-		}
-
-		if ch1 == nil {
-			fmt.Printf("+%v\n", b.path)
-			changes = true
-		} else if ch2 == nil {
-			fmt.Printf("-%v\n", a.path)
-			changes = true
-		} else if !a.equals(b) {
-			if a.path < b.path {
-				fmt.Printf("-%v\n", a.path)
-				changes = true
-				a = nil
-				continue
-			} else if a.path > b.path {
-				fmt.Printf("+%v\n", b.path)
-				changes = true
-				b = nil
-				continue
-			} else {
-				fmt.Printf("%%%v\n", a.path)
-				changes = true
-			}
-		}
-
-		a, b = nil, nil
-	}
-
-	if changes {
-		os.Exit(1)
-	}
-}

cmd/gentestdata/main.go

@@ -1,69 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"io"
-	"math/rand"
-	"os"
-	"path/filepath"
-)
-
-const (
-	MaxFiles = 23
-	MaxDepth = 3
-)
-
-var urnd *os.File
-
-func init() {
-	f, err := os.Open("/dev/urandom")
-	if err != nil {
-		panic(err)
-	}
-
-	urnd = f
-}
-
-func rndRd(bytes int) io.Reader {
-	return io.LimitReader(urnd, int64(bytes))
-}
-
-func createDir(target string, depth int) {
-	fmt.Printf("createDir %s, depth %d\n", target, depth)
-	err := os.Mkdir(target, 0755)
-	if err != nil && !os.IsExist(err) {
-		panic(err)
-	}
-
-	for i := 0; i < MaxFiles; i++ {
-		if depth == 0 {
-			filename := filepath.Join(target, fmt.Sprintf("file%d", i))
-			fmt.Printf("create file %v\n", filename)
-			f, err := os.Create(filename)
-			if err != nil {
-				panic(err)
-			}
-
-			_, err = io.Copy(f, rndRd(rand.Intn(1024)))
-			if err != nil {
-				panic(err)
-			}
-
-			err = f.Close()
-			if err != nil {
-				panic(err)
-			}
-		} else {
-			createDir(filepath.Join(target, fmt.Sprintf("dir%d", i)), depth-1)
-		}
-	}
-}
-
-func main() {
-	if len(os.Args) != 2 {
-		fmt.Fprintf(os.Stderr, "USAGE: %s TARGETDIR\n", os.Args[0])
-		os.Exit(1)
-	}
-
-	createDir(os.Args[1], MaxDepth)
-}

cmd/restic/cmd_backup.go

@@ -16,13 +16,15 @@ import (
 type CmdBackup struct {
 	Parent string `short:"p" long:"parent"    description:"use this parent snapshot (default: last snapshot in repo that has the same target)"`
 	Force  bool   `short:"f" long:"force" description:"Force re-reading the target. Overrides the \"parent\" flag"`
+
+	global *GlobalOptions
 }
 
 func init() {
 	_, err := parser.AddCommand("backup",
 		"save file/directory",
 		"The backup command creates a snapshot of a file or directory",
-		&CmdBackup{})
+		&CmdBackup{global: &globalOpts})
 	if err != nil {
 		panic(err)
 	}
@@ -97,8 +99,8 @@ func (cmd CmdBackup) Usage() string {
 	return "DIR/FILE [DIR/FILE] [...]"
 }
 
-func newScanProgress() *restic.Progress {
-	if disableProgress() {
+func (cmd CmdBackup) newScanProgress() *restic.Progress {
+	if !cmd.global.ShowProgress() {
 		return nil
 	}
 
@@ -113,8 +115,8 @@ func newScanProgress() *restic.Progress {
 	return p
 }
 
-func newArchiveProgress(todo restic.Stat) *restic.Progress {
-	if disableProgress() {
+func (cmd CmdBackup) newArchiveProgress(todo restic.Stat) *restic.Progress {
+	if !cmd.global.ShowProgress() {
 		return nil
 	}
 
@@ -213,7 +215,7 @@ func (cmd CmdBackup) Execute(args []string) error {
 		target = append(target, d)
 	}
 
-	s, err := OpenRepo()
+	s, err := cmd.global.OpenRepository()
 	if err != nil {
 		return err
 	}
@@ -232,7 +234,7 @@ func (cmd CmdBackup) Execute(args []string) error {
 			return fmt.Errorf("invalid id %q: %v", cmd.Parent, err)
 		}
 
-		verbosePrintf("found parent snapshot %v\n", parentSnapshotID.Str())
+		cmd.global.Verbosef("found parent snapshot %v\n", parentSnapshotID.Str())
 	}
 
 	// Find last snapshot to set it as parent, if not already set
@@ -243,13 +245,13 @@ func (cmd CmdBackup) Execute(args []string) error {
 		}
 
 		if parentSnapshotID != nil {
-			verbosePrintf("using parent snapshot %v\n", parentSnapshotID)
+			cmd.global.Verbosef("using parent snapshot %v\n", parentSnapshotID)
 		}
 	}
 
-	verbosePrintf("scan %v\n", target)
+	cmd.global.Verbosef("scan %v\n", target)
 
-	stat, err := restic.Scan(target, newScanProgress())
+	stat, err := restic.Scan(target, cmd.newScanProgress())
 
 	// TODO: add filter
 	// arch.Filter = func(dir string, fi os.FileInfo) bool {
@@ -260,16 +262,16 @@ func (cmd CmdBackup) Execute(args []string) error {
 
 	arch.Error = func(dir string, fi os.FileInfo, err error) error {
 		// TODO: make ignoring errors configurable
-		fmt.Fprintf(os.Stderr, "\x1b[2K\rerror for %s: %v\n", dir, err)
+		cmd.global.Warnf("\x1b[2K\rerror for %s: %v\n", dir, err)
 		return nil
 	}
 
-	_, id, err := arch.Snapshot(newArchiveProgress(stat), target, parentSnapshotID)
+	_, id, err := arch.Snapshot(cmd.newArchiveProgress(stat), target, parentSnapshotID)
 	if err != nil {
 		return err
 	}
 
-	verbosePrintf("snapshot %s saved\n", id.Str())
+	cmd.global.Verbosef("snapshot %s saved\n", id.Str())
 
 	return nil
 }

cmd/restic/cmd_cache.go

@@ -6,13 +6,15 @@ import (
 	"github.com/restic/restic"
 )
 
-type CmdCache struct{}
+type CmdCache struct {
+	global *GlobalOptions
+}
 
 func init() {
 	_, err := parser.AddCommand("cache",
 		"manage cache",
 		"The cache command creates and manages the local cache",
-		&CmdCache{})
+		&CmdCache{global: &globalOpts})
 	if err != nil {
 		panic(err)
 	}
@@ -27,12 +29,12 @@ func (cmd CmdCache) Execute(args []string) error {
 	// 	return fmt.Errorf("wrong number of parameters, Usage: %s", cmd.Usage())
 	// }
 
-	s, err := OpenRepo()
+	s, err := cmd.global.OpenRepository()
 	if err != nil {
 		return err
 	}
 
-	cache, err := restic.NewCache(s, opts.CacheDir)
+	cache, err := restic.NewCache(s, cmd.global.CacheDir)
 	if err != nil {
 		return err
 	}

cmd/restic/cmd_cat.go

@@ -14,13 +14,15 @@ import (
 	"github.com/restic/restic/repository"
 )
 
-type CmdCat struct{}
+type CmdCat struct {
+	global *GlobalOptions
+}
 
 func init() {
 	_, err := parser.AddCommand("cat",
 		"dump something",
 		"The cat command dumps data structures or data from a repository",
-		&CmdCat{})
+		&CmdCat{global: &globalOpts})
 	if err != nil {
 		panic(err)
 	}
@@ -35,7 +37,7 @@ func (cmd CmdCat) Execute(args []string) error {
 		return fmt.Errorf("type or ID not specified, Usage: %s", cmd.Usage())
 	}
 
-	s, err := OpenRepo()
+	s, err := cmd.global.OpenRepository()
 	if err != nil {
 		return err
 	}

cmd/restic/cmd_dump.go

@@ -15,13 +15,15 @@ import (
 	"github.com/restic/restic/repository"
 )
 
-type CmdDump struct{}
+type CmdDump struct {
+	global *MainOptions
+}
 
 func init() {
 	_, err := parser.AddCommand("dump",
 		"dump data structures",
 		"The dump command dumps data structures from a repository as JSON documents",
-		&CmdDump{})
+		&CmdDump{global: &mainOpts})
 	if err != nil {
 		panic(err)
 	}
@@ -102,7 +104,7 @@ func (cmd CmdDump) Execute(args []string) error {
 		return fmt.Errorf("type not specified, Usage: %s", cmd.Usage())
 	}
 
-	repo, err := OpenRepo()
+	repo, err := cmd.global.OpenRepository()
 	if err != nil {
 		return err
 	}

cmd/restic/cmd_find.go

@@ -23,6 +23,7 @@ type CmdFind struct {
 
 	oldest, newest time.Time
 	pattern        string
+	global         *GlobalOptions
 }
 
 var timeFormats = []string{
@@ -43,7 +44,7 @@ func init() {
 	_, err := parser.AddCommand("find",
 		"find a file/directory",
 		"The find command searches for files or directories in snapshots",
-		&CmdFind{})
+		&CmdFind{global: &globalOpts})
 	if err != nil {
 		panic(err)
 	}
@@ -156,7 +157,7 @@ func (c CmdFind) Execute(args []string) error {
 		}
 	}
 
-	s, err := OpenRepo()
+	s, err := c.global.OpenRepository()
 	if err != nil {
 		return err
 	}

cmd/restic/cmd_fsck.go

@@ -19,6 +19,8 @@ type CmdFsck struct {
 	Orphaned       bool   `short:"o" long:"orphaned"        description:"Check for orphaned blobs"`
 	RemoveOrphaned bool   `short:"r" long:"remove-orphaned" description:"Remove orphaned blobs (implies -o)"`
 
+	global *GlobalOptions
+
 	// lists checking for orphaned blobs
 	o_data  *backend.IDSet
 	o_trees *backend.IDSet
@@ -28,13 +30,13 @@ func init() {
 	_, err := parser.AddCommand("fsck",
 		"check the repository",
 		"The fsck command check the integrity and consistency of the repository",
-		&CmdFsck{})
+		&CmdFsck{global: &globalOpts})
 	if err != nil {
 		panic(err)
 	}
 }
 
-func fsckFile(opts CmdFsck, repo *repository.Repository, IDs []backend.ID) (uint64, error) {
+func fsckFile(global CmdFsck, repo *repository.Repository, IDs []backend.ID) (uint64, error) {
 	debug.Log("restic.fsckFile", "checking file %v", IDs)
 	var bytes uint64
 
@@ -50,7 +52,7 @@ func fsckFile(opts CmdFsck, repo *repository.Repository, IDs []backend.ID) (uint
 		bytes += uint64(length - crypto.Extension)
 		debug.Log("restic.fsck", "  blob found in pack %v\n", packID)
 
-		if opts.CheckData {
+		if global.CheckData {
 			// load content
 			_, err := repo.LoadBlob(pack.Data, id)
 			if err != nil {
@@ -69,16 +71,16 @@ func fsckFile(opts CmdFsck, repo *repository.Repository, IDs []backend.ID) (uint
 		}
 
 		// if orphan check is active, record storage id
-		if opts.o_data != nil {
+		if global.o_data != nil {
 			debug.Log("restic.fsck", "  recording blob %v as used\n", id)
-			opts.o_data.Insert(id)
+			global.o_data.Insert(id)
 		}
 	}
 
 	return bytes, nil
 }
 
-func fsckTree(opts CmdFsck, repo *repository.Repository, id backend.ID) error {
+func fsckTree(global CmdFsck, repo *repository.Repository, id backend.ID) error {
 	debug.Log("restic.fsckTree", "checking tree %v", id.Str())
 
 	tree, err := restic.LoadTree(repo, id)
@@ -87,9 +89,9 @@ func fsckTree(opts CmdFsck, repo *repository.Repository, id backend.ID) error {
 	}
 
 	// if orphan check is active, record storage id
-	if opts.o_trees != nil {
+	if global.o_trees != nil {
 		// add ID to list
-		opts.o_trees.Insert(id)
+		global.o_trees.Insert(id)
 	}
 
 	var firstErr error
@@ -123,7 +125,7 @@ func fsckTree(opts CmdFsck, repo *repository.Repository, id backend.ID) error {
 			}
 
 			debug.Log("restic.fsckTree", "check file %v (%v)", node.Name, id.Str())
-			bytes, err := fsckFile(opts, repo, node.Content)
+			bytes, err := fsckFile(global, repo, node.Content)
 			if err != nil {
 				return err
 			}
@@ -140,7 +142,7 @@ func fsckTree(opts CmdFsck, repo *repository.Repository, id backend.ID) error {
 			// record id
 			seenIDs.Insert(node.Subtree)
 
-			err = fsckTree(opts, repo, node.Subtree)
+			err = fsckTree(global, repo, node.Subtree)
 			if err != nil {
 				firstErr = err
 				fmt.Fprintf(os.Stderr, "%v\n", err)
@@ -158,7 +160,7 @@ func fsckTree(opts CmdFsck, repo *repository.Repository, id backend.ID) error {
 	return firstErr
 }
 
-func fsckSnapshot(opts CmdFsck, repo *repository.Repository, id backend.ID) error {
+func fsckSnapshot(global CmdFsck, repo *repository.Repository, id backend.ID) error {
 	debug.Log("restic.fsck", "checking snapshot %v\n", id)
 
 	sn, err := restic.LoadSnapshot(repo, id)
@@ -166,7 +168,7 @@ func fsckSnapshot(opts CmdFsck, repo *repository.Repository, id backend.ID) erro
 		return fmt.Errorf("loading snapshot %v failed: %v", id, err)
 	}
 
-	err = fsckTree(opts, repo, sn.Tree)
+	err = fsckTree(global, repo, sn.Tree)
 	if err != nil {
 		debug.Log("restic.fsck", "  checking tree %v for snapshot %v\n", sn.Tree, id)
 		fmt.Fprintf(os.Stderr, "snapshot %v:\n  error for tree %v:\n    %v\n", id, sn.Tree, err)
@@ -188,7 +190,7 @@ func (cmd CmdFsck) Execute(args []string) error {
 		cmd.Orphaned = true
 	}
 
-	s, err := OpenRepo()
+	s, err := cmd.global.OpenRepository()
 	if err != nil {
 		return err
 	}

cmd/restic/cmd_init.go

@@ -0,0 +1,52 @@
+package main
+
+import (
+	"errors"
+
+	"github.com/restic/restic/repository"
+)
+
+type CmdInit struct {
+	global *GlobalOptions
+}
+
+func (cmd CmdInit) Execute(args []string) error {
+	if cmd.global.Repo == "" {
+		return errors.New("Please specify repository location (-r)")
+	}
+
+	if cmd.global.password == "" {
+		cmd.global.password = cmd.global.ReadPasswordTwice(
+			"enter password for new backend: ",
+			"enter password again: ")
+	}
+
+	be, err := create(cmd.global.Repo)
+	if err != nil {
+		cmd.global.Exitf(1, "creating backend at %s failed: %v\n", cmd.global.Repo, err)
+	}
+
+	s := repository.New(be)
+	err = s.Init(cmd.global.password)
+	if err != nil {
+		cmd.global.Exitf(1, "creating key in backend at %s failed: %v\n", cmd.global.Repo, err)
+	}
+
+	cmd.global.Verbosef("created restic backend %v at %s\n", s.Config.ID[:10], cmd.global.Repo)
+	cmd.global.Verbosef("\n")
+	cmd.global.Verbosef("Please note that knowledge of your password is required to access\n")
+	cmd.global.Verbosef("the repository. Losing your password means that your data is\n")
+	cmd.global.Verbosef("irrecoverably lost.\n")
+
+	return nil
+}
+
+func init() {
+	_, err := parser.AddCommand("init",
+		"create repository",
+		"The init command creates a new repository",
+		&CmdInit{global: &globalOpts})
+	if err != nil {
+		panic(err)
+	}
+}

cmd/restic/cmd_key.go

@@ -3,25 +3,27 @@ package main
 import (
 	"errors"
 	"fmt"
-	"os"
 
 	"github.com/restic/restic/backend"
 	"github.com/restic/restic/repository"
 )
 
-type CmdKey struct{}
+type CmdKey struct {
+	global      *GlobalOptions
+	newPassword string
+}
 
 func init() {
 	_, err := parser.AddCommand("key",
 		"manage keys",
 		"The key command manages keys (passwords) of a repository",
-		&CmdKey{})
+		&CmdKey{global: &globalOpts})
 	if err != nil {
 		panic(err)
 	}
 }
 
-func listKeys(s *repository.Repository) error {
+func (cmd CmdKey) listKeys(s *repository.Repository) error {
 	tab := NewTable()
 	tab.Header = fmt.Sprintf(" %-10s  %-10s  %-10s  %s", "ID", "User", "Host", "Created")
 	tab.RowFormat = "%s%-10s  %-10s  %-10s  %s"
@@ -37,7 +39,7 @@ func listKeys(s *repository.Repository) error {
 	for id := range s.List(backend.Key, done) {
 		k, err := repository.LoadKey(s, id.String())
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "LoadKey() failed: %v\n", err)
+			cmd.global.Warnf("LoadKey() failed: %v\n", err)
 			continue
 		}
 
@@ -51,43 +53,31 @@ func listKeys(s *repository.Repository) error {
 			k.Username, k.Hostname, k.Created.Format(TimeFormat)})
 	}
 
-	tab.Write(os.Stdout)
-
-	return nil
+	return tab.Write(cmd.global.stdout)
 }
 
-func getNewPassword() (string, error) {
-	newPassword := os.Getenv("RESTIC_NEWPASSWORD")
-
-	if newPassword == "" {
-		newPassword = readPassword("enter password for new key: ")
-		newPassword2 := readPassword("enter password again: ")
-
-		if newPassword != newPassword2 {
-			return "", errors.New("passwords do not match")
-		}
+func (cmd CmdKey) getNewPassword() string {
+	if cmd.newPassword != "" {
+		return cmd.newPassword
 	}
 
-	return newPassword, nil
+	return cmd.global.ReadPasswordTwice(
+		"enter password for new key: ",
+		"enter password again: ")
 }
 
-func addKey(repo *repository.Repository) error {
-	newPassword, err := getNewPassword()
-	if err != nil {
-		return err
-	}
-
-	id, err := repository.AddKey(repo, newPassword, repo.Key())
+func (cmd CmdKey) addKey(repo *repository.Repository) error {
+	id, err := repository.AddKey(repo, cmd.getNewPassword(), repo.Key())
 	if err != nil {
 		return fmt.Errorf("creating new key failed: %v\n", err)
 	}
 
-	fmt.Printf("saved new key as %s\n", id)
+	cmd.global.Verbosef("saved new key as %s\n", id)
 
 	return nil
 }
 
-func deleteKey(repo *repository.Repository, name string) error {
+func (cmd CmdKey) deleteKey(repo *repository.Repository, name string) error {
 	if name == repo.KeyName() {
 		return errors.New("refusing to remove key currently used to access repository")
 	}
@@ -97,17 +87,12 @@ func deleteKey(repo *repository.Repository, name string) error {
 		return err
 	}
 
-	fmt.Printf("removed key %v\n", name)
+	cmd.global.Verbosef("removed key %v\n", name)
 	return nil
 }
 
-func changePassword(repo *repository.Repository) error {
-	newPassword, err := getNewPassword()
-	if err != nil {
-		return err
-	}
-
-	id, err := repository.AddKey(repo, newPassword, repo.Key())
+func (cmd CmdKey) changePassword(repo *repository.Repository) error {
+	id, err := repository.AddKey(repo, cmd.getNewPassword(), repo.Key())
 	if err != nil {
 		return fmt.Errorf("creating new key failed: %v\n", err)
 	}
@@ -117,7 +102,7 @@ func changePassword(repo *repository.Repository) error {
 		return err
 	}
 
-	fmt.Printf("saved new key as %s\n", id)
+	cmd.global.Verbosef("saved new key as %s\n", id)
 
 	return nil
 }
@@ -131,25 +116,25 @@ func (cmd CmdKey) Execute(args []string) error {
 		return fmt.Errorf("wrong number of arguments, Usage: %s", cmd.Usage())
 	}
 
-	s, err := OpenRepo()
+	s, err := cmd.global.OpenRepository()
 	if err != nil {
 		return err
 	}
 
 	switch args[0] {
 	case "list":
-		return listKeys(s)
+		return cmd.listKeys(s)
 	case "add":
-		return addKey(s)
+		return cmd.addKey(s)
 	case "rm":
 		id, err := backend.Find(s.Backend(), backend.Key, args[1])
 		if err != nil {
 			return err
 		}
 
-		return deleteKey(s, id)
+		return cmd.deleteKey(s, id)
 	case "passwd":
-		return changePassword(s)
+		return cmd.changePassword(s)
 	}
 
 	return nil

cmd/restic/cmd_list.go

@@ -7,13 +7,15 @@ import (
 	"github.com/restic/restic/backend"
 )
 
-type CmdList struct{}
+type CmdList struct {
+	global *GlobalOptions
+}
 
 func init() {
 	_, err := parser.AddCommand("list",
 		"lists data",
 		"The list command lists structures or data of a repository",
-		&CmdList{})
+		&CmdList{global: &globalOpts})
 	if err != nil {
 		panic(err)
 	}
@@ -28,7 +30,7 @@ func (cmd CmdList) Execute(args []string) error {
 		return fmt.Errorf("type not specified, Usage: %s", cmd.Usage())
 	}
 
-	s, err := OpenRepo()
+	s, err := cmd.global.OpenRepository()
 	if err != nil {
 		return err
 	}
@@ -42,7 +44,7 @@ func (cmd CmdList) Execute(args []string) error {
 		}
 
 		for blob := range s.Index().Each(nil) {
-			fmt.Println(blob.ID)
+			cmd.global.Printf("%s\n", blob.ID)
 		}
 
 		return nil
@@ -61,7 +63,7 @@ func (cmd CmdList) Execute(args []string) error {
 	}
 
 	for id := range s.List(t, nil) {
-		fmt.Printf("%s\n", id)
+		cmd.global.Printf("%s\n", id)
 	}
 
 	return nil

cmd/restic/cmd_ls.go

@@ -10,13 +10,15 @@ import (
 	"github.com/restic/restic/repository"
 )
 
-type CmdLs struct{}
+type CmdLs struct {
+	global *GlobalOptions
+}
 
 func init() {
 	_, err := parser.AddCommand("ls",
 		"list files",
 		"The ls command lists all files and directories in a snapshot",
-		&CmdLs{})
+		&CmdLs{global: &globalOpts})
 	if err != nil {
 		panic(err)
 	}
@@ -67,7 +69,7 @@ func (cmd CmdLs) Execute(args []string) error {
 		return fmt.Errorf("wrong number of arguments, Usage: %s", cmd.Usage())
 	}
 
-	s, err := OpenRepo()
+	s, err := cmd.global.OpenRepository()
 	if err != nil {
 		return err
 	}

cmd/restic/cmd_restore.go

@@ -2,19 +2,20 @@ package main
 
 import (
 	"fmt"
-	"os"
 	"path/filepath"
 
 	"github.com/restic/restic"
 )
 
-type CmdRestore struct{}
+type CmdRestore struct {
+	global *GlobalOptions
+}
 
 func init() {
 	_, err := parser.AddCommand("restore",
 		"restore a snapshot",
 		"The restore command restores a snapshot to a directory",
-		&CmdRestore{})
+		&CmdRestore{global: &globalOpts})
 	if err != nil {
 		panic(err)
 	}
@@ -29,7 +30,7 @@ func (cmd CmdRestore) Execute(args []string) error {
 		return fmt.Errorf("wrong number of arguments, Usage: %s", cmd.Usage())
 	}
 
-	s, err := OpenRepo()
+	s, err := cmd.global.OpenRepository()
 	if err != nil {
 		return err
 	}
@@ -41,7 +42,7 @@ func (cmd CmdRestore) Execute(args []string) error {
 
 	id, err := restic.FindSnapshot(s, args[0])
 	if err != nil {
-		errx(1, "invalid id %q: %v", args[0], err)
+		cmd.global.Exitf(1, "invalid id %q: %v", args[0], err)
 	}
 
 	target := args[1]
@@ -49,12 +50,11 @@ func (cmd CmdRestore) Execute(args []string) error {
 	// create restorer
 	res, err := restic.NewRestorer(s, id)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "creating restorer failed: %v\n", err)
-		os.Exit(2)
+		cmd.global.Exitf(2, "creating restorer failed: %v\n", err)
 	}
 
 	res.Error = func(dir string, node *restic.Node, err error) error {
-		fmt.Fprintf(os.Stderr, "error for %s: %+v\n", dir, err)
+		cmd.global.Warnf("error for %s: %+v\n", dir, err)
 
 		// if node.Type == "dir" {
 		// 	if e, ok := err.(*os.PathError); ok {
@@ -81,7 +81,7 @@ func (cmd CmdRestore) Execute(args []string) error {
 		}
 	}
 
-	fmt.Printf("restoring %s to %s\n", res.Snapshot(), target)
+	cmd.global.Verbosef("restoring %s to %s\n", res.Snapshot(), target)
 
 	err = res.RestoreTo(target)
 	if err != nil {

cmd/restic/cmd_snapshots.go

@@ -71,13 +71,15 @@ func reltime(t time.Time) string {
 	}
 }
 
-type CmdSnapshots struct{}
+type CmdSnapshots struct {
+	global *GlobalOptions
+}
 
 func init() {
 	_, err := parser.AddCommand("snapshots",
 		"show snapshots",
 		"The snapshots command lists all snapshots stored in a repository",
-		&CmdSnapshots{})
+		&CmdSnapshots{global: &globalOpts})
 	if err != nil {
 		panic(err)
 	}
@@ -92,7 +94,7 @@ func (cmd CmdSnapshots) Execute(args []string) error {
 		return fmt.Errorf("wrong number of arguments, usage: %s", cmd.Usage())
 	}
 
-	s, err := OpenRepo()
+	s, err := cmd.global.OpenRepository()
 	if err != nil {
 		return err
 	}

cmd/restic/global.go

@@ -0,0 +1,171 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"net/url"
+	"os"
+
+	"github.com/jessevdk/go-flags"
+	"github.com/restic/restic/backend"
+	"github.com/restic/restic/backend/local"
+	"github.com/restic/restic/backend/sftp"
+	"github.com/restic/restic/repository"
+	"golang.org/x/crypto/ssh/terminal"
+)
+
+var version = "compiled manually"
+
+type GlobalOptions struct {
+	Repo     string `short:"r" long:"repo"                      description:"Repository directory to backup to/restore from"`
+	CacheDir string `          long:"cache-dir"                 description:"Directory to use as a local cache"`
+	Quiet    bool   `short:"q" long:"quiet"     default:"false" description:"Do not output comprehensive progress report"`
+
+	password string
+	stdout   io.Writer
+	stderr   io.Writer
+}
+
+var globalOpts = GlobalOptions{stdout: os.Stdout, stderr: os.Stderr}
+var parser = flags.NewParser(&globalOpts, flags.Default)
+
+func (o GlobalOptions) Printf(format string, args ...interface{}) {
+	_, err := fmt.Fprintf(o.stdout, format, args...)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "unable to write to stdout: %v\n", err)
+		os.Exit(100)
+	}
+}
+
+func (o GlobalOptions) Verbosef(format string, args ...interface{}) {
+	if o.Quiet {
+		return
+	}
+
+	o.Printf(format, args...)
+}
+
+func (o GlobalOptions) ShowProgress() bool {
+	if o.Quiet {
+		return false
+	}
+
+	if !terminal.IsTerminal(int(os.Stdout.Fd())) {
+		return false
+	}
+
+	return true
+}
+
+func (o GlobalOptions) Warnf(format string, args ...interface{}) {
+	_, err := fmt.Fprintf(o.stderr, format, args...)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "unable to write to stderr: %v\n", err)
+		os.Exit(100)
+	}
+}
+
+func (o GlobalOptions) Exitf(exitcode int, format string, args ...interface{}) {
+	if format[len(format)-1] != '\n' {
+		format += "\n"
+	}
+
+	o.Warnf(format, args...)
+	os.Exit(exitcode)
+}
+
+func (o GlobalOptions) ReadPassword(prompt string) string {
+	fmt.Fprint(os.Stderr, prompt)
+	pw, err := terminal.ReadPassword(int(os.Stdin.Fd()))
+	if err != nil {
+		o.Exitf(2, "unable to read password: %v", err)
+	}
+	fmt.Fprintln(os.Stderr)
+
+	if len(pw) == 0 {
+		o.Exitf(1, "an empty password is not a password")
+	}
+
+	return string(pw)
+}
+
+func (o GlobalOptions) ReadPasswordTwice(prompt1, prompt2 string) string {
+	pw1 := o.ReadPassword(prompt1)
+	pw2 := o.ReadPassword(prompt2)
+	if pw1 != pw2 {
+		o.Exitf(1, "passwords do not match")
+	}
+
+	return pw1
+}
+
+func (o GlobalOptions) OpenRepository() (*repository.Repository, error) {
+	if o.Repo == "" {
+		return nil, errors.New("Please specify repository location (-r)")
+	}
+
+	be, err := open(o.Repo)
+	if err != nil {
+		return nil, err
+	}
+
+	s := repository.New(be)
+
+	if o.password == "" {
+		o.password = o.ReadPassword("enter password for repository: ")
+	}
+
+	err = s.SearchKey(o.password)
+	if err != nil {
+		return nil, fmt.Errorf("unable to open repo: %v", err)
+	}
+
+	return s, nil
+}
+
+// Open the backend specified by URI.
+// Valid formats are:
+// * /foo/bar -> local repository at /foo/bar
+// * sftp://user@host/foo/bar -> remote sftp repository on host for user at path foo/bar
+// * sftp://host//tmp/backup -> remote sftp repository on host at path /tmp/backup
+func open(u string) (backend.Backend, error) {
+	url, err := url.Parse(u)
+	if err != nil {
+		return nil, err
+	}
+
+	if url.Scheme == "" {
+		return local.Open(url.Path)
+	}
+
+	args := []string{url.Host}
+	if url.User != nil && url.User.Username() != "" {
+		args = append(args, "-l")
+		args = append(args, url.User.Username())
+	}
+	args = append(args, "-s")
+	args = append(args, "sftp")
+	return sftp.Open(url.Path[1:], "ssh", args...)
+}
+
+// Create the backend specified by URI.
+func create(u string) (backend.Backend, error) {
+	url, err := url.Parse(u)
+	if err != nil {
+		return nil, err
+	}
+
+	if url.Scheme == "" {
+		return local.Create(url.Path)
+	}
+
+	args := []string{url.Host}
+	if url.User != nil && url.User.Username() != "" {
+		args = append(args, "-l")
+		args = append(args, url.User.Username())
+	}
+	args = append(args, "-s")
+	args = append(args, "sftp")
+	return sftp.Create(url.Path[1:], "ssh", args...)
+}

cmd/restic/integration_helpers_test.go

@@ -0,0 +1,226 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"syscall"
+	"testing"
+
+	. "github.com/restic/restic/test"
+)
+
+type dirEntry struct {
+	path string
+	fi   os.FileInfo
+}
+
+func walkDir(dir string) <-chan *dirEntry {
+	ch := make(chan *dirEntry, 100)
+
+	go func() {
+		err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "error: %v\n", err)
+				return nil
+			}
+
+			name, err := filepath.Rel(dir, path)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "error: %v\n", err)
+				return nil
+			}
+
+			ch <- &dirEntry{
+				path: name,
+				fi:   info,
+			}
+
+			return nil
+		})
+
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Walk() error: %v\n", err)
+		}
+
+		close(ch)
+	}()
+
+	// first element is root
+	_ = <-ch
+
+	return ch
+}
+
+func (e *dirEntry) equals(other *dirEntry) bool {
+	if e.path != other.path {
+		fmt.Fprintf(os.Stderr, "%v: path does not match\n", e.path)
+		return false
+	}
+
+	if e.fi.Mode() != other.fi.Mode() {
+		fmt.Fprintf(os.Stderr, "%v: mode does not match\n", e.path)
+		return false
+	}
+
+	if e.fi.ModTime() != other.fi.ModTime() {
+		fmt.Fprintf(os.Stderr, "%v: ModTime does not match\n", e.path)
+		return false
+	}
+
+	stat, _ := e.fi.Sys().(*syscall.Stat_t)
+	stat2, _ := other.fi.Sys().(*syscall.Stat_t)
+
+	if stat.Uid != stat2.Uid || stat2.Gid != stat2.Gid {
+		fmt.Fprintf(os.Stderr, "%v: UID or GID do not match\n", e.path)
+		return false
+	}
+
+	return true
+}
+
+// directoriesEqualContents checks if both directories contain exactly the same
+// contents.
+func directoriesEqualContents(dir1, dir2 string) bool {
+	ch1 := walkDir(dir1)
+	ch2 := walkDir(dir2)
+
+	changes := false
+
+	var a, b *dirEntry
+	for {
+		var ok bool
+
+		if ch1 != nil && a == nil {
+			a, ok = <-ch1
+			if !ok {
+				ch1 = nil
+			}
+		}
+
+		if ch2 != nil && b == nil {
+			b, ok = <-ch2
+			if !ok {
+				ch2 = nil
+			}
+		}
+
+		if ch1 == nil && ch2 == nil {
+			break
+		}
+
+		if ch1 == nil {
+			fmt.Printf("+%v\n", b.path)
+			changes = true
+		} else if ch2 == nil {
+			fmt.Printf("-%v\n", a.path)
+			changes = true
+		} else if !a.equals(b) {
+			if a.path < b.path {
+				fmt.Printf("-%v\n", a.path)
+				changes = true
+				a = nil
+				continue
+			} else if a.path > b.path {
+				fmt.Printf("+%v\n", b.path)
+				changes = true
+				b = nil
+				continue
+			} else {
+				fmt.Printf("%%%v\n", a.path)
+				changes = true
+			}
+		}
+
+		a, b = nil, nil
+	}
+
+	if changes {
+		return false
+	}
+
+	return true
+}
+
+type dirStat struct {
+	files, dirs, other uint
+	size               uint64
+}
+
+func isFile(fi os.FileInfo) bool {
+	return fi.Mode()&(os.ModeType|os.ModeCharDevice) == 0
+}
+
+// dirStats walks dir and collects stats.
+func dirStats(dir string) (stat dirStat) {
+	for entry := range walkDir(dir) {
+		if isFile(entry.fi) {
+			stat.files++
+			stat.size += uint64(entry.fi.Size())
+			continue
+		}
+
+		if entry.fi.IsDir() {
+			stat.dirs++
+			continue
+		}
+
+		stat.other++
+	}
+
+	return stat
+}
+
+type testEnvironment struct {
+	base, cache, repo, testdata string
+}
+
+func configureRestic(t testing.TB, cache, repo string) GlobalOptions {
+	return GlobalOptions{
+		CacheDir: cache,
+		Repo:     repo,
+		Quiet:    true,
+
+		password: TestPassword,
+		stdout:   os.Stdout,
+	}
+}
+
+func cleanupTempdir(t testing.TB, tempdir string) {
+	if !TestCleanup {
+		t.Logf("leaving temporary directory %v used for test", tempdir)
+		return
+	}
+
+	OK(t, os.RemoveAll(tempdir))
+}
+
+// withTestEnvironment creates a test environment and calls f with it. After f has
+// returned, the temporary directory is removed.
+func withTestEnvironment(t testing.TB, f func(*testEnvironment, GlobalOptions)) {
+	if !RunIntegrationTest {
+		t.Skip("integration tests disabled")
+	}
+
+	tempdir, err := ioutil.TempDir(TestTempDir, "restic-test-")
+	OK(t, err)
+
+	env := testEnvironment{
+		base:     tempdir,
+		cache:    filepath.Join(tempdir, "cache"),
+		repo:     filepath.Join(tempdir, "repo"),
+		testdata: filepath.Join(tempdir, "testdata"),
+	}
+
+	OK(t, os.MkdirAll(env.testdata, 0700))
+
+	f(&env, configureRestic(t, env.cache, env.repo))
+
+	if !TestCleanup {
+		t.Logf("leaving temporary directory %v used for test", tempdir)
+		return
+	}
+
+	OK(t, os.RemoveAll(tempdir))
+}

cmd/restic/integration_test.go

@@ -0,0 +1,390 @@
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"crypto/rand"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"testing"
+
+	"github.com/restic/restic/backend"
+	"github.com/restic/restic/debug"
+	. "github.com/restic/restic/test"
+)
+
+func setupTarTestFixture(t testing.TB, outputDir, tarFile string) {
+	err := system("sh", "-c", `(cd "$1" && tar xz) < "$2"`,
+		"sh", outputDir, tarFile)
+	OK(t, err)
+}
+
+func system(command string, args ...string) error {
+	cmd := exec.Command(command, args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+
+	return cmd.Run()
+}
+
+func parseIDsFromReader(t testing.TB, rd io.Reader) backend.IDs {
+	IDs := backend.IDs{}
+	sc := bufio.NewScanner(rd)
+
+	for sc.Scan() {
+		id, err := backend.ParseID(sc.Text())
+		if err != nil {
+			t.Logf("parse id %v: %v", sc.Text(), err)
+			continue
+		}
+
+		IDs = append(IDs, id)
+	}
+
+	return IDs
+}
+
+func cmdInit(t testing.TB, global GlobalOptions) {
+	cmd := &CmdInit{global: &global}
+	OK(t, cmd.Execute(nil))
+
+	t.Logf("repository initialized at %v", global.Repo)
+}
+
+func cmdBackup(t testing.TB, global GlobalOptions, target []string, parentID backend.ID) {
+	cmd := &CmdBackup{global: &global}
+	cmd.Parent = parentID.String()
+
+	t.Logf("backing up %v", target)
+
+	OK(t, cmd.Execute(target))
+}
+
+func cmdList(t testing.TB, global GlobalOptions, tpe string) []backend.ID {
+	var buf bytes.Buffer
+	global.stdout = &buf
+	cmd := &CmdList{global: &global}
+
+	OK(t, cmd.Execute([]string{tpe}))
+	IDs := parseIDsFromReader(t, &buf)
+
+	return IDs
+}
+
+func cmdRestore(t testing.TB, global GlobalOptions, dir string, snapshotID backend.ID) {
+	cmd := &CmdRestore{global: &global}
+	cmd.Execute([]string{snapshotID.String(), dir})
+}
+
+func cmdFsck(t testing.TB, global GlobalOptions) {
+	cmd := &CmdFsck{global: &global, CheckData: true, Orphaned: true}
+	OK(t, cmd.Execute(nil))
+}
+
+func TestBackup(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) {
+		datafile := filepath.Join("testdata", "backup-data.tar.gz")
+		fd, err := os.Open(datafile)
+		if os.IsNotExist(err) {
+			t.Skipf("unable to find data file %q, skipping", datafile)
+			return
+		}
+		OK(t, err)
+		OK(t, fd.Close())
+
+		cmdInit(t, global)
+
+		setupTarTestFixture(t, env.testdata, datafile)
+
+		// first backup
+		cmdBackup(t, global, []string{env.testdata}, nil)
+		snapshotIDs := cmdList(t, global, "snapshots")
+		Assert(t, len(snapshotIDs) == 1,
+			"expected one snapshot, got %v", snapshotIDs)
+
+		cmdFsck(t, global)
+		stat1 := dirStats(env.repo)
+
+		// second backup, implicit incremental
+		cmdBackup(t, global, []string{env.testdata}, nil)
+		snapshotIDs = cmdList(t, global, "snapshots")
+		Assert(t, len(snapshotIDs) == 2,
+			"expected two snapshots, got %v", snapshotIDs)
+
+		stat2 := dirStats(env.repo)
+		if stat2.size > stat1.size+stat1.size/10 {
+			t.Error("repository size has grown by more than 10 percent")
+		}
+		t.Logf("repository grown by %d bytes", stat2.size-stat1.size)
+
+		cmdFsck(t, global)
+		// third backup, explicit incremental
+		cmdBackup(t, global, []string{env.testdata}, snapshotIDs[0])
+		snapshotIDs = cmdList(t, global, "snapshots")
+		Assert(t, len(snapshotIDs) == 3,
+			"expected three snapshots, got %v", snapshotIDs)
+
+		stat3 := dirStats(env.repo)
+		if stat3.size > stat1.size+stat1.size/10 {
+			t.Error("repository size has grown by more than 10 percent")
+		}
+		t.Logf("repository grown by %d bytes", stat3.size-stat2.size)
+
+		// restore all backups and compare
+		for i, snapshotID := range snapshotIDs {
+			restoredir := filepath.Join(env.base, fmt.Sprintf("restore%d", i))
+			t.Logf("restoring snapshot %v to %v", snapshotID.Str(), restoredir)
+			cmdRestore(t, global, restoredir, snapshotIDs[0])
+			Assert(t, directoriesEqualContents(env.testdata, filepath.Join(restoredir, "testdata")),
+				"directories are not equal")
+		}
+
+		cmdFsck(t, global)
+	})
+}
+
+func TestBackupNonExistingFile(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) {
+		datafile := filepath.Join("testdata", "backup-data.tar.gz")
+		fd, err := os.Open(datafile)
+		if os.IsNotExist(err) {
+			t.Skipf("unable to find data file %q, skipping", datafile)
+			return
+		}
+		OK(t, err)
+		OK(t, fd.Close())
+
+		setupTarTestFixture(t, env.testdata, datafile)
+
+		cmdInit(t, global)
+
+		global.stderr = ioutil.Discard
+
+		p := filepath.Join(env.testdata, "0", "0")
+		dirs := []string{
+			filepath.Join(p, "0"),
+			filepath.Join(p, "1"),
+			filepath.Join(p, "nonexisting"),
+			filepath.Join(p, "5"),
+		}
+		cmdBackup(t, global, dirs, nil)
+	})
+}
+
+func TestBackupMissingFile1(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) {
+		datafile := filepath.Join("testdata", "backup-data.tar.gz")
+		fd, err := os.Open(datafile)
+		if os.IsNotExist(err) {
+			t.Skipf("unable to find data file %q, skipping", datafile)
+			return
+		}
+		OK(t, err)
+		OK(t, fd.Close())
+
+		setupTarTestFixture(t, env.testdata, datafile)
+
+		cmdInit(t, global)
+
+		ranHook := false
+		debug.Hook("pipe.walk1", func(context interface{}) {
+			pathname := context.(string)
+
+			if pathname != filepath.Join("testdata", "0", "0", "9") {
+				return
+			}
+
+			t.Logf("in hook, removing test file testdata/0/0/9/37")
+			ranHook = true
+
+			OK(t, os.Remove(filepath.Join(env.testdata, "0", "0", "9", "37")))
+		})
+
+		cmdBackup(t, global, []string{env.testdata}, nil)
+		cmdFsck(t, global)
+
+		Assert(t, ranHook, "hook did not run")
+		debug.RemoveHook("pipe.walk1")
+	})
+}
+
+func TestBackupMissingFile2(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) {
+		datafile := filepath.Join("testdata", "backup-data.tar.gz")
+		fd, err := os.Open(datafile)
+		if os.IsNotExist(err) {
+			t.Skipf("unable to find data file %q, skipping", datafile)
+			return
+		}
+		OK(t, err)
+		OK(t, fd.Close())
+
+		setupTarTestFixture(t, env.testdata, datafile)
+
+		cmdInit(t, global)
+
+		ranHook := false
+		debug.Hook("pipe.walk2", func(context interface{}) {
+			pathname := context.(string)
+
+			if pathname != filepath.Join("testdata", "0", "0", "9", "37") {
+				return
+			}
+
+			t.Logf("in hook, removing test file testdata/0/0/9/37")
+			ranHook = true
+
+			OK(t, os.Remove(filepath.Join(env.testdata, "0", "0", "9", "37")))
+		})
+
+		cmdBackup(t, global, []string{env.testdata}, nil)
+		cmdFsck(t, global)
+
+		Assert(t, ranHook, "hook did not run")
+		debug.RemoveHook("pipe.walk2")
+	})
+}
+
+const (
+	incrementalFirstWrite  = 20 * 1042 * 1024
+	incrementalSecondWrite = 12 * 1042 * 1024
+	incrementalThirdWrite  = 4 * 1042 * 1024
+)
+
+func appendRandomData(filename string, bytes uint) error {
+	f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE, 0666)
+	if err != nil {
+		fmt.Fprint(os.Stderr, err)
+		return err
+	}
+
+	_, err = f.Seek(0, 2)
+	if err != nil {
+		fmt.Fprint(os.Stderr, err)
+		return err
+	}
+
+	_, err = io.Copy(f, io.LimitReader(rand.Reader, int64(bytes)))
+	if err != nil {
+		fmt.Fprint(os.Stderr, err)
+		return err
+	}
+
+	return f.Close()
+}
+
+func TestIncrementalBackup(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) {
+		cmdInit(t, global)
+
+		datadir := filepath.Join(env.base, "testdata")
+		testfile := filepath.Join(datadir, "testfile")
+
+		OK(t, appendRandomData(testfile, incrementalFirstWrite))
+
+		cmdBackup(t, global, []string{datadir}, nil)
+		cmdFsck(t, global)
+		stat1 := dirStats(env.repo)
+
+		OK(t, appendRandomData(testfile, incrementalSecondWrite))
+
+		cmdBackup(t, global, []string{datadir}, nil)
+		cmdFsck(t, global)
+		stat2 := dirStats(env.repo)
+		if stat2.size-stat1.size > incrementalFirstWrite {
+			t.Errorf("repository size has grown by more than %d bytes", incrementalFirstWrite)
+		}
+		t.Logf("repository grown by %d bytes", stat2.size-stat1.size)
+
+		OK(t, appendRandomData(testfile, incrementalThirdWrite))
+
+		cmdBackup(t, global, []string{datadir}, nil)
+		cmdFsck(t, global)
+		stat3 := dirStats(env.repo)
+		if stat3.size-stat2.size > incrementalFirstWrite {
+			t.Errorf("repository size has grown by more than %d bytes", incrementalFirstWrite)
+		}
+		t.Logf("repository grown by %d bytes", stat3.size-stat2.size)
+	})
+}
+
+func cmdKey(t testing.TB, global GlobalOptions, args ...string) string {
+	var buf bytes.Buffer
+
+	global.stdout = &buf
+	cmd := &CmdKey{global: &global}
+	OK(t, cmd.Execute(args))
+
+	return buf.String()
+}
+
+func cmdKeyListOtherIDs(t testing.TB, global GlobalOptions) []string {
+	var buf bytes.Buffer
+
+	global.stdout = &buf
+	cmd := &CmdKey{global: &global}
+	OK(t, cmd.Execute([]string{"list"}))
+
+	scanner := bufio.NewScanner(&buf)
+	exp := regexp.MustCompile(`^ ([a-f0-9]+) `)
+
+	IDs := []string{}
+	for scanner.Scan() {
+		if id := exp.FindStringSubmatch(scanner.Text()); id != nil {
+			IDs = append(IDs, id[1])
+		}
+	}
+
+	return IDs
+}
+
+func cmdKeyAddNewKey(t testing.TB, global GlobalOptions, newPassword string) {
+	cmd := &CmdKey{global: &global, newPassword: newPassword}
+	OK(t, cmd.Execute([]string{"add"}))
+}
+
+func cmdKeyPasswd(t testing.TB, global GlobalOptions, newPassword string) {
+	cmd := &CmdKey{global: &global, newPassword: newPassword}
+	OK(t, cmd.Execute([]string{"passwd"}))
+}
+
+func cmdKeyRemove(t testing.TB, global GlobalOptions, IDs []string) {
+	cmd := &CmdKey{global: &global}
+	t.Logf("remove %d keys: %q\n", len(IDs), IDs)
+	for _, id := range IDs {
+		OK(t, cmd.Execute([]string{"rm", id}))
+	}
+}
+
+func TestKeyAddRemove(t *testing.T) {
+	passwordList := []string{
+		"OnnyiasyatvodsEvVodyawit",
+		"raicneirvOjEfEigonOmLasOd",
+	}
+
+	withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) {
+		cmdInit(t, global)
+
+		cmdKeyPasswd(t, global, "geheim2")
+		global.password = "geheim2"
+		t.Logf("changed password to %q", global.password)
+
+		for _, newPassword := range passwordList {
+			cmdKeyAddNewKey(t, global, newPassword)
+			t.Logf("added new password %q", newPassword)
+			global.password = newPassword
+			cmdKeyRemove(t, global, cmdKeyListOtherIDs(t, global))
+		}
+
+		global.password = passwordList[len(passwordList)-1]
+		t.Logf("testing access with last password %q\n", global.password)
+		cmdKey(t, global, "list")
+
+		cmdFsck(t, global)
+	})
+}

cmd/restic/main.go

@@ -1,209 +1,23 @@
 package main
 
 import (
-	"errors"
-	"fmt"
-	"net/url"
 	"os"
 	"runtime"
 
-	"golang.org/x/crypto/ssh/terminal"
-
 	"github.com/jessevdk/go-flags"
-	"github.com/restic/restic/backend"
-	"github.com/restic/restic/backend/local"
-	"github.com/restic/restic/backend/sftp"
 	"github.com/restic/restic/debug"
-	"github.com/restic/restic/repository"
 )
 
-var version = "compiled manually"
-
-var opts struct {
-	Repo     string `short:"r" long:"repo"                      description:"Repository directory to backup to/restore from"`
-	CacheDir string `          long:"cache-dir"                 description:"Directory to use as a local cache"`
-	Quiet    bool   `short:"q" long:"quiet"     default:"false" description:"Do not output comprehensive progress report"`
-
-	password string
-}
-
-var parser = flags.NewParser(&opts, flags.Default)
-
-func errx(code int, format string, data ...interface{}) {
-	if len(format) > 0 && format[len(format)-1] != '\n' {
-		format += "\n"
-	}
-	fmt.Fprintf(os.Stderr, format, data...)
-	os.Exit(code)
-}
-
-func readPassword(prompt string) string {
-	fmt.Fprint(os.Stderr, prompt)
-	pw, err := terminal.ReadPassword(int(os.Stdin.Fd()))
-	if err != nil {
-		errx(2, "unable to read password: %v", err)
-	}
-	fmt.Fprintln(os.Stderr)
-
-	return string(pw)
-}
-
-func disableProgress() bool {
-	if opts.Quiet {
-		return true
-	}
-
-	if !terminal.IsTerminal(int(os.Stdout.Fd())) {
-		return true
-	}
-
-	return false
-}
-
-func silenceRequested() bool {
-	if opts.Quiet {
-		return true
-	}
-
-	return false
-}
-
-func verbosePrintf(format string, args ...interface{}) {
-	if silenceRequested() {
-		return
-	}
-
-	fmt.Printf(format, args...)
-}
-
-type CmdInit struct{}
-
-func (cmd CmdInit) Execute(args []string) error {
-	if opts.Repo == "" {
-		return errors.New("Please specify repository location (-r)")
-	}
-
-	if opts.password == "" {
-		pw := readPassword("enter password for new backend: ")
-		pw2 := readPassword("enter password again: ")
-
-		if pw != pw2 {
-			errx(1, "passwords do not match")
-		}
-
-		opts.password = pw
-	}
-
-	be, err := create(opts.Repo)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "creating backend at %s failed: %v\n", opts.Repo, err)
-		os.Exit(1)
-	}
-
-	s := repository.New(be)
-	err = s.Init(opts.password)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "creating key in backend at %s failed: %v\n", opts.Repo, err)
-		os.Exit(1)
-	}
-
-	verbosePrintf("created restic backend %v at %s\n", s.Config.ID[:10], opts.Repo)
-	verbosePrintf("\n")
-	verbosePrintf("Please note that knowledge of your password is required to access\n")
-	verbosePrintf("the repository. Losing your password means that your data is\n")
-	verbosePrintf("irrecoverably lost.\n")
-
-	return nil
-}
-
-// Open the backend specified by URI.
-// Valid formats are:
-// * /foo/bar -> local repository at /foo/bar
-// * sftp://user@host/foo/bar -> remote sftp repository on host for user at path foo/bar
-// * sftp://host//tmp/backup -> remote sftp repository on host at path /tmp/backup
-func open(u string) (backend.Backend, error) {
-	url, err := url.Parse(u)
-	if err != nil {
-		return nil, err
-	}
-
-	if url.Scheme == "" {
-		return local.Open(url.Path)
-	}
-
-	args := []string{url.Host}
-	if url.User != nil && url.User.Username() != "" {
-		args = append(args, "-l")
-		args = append(args, url.User.Username())
-	}
-	args = append(args, "-s")
-	args = append(args, "sftp")
-	return sftp.Open(url.Path[1:], "ssh", args...)
-}
-
-// Create the backend specified by URI.
-func create(u string) (backend.Backend, error) {
-	url, err := url.Parse(u)
-	if err != nil {
-		return nil, err
-	}
-
-	if url.Scheme == "" {
-		return local.Create(url.Path)
-	}
-
-	args := []string{url.Host}
-	if url.User != nil && url.User.Username() != "" {
-		args = append(args, "-l")
-		args = append(args, url.User.Username())
-	}
-	args = append(args, "-s")
-	args = append(args, "sftp")
-	return sftp.Create(url.Path[1:], "ssh", args...)
-}
-
-func OpenRepo() (*repository.Repository, error) {
-	if opts.Repo == "" {
-		return nil, errors.New("Please specify repository location (-r)")
-	}
-
-	be, err := open(opts.Repo)
-	if err != nil {
-		return nil, err
-	}
-
-	s := repository.New(be)
-
-	if opts.password == "" {
-		opts.password = readPassword("enter password for repository: ")
-	}
-
-	err = s.SearchKey(opts.password)
-	if err != nil {
-		return nil, fmt.Errorf("unable to open repo: %v", err)
-	}
-
-	return s, nil
-}
-
 func init() {
 	// set GOMAXPROCS to number of CPUs
 	runtime.GOMAXPROCS(runtime.NumCPU())
-
-	_, err := parser.AddCommand("init",
-		"create repository",
-		"The init command creates a new repository",
-		&CmdInit{})
-	if err != nil {
-		panic(err)
-	}
 }
 
 func main() {
 	// defer profile.Start(profile.MemProfileRate(100000), profile.ProfilePath(".")).Stop()
 	// defer profile.Start(profile.CPUProfile, profile.ProfilePath(".")).Stop()
-	opts.Repo = os.Getenv("RESTIC_REPOSITORY")
-	opts.password = os.Getenv("RESTIC_PASSWORD")
+	globalOpts.Repo = os.Getenv("RESTIC_REPOSITORY")
+	globalOpts.password = os.Getenv("RESTIC_PASSWORD")
 
 	debug.Log("restic", "main %#v", os.Args)
 

coverage_all.sh

@@ -1,10 +0,0 @@
-#!/bin/bash
-
-TARGETFILE="$1"
-
-go list ./... | while read pkg; do
-    go test -covermode=count -coverprofile=$(base64 <<< $pkg).cov $pkg
-done
-
-echo "mode: count" > $TARGETFILE
-tail -q -n +2 *.cov >> $TARGETFILE

debug/debug.go

@@ -18,7 +18,6 @@ import (
 var opts struct {
 	logger *log.Logger
 	tags   map[string]bool
-	breaks map[string]bool
 	m      sync.Mutex
 }
 
@@ -29,7 +28,6 @@ var _ = initDebug()
 func initDebug() bool {
 	initDebugLogger()
 	initDebugTags()
-	initDebugBreaks()
 
 	fmt.Fprintf(os.Stderr, "debug enabled\n")
 
@@ -105,25 +103,6 @@ func initDebugTags() {
 	fmt.Fprintf(os.Stderr, "debug log enabled for: %v\n", tags)
 }
 
-func initDebugBreaks() {
-	opts.breaks = make(map[string]bool)
-
-	env := os.Getenv("DEBUG_BREAK")
-	if len(env) == 0 {
-		return
-	}
-
-	breaks := []string{}
-
-	for _, tag := range strings.Split(env, ",") {
-		t := strings.TrimSpace(tag)
-		opts.breaks[t] = true
-		breaks = append(breaks, t)
-	}
-
-	fmt.Fprintf(os.Stderr, "debug breaks enabled for: %v\n", breaks)
-}
-
 // taken from https://github.com/VividCortex/trace
 func goroutineNum() int {
 	b := make([]byte, 20)
@@ -194,38 +173,3 @@ func Log(tag string, f string, args ...interface{}) {
 		dbgprint()
 	}
 }
-
-// Break stops the program if the debug tag is active and the string in tag is
-// contained in the DEBUG_BREAK environment variable.
-func Break(tag string) {
-	// check if breaking is enabled
-	if v, ok := opts.breaks[tag]; !ok || !v {
-		return
-	}
-
-	_, file, line, _ := runtime.Caller(1)
-	Log("break", "stopping process %d at %s (%v:%v)\n", os.Getpid(), tag, file, line)
-	p, err := os.FindProcess(os.Getpid())
-	if err != nil {
-		panic(err)
-	}
-
-	err = p.Signal(syscall.SIGSTOP)
-	if err != nil {
-		panic(err)
-	}
-}
-
-// BreakIf stops the program if the debug tag is active and the string in tag
-// is contained in the DEBUG_BREAK environment variable and the return value of
-// fn is true.
-func BreakIf(tag string, fn func() bool) {
-	// check if breaking is enabled
-	if v, ok := opts.breaks[tag]; !ok || !v {
-		return
-	}
-
-	if fn() {
-		Break(tag)
-	}
-}

debug/debug_release.go

@@ -3,7 +3,3 @@
 package debug
 
 func Log(tag string, fmt string, args ...interface{}) {}
-
-func Break(string) {}
-
-func BreakIf(string, func() bool) {}

debug/hooks.go

@@ -0,0 +1,28 @@
+// +build !release
+
+package debug
+
+var (
+	hooks map[string]func(interface{})
+)
+
+func init() {
+	hooks = make(map[string]func(interface{}))
+}
+
+func Hook(name string, f func(interface{})) {
+	hooks[name] = f
+}
+
+func RunHook(name string, context interface{}) {
+	f, ok := hooks[name]
+	if !ok {
+		return
+	}
+
+	f(context)
+}
+
+func RemoveHook(name string) {
+	delete(hooks, name)
+}

debug/hooks_release.go

@@ -0,0 +1,9 @@
+// +build release
+
+package debug
+
+func Hook(name string, f func(interface{})) {}
+
+func RunHook(name string, context interface{}) {}
+
+func RemoveHook(name string) {}

node_test.go

@@ -103,11 +103,11 @@ var nodeTests = []restic.Node{
 }
 
 func TestNodeRestoreAt(t *testing.T) {
-	tempdir, err := ioutil.TempDir(*TestTempDir, "restic-test-")
+	tempdir, err := ioutil.TempDir(TestTempDir, "restic-test-")
 	OK(t, err)
 
 	defer func() {
-		if *TestCleanup {
+		if TestCleanup {
 			OK(t, os.RemoveAll(tempdir))
 		} else {
 			t.Logf("leaving tempdir at %v", tempdir)

pipe/pipe.go

@@ -4,7 +4,6 @@ import (
 	"errors"
 	"fmt"
 	"os"
-	"path"
 	"path/filepath"
 	"sort"
 
@@ -108,17 +107,7 @@ func walk(basedir, dir string, done chan struct{}, jobs chan<- Job, res chan<- R
 
 	// Insert breakpoint to allow testing behaviour with vanishing files
 	// between Readdir() and lstat()
-	debug.BreakIf("pipe.walk1", func() bool {
-		match, err := path.Match(os.Getenv("DEBUG_BREAK_PIPE"), relpath)
-		if err != nil {
-			panic(err)
-		}
-		if match {
-			debug.Log("break", "break pattern matches for %v\n", relpath)
-		}
-
-		return match
-	})
+	debug.RunHook("pipe.walk1", relpath)
 
 	entries := make([]<-chan Result, 0, len(names))
 
@@ -140,19 +129,7 @@ func walk(basedir, dir string, done chan struct{}, jobs chan<- Job, res chan<- R
 
 		// Insert breakpoint to allow testing behaviour with vanishing files
 		// between walk and open
-		debug.BreakIf("pipe.walk2", func() bool {
-			p := filepath.Join(relpath, name)
-
-			match, err := path.Match(os.Getenv("DEBUG_BREAK_PIPE"), p)
-			if err != nil {
-				panic(err)
-			}
-			if match {
-				debug.Log("break", "break pattern matches for %v\n", p)
-			}
-
-			return match
-		})
+		debug.RunHook("pipe.walk2", filepath.Join(relpath, name))
 
 		if isDir(fi) {
 			err = walk(basedir, subpath, done, jobs, ch)

run_tests.go

@@ -0,0 +1,178 @@
+// +build ignore
+
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+)
+
+func specialDir(name string) bool {
+	if name == "." {
+		return false
+	}
+
+	base := filepath.Base(name)
+	return base[0] == '_' || base[0] == '.'
+}
+
+func emptyDir(name string) bool {
+	dir, err := os.Open(name)
+	defer dir.Close()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "unable to open directory %v: %v\n", name, err)
+		return true
+	}
+
+	fis, err := dir.Readdir(-1)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Readdirnames(%v): %v\n", name, err)
+		return true
+	}
+
+	for _, fi := range fis {
+		if fi.IsDir() {
+			continue
+		}
+
+		if filepath.Ext(fi.Name()) == ".go" {
+			return false
+		}
+	}
+
+	return true
+}
+
+func forceRelativeDirname(dirname string) string {
+	if dirname == "." {
+		return dirname
+	}
+
+	if strings.HasPrefix(dirname, "./") {
+		return dirname
+	}
+
+	return "./" + dirname
+}
+
+func mergeCoverprofile(file *os.File, out io.Writer) error {
+	_, err := file.Seek(0, 0)
+	if err != nil {
+		return err
+	}
+
+	rd := bufio.NewReader(file)
+	_, err = rd.ReadString('\n')
+	if err == io.EOF {
+		return nil
+	}
+
+	if err != nil {
+		return err
+	}
+
+	_, err = io.Copy(out, rd)
+	if err != nil {
+		return err
+	}
+
+	err = file.Close()
+	if err != nil {
+		return err
+	}
+
+	return err
+}
+
+func testPackage(pkg string, params []string, out io.Writer) error {
+	file, err := ioutil.TempFile("", "test-coverage-")
+	defer os.Remove(file.Name())
+	defer file.Close()
+	if err != nil {
+		return err
+	}
+
+	args := []string{"test", "-cover", "-covermode", "set", "-coverprofile",
+		file.Name(), pkg}
+	args = append(args, params...)
+	cmd := exec.Command("go", args...)
+	cmd.Stderr = os.Stderr
+	cmd.Stdout = os.Stdout
+
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+
+	return mergeCoverprofile(file, out)
+}
+
+func main() {
+	if len(os.Args) < 2 {
+		fmt.Fprintln(os.Stderr, "USAGE: run_tests COVERPROFILE [TESTFLAGS] [-- [PATHS]]")
+		os.Exit(1)
+	}
+
+	target := os.Args[1]
+	args := os.Args[2:]
+
+	paramsForTest := []string{}
+	dirs := []string{}
+	for i, arg := range args {
+		if arg == "--" {
+			dirs = args[i+1:]
+			break
+		}
+
+		paramsForTest = append(paramsForTest, arg)
+	}
+
+	if len(dirs) == 0 {
+		dirs = append(dirs, ".")
+	}
+
+	file, err := os.Create(target)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "create coverprofile failed: %v\n", err)
+		os.Exit(1)
+	}
+
+	fmt.Fprintln(file, "mode: set")
+
+	for _, dir := range dirs {
+		err := filepath.Walk(dir,
+			func(p string, fi os.FileInfo, e error) error {
+				if e != nil {
+					return e
+				}
+
+				if !fi.IsDir() {
+					return nil
+				}
+
+				if specialDir(p) {
+					return filepath.SkipDir
+				}
+
+				if emptyDir(p) {
+					return nil
+				}
+
+				return testPackage(forceRelativeDirname(p), paramsForTest, file)
+			})
+
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "walk(%q): %v\n", dir, err)
+		}
+	}
+
+	err = file.Close()
+
+	fmt.Printf("coverprofile: %v\n", file.Name())
+}

test/backend.go

@@ -1,8 +1,9 @@
 package test_helper
 
 import (
-	"flag"
+	"fmt"
 	"io/ioutil"
+	"os"
 	"path/filepath"
 	"testing"
 
@@ -12,12 +13,40 @@ import (
 	"github.com/restic/restic/repository"
 )
 
-var TestPassword = flag.String("test.password", "geheim", `use this password for repositories created during tests (default: "geheim")`)
-var TestCleanup = flag.Bool("test.cleanup", true, "clean up after running tests (remove local backend directory with all content)")
-var TestTempDir = flag.String("test.tempdir", "", "use this directory for temporary storage (default: system temp dir)")
+var (
+	TestPassword       = getStringVar("RESTIC_TEST_PASSWORD", "geheim")
+	TestCleanup        = getBoolVar("RESTIC_TEST_CLEANUP", true)
+	TestTempDir        = getStringVar("RESTIC_TEST_TMPDIR", "")
+	RunIntegrationTest = getBoolVar("RESTIC_TEST_INTEGRATION", true)
+	TestSFTPPath       = getStringVar("RESTIC_TEST_SFTPPATH",
+		"/usr/lib/ssh:/usr/lib/openssh")
+)
+
+func getStringVar(name, defaultValue string) string {
+	if e := os.Getenv(name); e != "" {
+		return e
+	}
+
+	return defaultValue
+}
+
+func getBoolVar(name string, defaultValue bool) bool {
+	if e := os.Getenv(name); e != "" {
+		switch e {
+		case "1":
+			return true
+		case "0":
+			return false
+		default:
+			fmt.Fprintf(os.Stderr, "invalid value for variable %q, using default\n", name)
+		}
+	}
+
+	return defaultValue
+}
 
 func SetupRepo(t testing.TB) *repository.Repository {
-	tempdir, err := ioutil.TempDir(*TestTempDir, "restic-test-")
+	tempdir, err := ioutil.TempDir(TestTempDir, "restic-test-")
 	OK(t, err)
 
 	// create repository below temp dir
@@ -25,12 +54,12 @@ func SetupRepo(t testing.TB) *repository.Repository {
 	OK(t, err)
 
 	repo := repository.New(b)
-	OK(t, repo.Init(*TestPassword))
+	OK(t, repo.Init(TestPassword))
 	return repo
 }
 
 func TeardownRepo(t testing.TB, repo *repository.Repository) {
-	if !*TestCleanup {
+	if !TestCleanup {
 		l := repo.Backend().(*local.Local)
 		t.Logf("leaving local backend at %s\n", l.Location())
 		return

testsuite.sh

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-# tempdir for binaries
-export BASEDIR="$(mktemp --tmpdir --directory restic-testsuite-XXXXXX)"
-export DEBUG_LOG="${BASEDIR}/restic.log"
-
-export TZ=UTC
-
-echo "restic testsuite basedir ${BASEDIR}"
-
-# run tests
-testsuite/run.sh "$@"

testsuite/run.sh

@@ -1,117 +0,0 @@
-#!/bin/bash
-
-set -e
-
-export dir=$(dirname "$0")
-export fake_data_file="${dir}/fake-data.tar.gz"
-
-prepare() {
-    export BASE="$(mktemp --tmpdir --directory restic-testsuite-XXXXXX)"
-    export RESTIC_REPOSITORY="${BASE}/restic-backup"
-    export RESTIC_PASSWORD="foobar"
-    export DATADIR="${BASE}/fake-data"
-    debug "repository is at ${RESTIC_REPOSITORY}"
-
-    mkdir -p "$DATADIR"
-    (cd "$DATADIR"; tar xz) < "$fake_data_file"
-    debug "extracted fake data to ${DATADIR}"
-}
-
-cleanup() {
-    if [ "$DEBUG" = "1" ]; then
-        debug "leaving dir ${BASE}"
-        return
-    fi
-
-    rm -rf "${BASE}"
-    debug "removed dir ${BASE}"
-    unset BASE
-    unset RESTIC_REPOSITORY
-}
-
-msg() {
-    printf "%s\n" "$*"
-}
-
-pass() {
-    printf "\e[32m%s\e[39m\n" "$*"
-}
-
-err() {
-    printf "\e[31m%s\e[39m\n" "$*"
-}
-
-debug() {
-    if [ "$DEBUG" = "1" ]; then
-        printf "\e[33m%s\e[39m\n" "$*"
-    fi
-}
-
-fail() {
-    err "$@"
-    exit 1
-}
-
-run() {
-    if [ "$DEBUG" = "1" ]; then
-        "$@"
-    else
-        "$@" > /dev/null
-    fi
-}
-
-export -f prepare cleanup msg debug pass err fail run
-
-if [ -z "$BASEDIR" ]; then
-    echo "BASEDIR not set" >&2
-    exit 2
-fi
-
-which restic > /dev/null || fail "restic binary not found!"
-which restic.debug > /dev/null || fail "restic.debug binary not found!"
-which dirdiff > /dev/null || fail "dirdiff binary not found!"
-
-debug "restic path: $(which restic)"
-debug "restic.debug path: $(which restic.debug)"
-debug "dirdiff path: $(which dirdiff)"
-debug "path: $PATH"
-
-debug "restic versions:"
-run restic version
-run restic.debug version
-
-if [ "$#" -gt 0 ]; then
-    testfiles="$1"
-else
-    testfiles=(${dir}/test-*.sh)
-fi
-
-echo "testfiles: ${testfiles[@]}"
-
-failed=""
-for testfile in "${testfiles[@]}"; do
-    msg "================================================================================"
-    msg "run test $testfile"
-    msg ""
-
-    current=$(basename "${testfile}" .sh)
-
-    if [ "$DEBUG" = "1" ]; then
-        OPTS="-v"
-    fi
-
-    if bash $OPTS "${testfile}"; then
-        pass "${current} pass"
-    else
-        err "${current} failed!"
-        failed+=" ${current}"
-    fi
-done
-
-if [ -n "$failed" ]; then
-    err "failed tests: ${failed}"
-    msg "restic versions:"
-    run restic version
-    run restic.debug version
-    exit 1
-fi

testsuite/test-backup-dedup.sh

@@ -1,23 +0,0 @@
-set -e
-
-prepare
-run restic init
-
-# first backup without dedup
-run restic backup "${BASE}/fake-data"
-size=$(du -sm "$RESTIC_REPOSITORY" | cut -f1)
-debug "size before: $size"
-
-# second backup with dedup
-run restic backup "${BASE}/fake-data"
-size2=$(du -sm "$RESTIC_REPOSITORY" | cut -f1)
-debug "size after: $size2"
-
-# check if the repository hasn't grown more than 5%
-threshhold=$(($size+$size/20))
-debug "threshhold is $threshhold"
-if [[ "$size2" -gt "$threshhold" ]]; then
-    fail "dedup failed, repo grown more than 5%, before ${size}MiB after ${size2}MiB threshhold ${threshhold}MiB"
-fi
-
-cleanup

testsuite/test-backup-incremental.sh

@@ -1,28 +0,0 @@
-set -e
-
-prepare
-run restic init
-
-# create testfile
-echo "testfile" > ${BASE}/fake-data/file
-
-# run first backup
-run restic backup "${BASE}/fake-data"
-
-# remember snapshot id
-SNAPSHOT=$(run restic list snapshots)
-
-# add data to testfile
-date >> ${BASE}/fake-data/file
-
-# run backup again
-run restic backup "${BASE}/fake-data"
-
-# add data to testfile
-date >> ${BASE}/fake-data/file
-
-# run incremental backup
-run restic backup -p "$SNAPSHOT" "${BASE}/fake-data"
-
-run restic fsck -o --check-data
-cleanup

testsuite/test-backup-missing-file1.sh

@@ -1,16 +0,0 @@
-set -em
-
-# setup restic
-prepare
-run restic init
-
-# start backup, break between readdir and lstat
-DEBUG_BREAK=pipe.walk1 DEBUG_BREAK_PIPE="fake-data/0/0/9" run restic.debug backup "${BASE}/fake-data" && debug "done"
-
-# remove file
-rm -f "${BASE}/fake-data/0/0/9/37"
-
-# resume backup
-fg
-
-cleanup

testsuite/test-backup-missing-file2.sh

@@ -1,16 +0,0 @@
-set -em
-
-# setup restic
-prepare
-run restic init
-
-# start backup, break between walk and save
-DEBUG_BREAK=pipe.walk2 DEBUG_BREAK_PIPE="fake-data/0/0/9/37" run restic.debug backup "${BASE}/fake-data" && debug "done"
-
-# remove file
-rm -f "${BASE}/fake-data/0/0/9/37"
-
-# resume backup
-fg
-
-cleanup

testsuite/test-backup-non-existing-file.sh

@@ -1,10 +0,0 @@
-set -em
-
-# setup restic
-prepare
-run restic init
-
-# start backup with non existing dir
-run timeout 10s restic.debug backup "${BASE}/fake-data/0/0/"{0,1,foobar,5} && debug "done" || false
-
-cleanup

testsuite/test-backup.sh

@@ -1,18 +0,0 @@
-set -e
-
-prepare
-run restic init
-run restic backup "${BASE}/fake-data"
-run restic restore "$(basename "$RESTIC_REPOSITORY"/snapshots/*)" "${BASE}/fake-data-restore"
-dirdiff "${BASE}/fake-data" "${BASE}/fake-data-restore/fake-data"
-
-SNAPSHOT=$(restic list snapshots)
-run restic backup -p "$SNAPSHOT" "${BASE}/fake-data"
-run restic restore "$(basename "$RESTIC_REPOSITORY"/snapshots/*)" "${BASE}/fake-data-restore-incremental"
-dirdiff "${BASE}/fake-data" "${BASE}/fake-data-restore-incremental/fake-data"
-
-echo "snapshot id is $SNAPSHOT"
-restic ls "$SNAPSHOT" fake-data/0/0/1 | head -n 10
-
-run restic fsck -o --check-data
-cleanup

testsuite/test-key-add-remove.sh

@@ -1,42 +0,0 @@
-set -e
-
-dump_repo() {
-    if [ "$FAILED" == "1" ]; then
-        tar cvz "$RESTIC_REPOSITORY" | base64 >&2
-    fi
-}
-
-FAILED=1
-
-trap dump_repo 0
-
-prepare
-unset RESTIC_PASSWORD
-RESTIC_PASSWORD=foo run restic init
-RESTIC_PASSWORD=foo run restic key list
-
-RESTIC_PASSWORD=foo RESTIC_NEWPASSWORD=foobar run restic key passwd
-RESTIC_PASSWORD=foobar run restic key list
-RESTIC_PASSWORD=foobar RESTIC_NEWPASSWORD=foo run restic key passwd
-
-OLD_PWD=foo
-for i in {1..3}; do
-    NEW_PWD=bar$i
-    RESTIC_PASSWORD=$OLD_PWD RESTIC_NEWPASSWORD=$NEW_PWD run restic key add
-    RESTIC_PASSWORD=$OLD_PWD run restic key list
-    RESTIC_PASSWORD=$NEW_PWD run restic key list
-
-    export RESTIC_PASSWORD=$OLD_PWD
-    ID=$(restic key list | grep '^\*'|cut -d ' ' -f 1| sed 's/^.//')
-    unset RESTIC_PASSWORD
-    RESTIC_PASSWORD=$NEW_PWD run restic key rm $ID
-    RESTIC_PASSWORD=$NEW_PWD run restic key list
-
-    OLD_PWD=bar$i
-done
-
-RESTIC_PASSWORD=$OLD_PWD run restic fsck -o --check-data
-
-cleanup
-
-FAILED=0

tree_test.go

@@ -22,7 +22,7 @@ var testFiles = []struct {
 }
 
 func createTempDir(t *testing.T) string {
-	tempdir, err := ioutil.TempDir(*TestTempDir, "restic-test-")
+	tempdir, err := ioutil.TempDir(TestTempDir, "restic-test-")
 	OK(t, err)
 
 	for _, test := range testFiles {
@@ -49,7 +49,7 @@ func createTempDir(t *testing.T) string {
 func TestTree(t *testing.T) {
 	dir := createTempDir(t)
 	defer func() {
-		if *TestCleanup {
+		if TestCleanup {
 			OK(t, os.RemoveAll(dir))
 		}
 	}()