diff --git a/.travis.yml b/.travis.yml index 199559449..38bef3d1b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,7 @@ os: - linux - osx -env: GOX_OS="linux darwin openbsd freebsd" GOX_ARCH="386 amd64 arm" +env: GOX_OS="linux darwin openbsd freebsd windows" GOX_ARCH="386 amd64 arm" notifications: irc: @@ -23,24 +23,10 @@ install: - go version - export GOBIN="$GOPATH/bin" - export PATH="$PATH:$GOBIN" - - export GOPATH="$GOPATH:${TRAVIS_BUILD_DIR}/Godeps/_workspace" - go env - - go get github.com/mattn/goveralls - - go get github.com/mitchellh/gox - - go version | grep -q "go1\.3" && export GOX_ARCH="386 amd64" || true - - go version | grep -q "darwin" && export GOX_OS="darwin" || true - - uname -s | grep -qi darwin && brew install caskroom/cask/brew-cask || true - - uname -s | grep -qi darwin && brew cask install osxfuse || true - - uname -s | grep -vqi darwin && export RESTIC_TEST_FUSE="0" || true - - echo "cross-compile for \"$GOX_OS\" on \"$GOX_ARCH\"" - - gox -build-toolchain -os "$GOX_OS" -arch "$GOX_ARCH" script: - - gox -verbose -os "$GOX_OS" -arch "$GOX_ARCH" -tags "release" ./cmd/restic - - gox -verbose -os "$GOX_OS" -arch "$GOX_ARCH" -tags "debug" ./cmd/restic - - go run build.go - - go run run_tests.go all.cov - - GOARCH=386 RESTIC_TEST_INTEGRATION=0 go test ./... + - go run run_integration_tests.go + +after_success: - goveralls -coverprofile=all.cov -service=travis-ci -repotoken "$COVERALLS_TOKEN" || true - - gofmt -l *.go */*.go */*/*.go - - test -z "$(gofmt -l *.go */*.go */*/*.go)" diff --git a/Makefile b/Makefile index e5a11720b..9052c8d46 100644 --- a/Makefile +++ b/Makefile @@ -1,22 +1,12 @@ .PHONY: all clean test -SOURCE=$(wildcard *.go) $(wildcard */*.go) $(wildcard */*/*.go) - -export GOPATH GOX_OS - all: restic restic: $(SOURCE) go run build.go -restic.debug: $(SOURCE) - go run build.go -tags debug - clean: - rm -rf restic restic.debug + rm -rf restic test: $(SOURCE) go run run_tests.go /dev/null - -all.cov: $(SOURCE) - go run run_tests.go all.cov diff --git a/README.md b/README.md index bdd3b3487..136e0b31d 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ [![Stories in Ready](https://badge.waffle.io/restic/restic.png?label=ready&title=Ready)](https://waffle.io/restic/restic) [![Build Status](https://travis-ci.org/restic/restic.svg?branch=master)](https://travis-ci.org/restic/restic) +[![Build status](https://ci.appveyor.com/api/projects/status/nuy4lfbgfbytw92q/branch/master?svg=true)](https://ci.appveyor.com/project/fd0/restic/branch/master) [![sourcegraph status](https://sourcegraph.com/api/repos/github.com/restic/restic/.badges/status.png)](https://sourcegraph.com/github.com/restic/restic) [![Coverage Status](https://coveralls.io/repos/restic/restic/badge.svg)](https://coveralls.io/r/restic/restic) diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 000000000..cc936f6f7 --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,14 @@ +clone_folder: c:\gopath\src\github.com\restic\restic + +environment: + GOPATH: c:\gopath;c:\gopath\src\github.com\restic\restic\Godeps\_workspace + +install: + - go version + - go env + - appveyor DownloadFile http://downloads.sourceforge.net/project/gnuwin32/tar/1.13-1/tar-1.13-1-bin.zip -FileName tar.zip + - 7z x tar.zip bin/tar.exe + - set PATH=bin/;%PATH% + +build_script: + - go run run_integration_tests.go diff --git a/backend/local/local.go b/backend/local/local.go index 6f5d9d976..8ba369ca3 100644 --- a/backend/local/local.go +++ b/backend/local/local.go @@ -267,6 +267,12 @@ func (b *Local) Remove(t backend.Type, name string) error { b.open[fn] = nil b.mu.Unlock() + // reset read-only flag + err := os.Chmod(fn, 0666) + if err != nil { + return err + } + return os.Remove(fn) } diff --git a/build.go b/build.go index 1e62384bc..05c930ae3 100644 --- a/build.go +++ b/build.go @@ -18,6 +18,7 @@ import ( var ( verbose bool keepGopath bool + runTests bool ) const timeFormat = "2006-01-02 15:04:05" @@ -32,6 +33,21 @@ func specialDir(name string) bool { return base[0] == '_' || base[0] == '.' } +// excludePath returns true if the file should not be copied to the new GOPATH. +func excludePath(name string) bool { + ext := path.Ext(name) + if ext == ".go" || ext == ".s" { + return false + } + + parentDir := filepath.Base(filepath.Dir(name)) + if parentDir == "testdata" { + return false + } + + return true +} + // updateGopath builds a valid GOPATH at dst, with all Go files in src/ copied // to dst/prefix/, so calling // @@ -60,8 +76,7 @@ func updateGopath(dst, src, prefix string) error { return nil } - ext := path.Ext(name) - if ext != ".go" && ext != ".s" { + if excludePath(name) { return nil } @@ -133,7 +148,10 @@ func showUsage(output io.Writer) { fmt.Fprintf(output, "USAGE: go run build.go OPTIONS\n") fmt.Fprintf(output, "\n") fmt.Fprintf(output, "OPTIONS:\n") - fmt.Fprintf(output, " -v --verbose output more messages\n") + fmt.Fprintf(output, " -v --verbose output more messages\n") + fmt.Fprintf(output, " -t --tags specify additional build tags\n") + fmt.Fprintf(output, " -k --keep-gopath do not remove the GOPATH after build\n") + fmt.Fprintf(output, " -T --test run tests\n") } func verbosePrintf(message string, args ...interface{}) { @@ -170,6 +188,18 @@ func build(gopath string, args ...string) error { return cmd.Run() } +// test runs "go test args..." with GOPATH set to gopath. +func test(gopath string, args ...string) error { + args = append([]string{"test"}, args...) + cmd := exec.Command("go", args...) + cmd.Env = append(cleanEnv(), "GOPATH="+gopath) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + verbosePrintf("go %s\n", args) + + return cmd.Run() +} + // getVersion returns a version string, either from the file VERSION in the // current directory or from git. func getVersion() string { @@ -218,6 +248,8 @@ func main() { case "-t", "-tags", "--tags": skipNext = true buildTags = strings.Split(params[i+1], " ") + case "-T", "--test": + runTests = true case "-h": showUsage(os.Stdout) default: @@ -258,6 +290,17 @@ func main() { die("copying files from %v to %v failed: %v\n", root, gopath, err) } + defer func() { + if !keepGopath { + verbosePrintf("remove %v\n", gopath) + if err = os.RemoveAll(gopath); err != nil { + die("remove GOPATH at %s failed: %v\n", err) + } + } else { + fmt.Printf("leaving temporary GOPATH at %v\n", gopath) + } + }() + output := "restic" if runtime.GOOS == "windows" { output = "restic.exe" @@ -276,15 +319,15 @@ func main() { err = build(gopath, args...) if err != nil { - fmt.Fprintf(os.Stderr, "build failed: %v\n", err) + die("build failed: %v\n", err) } - if !keepGopath { - verbosePrintf("remove %v\n", gopath) - if err = os.RemoveAll(gopath); err != nil { - die("remove GOPATH at %s failed: %v\n", err) + if runTests { + verbosePrintf("running tests\n") + + err = test(gopath, "github.com/restic/restic/...") + if err != nil { + die("running tests failed: %v\n", err) } - } else { - fmt.Printf("leaving temporary GOPATH at %v\n", gopath) } } diff --git a/cache.go b/cache.go index 8dfd9ff84..bf3fc2d05 100644 --- a/cache.go +++ b/cache.go @@ -235,6 +235,8 @@ func getWindowsCacheDir() (string, error) { if err != nil { return "", err } + + return cachedir, nil } if err != nil { diff --git a/lock.go b/lock.go index 6d65637c1..c305e19e2 100644 --- a/lock.go +++ b/lock.go @@ -207,17 +207,10 @@ func (l *Lock) Stale() bool { return false } - proc, err := os.FindProcess(l.PID) - defer proc.Release() - if err != nil { - debug.Log("Lock.Stale", "error searching for process %d: %v\n", l.PID, err) - return true - } - - debug.Log("Lock.Stale", "sending SIGHUP to process %d\n", l.PID) - err = proc.Signal(syscall.SIGHUP) - if err != nil { - debug.Log("Lock.Stale", "signal error: %v, lock is probably stale\n", err) + // check if we can reach the process retaining the lock + exists := l.processExists() + if !exists { + debug.Log("Lock.Stale", "could not reach process, %d, lock is probably stale\n", l.PID) return true } diff --git a/lock_test.go b/lock_test.go index 8bac2362b..6890c44da 100644 --- a/lock_test.go +++ b/lock_test.go @@ -134,7 +134,7 @@ var staleLockTests = []struct { timestamp: time.Now(), stale: true, staleOnOtherHost: false, - pid: os.Getpid() + 500, + pid: os.Getpid() + 500000, }, } @@ -204,7 +204,7 @@ func TestRemoveAllLocks(t *testing.T) { id2, err := createFakeLock(repo, time.Now().Add(-time.Minute), os.Getpid()) OK(t, err) - id3, err := createFakeLock(repo, time.Now().Add(-time.Minute), os.Getpid()+500) + id3, err := createFakeLock(repo, time.Now().Add(-time.Minute), os.Getpid()+500000) OK(t, err) OK(t, restic.RemoveAllLocks(repo)) diff --git a/run_integration_tests.go b/run_integration_tests.go new file mode 100644 index 000000000..d9a4a6cd7 --- /dev/null +++ b/run_integration_tests.go @@ -0,0 +1,186 @@ +// +build ignore + +package main + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" +) + +type CIEnvironment interface { + Prepare() + RunTests() +} + +type TravisEnvironment struct { + goxArch []string + goxOS []string +} + +func (env *TravisEnvironment) Prepare() { + msg("preparing environment for Travis CI\n") + + run("go", "get", "github.com/mattn/goveralls") + run("go", "get", "github.com/mitchellh/gox") + + if runtime.GOOS == "darwin" { + // install the libraries necessary for fuse + run("brew", "install", "caskroom/cask/brew-cask") + run("brew", "cask", "install", "osxfuse") + } + + // only test cross compilation on linux with Travis + if runtime.GOOS == "linux" { + env.goxArch = []string{"386", "amd64"} + if !strings.HasPrefix(runtime.Version(), "go1.3") { + env.goxArch = append(env.goxArch, "arm") + } + + env.goxOS = []string{"linux", "darwin", "freebsd", "openbsd", "windows"} + } else { + env.goxArch = []string{runtime.GOARCH} + env.goxOS = []string{runtime.GOOS} + } + + msg("gox: OS %v, ARCH %v\n", env.goxOS, env.goxArch) + run("gox", "-build-toolchain", + "-os", strings.Join(env.goxOS, " "), + "-arch", strings.Join(env.goxArch, " ")) +} + +func (env *TravisEnvironment) RunTests() { + // run fuse tests on darwin + if runtime.GOOS != "darwin" { + msg("skip fuse integration tests on %v\n", runtime.GOOS) + os.Setenv("RESTIC_TEST_FUSE", "0") + } + + // compile for all target architectures with tags + for _, tags := range []string{"release", "debug"} { + run("gox", "-verbose", + "-os", strings.Join(env.goxOS, " "), + "-arch", strings.Join(env.goxArch, " "), + "-tags", tags, + "./cmd/restic") + } + + // run the build script + run("go", "run", "build.go") + + // gather coverage information + run("go", "run", "run_tests.go", "all.cov") + + runGofmt() +} + +type AppveyorEnvironment struct{} + +func (env *AppveyorEnvironment) Prepare() { + msg("preparing environment for Appveyor CI\n") +} + +func (env *AppveyorEnvironment) RunTests() { + run("go", "run", "build.go", "-v", "-T") +} + +// findGoFiles returns a list of go source code file names below dir. +func findGoFiles(dir string) (list []string, err error) { + err = filepath.Walk(dir, func(name string, fi os.FileInfo, err error) error { + if filepath.Base(name) == "Godeps" { + return filepath.SkipDir + } + + if filepath.Ext(name) == ".go" { + relpath, err := filepath.Rel(dir, name) + if err != nil { + return err + } + + list = append(list, relpath) + } + + return err + }) + + return list, err +} + +func msg(format string, args ...interface{}) { + fmt.Printf("CI: "+format, args...) +} + +func runGofmt() { + dir, err := os.Getwd() + if err != nil { + fmt.Fprintf(os.Stderr, "Getwd(): %v\n", err) + os.Exit(5) + } + + files, err := findGoFiles(dir) + if err != nil { + fmt.Fprintf(os.Stderr, "error finding Go files: %v\n", err) + os.Exit(4) + } + + msg("runGofmt() with %d files\n", len(files)) + args := append([]string{"-l"}, files...) + cmd := exec.Command("gofmt", args...) + cmd.Stderr = os.Stderr + + buf, err := cmd.Output() + if err != nil { + fmt.Fprintf(os.Stderr, "error running gofmt: %v", err) + fmt.Fprintf(os.Stderr, "output:\n%s\n", buf) + os.Exit(3) + } + + if len(buf) > 0 { + fmt.Fprintf(os.Stderr, "not formatted with `gofmt`:\n") + fmt.Fprintln(os.Stderr, string(buf)) + os.Exit(6) + } +} + +func run(command string, args ...string) { + msg("run %v %v\n", command, strings.Join(args, " ")) + cmd := exec.Command(command, args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err := cmd.Run() + + if err != nil { + fmt.Fprintf(os.Stderr, "error running %v %v: %v", + command, strings.Join(args, " "), err) + os.Exit(3) + } +} + +func isTravis() bool { + return os.Getenv("TRAVIS_BUILD_DIR") != "" +} + +func isAppveyor() bool { + return runtime.GOOS == "windows" +} + +func main() { + var env CIEnvironment + + switch { + case isTravis(): + env = &TravisEnvironment{} + case isAppveyor(): + env = &AppveyorEnvironment{} + default: + fmt.Fprintln(os.Stderr, "unknown CI environment") + os.Exit(1) + } + + for _, f := range []func(){env.Prepare, env.RunTests} { + f() + } +} diff --git a/test/helpers.go b/test/helpers.go index 1b09b7a0e..eab73ddfb 100644 --- a/test/helpers.go +++ b/test/helpers.go @@ -2,7 +2,10 @@ package test_helper import ( "bytes" + "compress/bzip2" + "compress/gzip" "fmt" + "io" "io/ioutil" "math/rand" "os" @@ -89,14 +92,28 @@ func RandomReader(seed, size int) *bytes.Reader { // SetupTarTestFixture extracts the tarFile to outputDir. func SetupTarTestFixture(t testing.TB, outputDir, tarFile string) { - f, err := os.Open(tarFile) - defer f.Close() + input, err := os.Open(tarFile) + defer input.Close() OK(t, err) - cmd := exec.Command("tar", "xzf", "-") + var rd io.Reader + switch filepath.Ext(tarFile) { + case ".gz": + r, err := gzip.NewReader(input) + OK(t, err) + + defer r.Close() + rd = r + case ".bzip2": + rd = bzip2.NewReader(input) + default: + rd = input + } + + cmd := exec.Command("tar", "xf", "-") cmd.Dir = outputDir - cmd.Stdin = f + cmd.Stdin = rd cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr