Merge pull request #268 from restic/cross-compile-windows

CI: Add Windows

.travis.yml

@@ -9,7 +9,7 @@ os:
   - linux
   - osx
 
-env: GOX_OS="linux darwin openbsd freebsd" GOX_ARCH="386 amd64 arm"
+env: GOX_OS="linux darwin openbsd freebsd windows" GOX_ARCH="386 amd64 arm"
 
 notifications:
   irc:
@@ -23,24 +23,10 @@ install:
   - go version
   - export GOBIN="$GOPATH/bin"
   - export PATH="$PATH:$GOBIN"
-  - export GOPATH="$GOPATH:${TRAVIS_BUILD_DIR}/Godeps/_workspace"
   - go env
-  - go get github.com/mattn/goveralls
-  - go get github.com/mitchellh/gox
-  - go version | grep -q "go1\.3" && export GOX_ARCH="386 amd64" || true
-  - go version | grep -q "darwin" && export GOX_OS="darwin" || true
-  - uname -s | grep -qi darwin && brew install caskroom/cask/brew-cask || true
-  - uname -s | grep -qi darwin && brew cask install osxfuse || true
-  - uname -s | grep -vqi darwin && export RESTIC_TEST_FUSE="0" || true
-  - echo "cross-compile for \"$GOX_OS\" on \"$GOX_ARCH\""
-  - gox -build-toolchain -os "$GOX_OS" -arch "$GOX_ARCH"
 
 script:
-  - gox -verbose -os "$GOX_OS" -arch "$GOX_ARCH" -tags "release" ./cmd/restic
-  - gox -verbose -os "$GOX_OS" -arch "$GOX_ARCH" -tags "debug" ./cmd/restic
-  - go run build.go
-  - go run run_tests.go all.cov
-  - GOARCH=386 RESTIC_TEST_INTEGRATION=0 go test ./...
+  - go run run_integration_tests.go
+
+after_success:
   - goveralls -coverprofile=all.cov -service=travis-ci -repotoken "$COVERALLS_TOKEN" || true
-  - gofmt -l *.go */*.go */*/*.go
-  - test -z "$(gofmt -l *.go */*.go */*/*.go)"

Makefile

@@ -1,22 +1,12 @@
 .PHONY: all clean test
 
-SOURCE=$(wildcard *.go) $(wildcard */*.go) $(wildcard */*/*.go)
-
-export GOPATH GOX_OS
-
 all: restic
 
 restic: $(SOURCE)
 	go run build.go
 
-restic.debug: $(SOURCE)
-	go run build.go -tags debug
-
 clean:
-	rm -rf restic restic.debug
+	rm -rf restic
 
 test: $(SOURCE)
 	go run run_tests.go /dev/null
-
-all.cov: $(SOURCE)
-	go run run_tests.go all.cov

README.md

@@ -1,5 +1,6 @@
 [![Stories in Ready](https://badge.waffle.io/restic/restic.png?label=ready&title=Ready)](https://waffle.io/restic/restic)
 [![Build Status](https://travis-ci.org/restic/restic.svg?branch=master)](https://travis-ci.org/restic/restic)
+[![Build status](https://ci.appveyor.com/api/projects/status/nuy4lfbgfbytw92q/branch/master?svg=true)](https://ci.appveyor.com/project/fd0/restic/branch/master)
 [![sourcegraph status](https://sourcegraph.com/api/repos/github.com/restic/restic/.badges/status.png)](https://sourcegraph.com/github.com/restic/restic)
 [![Coverage Status](https://coveralls.io/repos/restic/restic/badge.svg)](https://coveralls.io/r/restic/restic)
 

appveyor.yml

@@ -0,0 +1,14 @@
+clone_folder: c:\gopath\src\github.com\restic\restic
+
+environment:
+  GOPATH: c:\gopath;c:\gopath\src\github.com\restic\restic\Godeps\_workspace
+
+install:
+  - go version
+  - go env
+  - appveyor DownloadFile http://downloads.sourceforge.net/project/gnuwin32/tar/1.13-1/tar-1.13-1-bin.zip -FileName tar.zip
+  - 7z x tar.zip bin/tar.exe
+  - set PATH=bin/;%PATH%
+
+build_script:
+  - go run run_integration_tests.go

backend/local/local.go

@@ -267,6 +267,12 @@ func (b *Local) Remove(t backend.Type, name string) error {
 	b.open[fn] = nil
 	b.mu.Unlock()
 
+	// reset read-only flag
+	err := os.Chmod(fn, 0666)
+	if err != nil {
+		return err
+	}
+
 	return os.Remove(fn)
 }
 

build.go

@@ -18,6 +18,7 @@ import (
 var (
 	verbose    bool
 	keepGopath bool
+	runTests   bool
 )
 
 const timeFormat = "2006-01-02 15:04:05"
@@ -32,6 +33,21 @@ func specialDir(name string) bool {
 	return base[0] == '_' || base[0] == '.'
 }
 
+// excludePath returns true if the file should not be copied to the new GOPATH.
+func excludePath(name string) bool {
+	ext := path.Ext(name)
+	if ext == ".go" || ext == ".s" {
+		return false
+	}
+
+	parentDir := filepath.Base(filepath.Dir(name))
+	if parentDir == "testdata" {
+		return false
+	}
+
+	return true
+}
+
 // updateGopath builds a valid GOPATH at dst, with all Go files in src/ copied
 // to dst/prefix/, so calling
 //
@@ -60,8 +76,7 @@ func updateGopath(dst, src, prefix string) error {
 			return nil
 		}
 
-		ext := path.Ext(name)
-		if ext != ".go" && ext != ".s" {
+		if excludePath(name) {
 			return nil
 		}
 
@@ -134,6 +149,9 @@ func showUsage(output io.Writer) {
 	fmt.Fprintf(output, "\n")
 	fmt.Fprintf(output, "OPTIONS:\n")
 	fmt.Fprintf(output, "  -v     --verbose     output more messages\n")
+	fmt.Fprintf(output, "  -t     --tags        specify additional build tags\n")
+	fmt.Fprintf(output, "  -k     --keep-gopath do not remove the GOPATH after build\n")
+	fmt.Fprintf(output, "  -T     --test        run tests\n")
 }
 
 func verbosePrintf(message string, args ...interface{}) {
@@ -170,6 +188,18 @@ func build(gopath string, args ...string) error {
 	return cmd.Run()
 }
 
+// test runs "go test args..." with GOPATH set to gopath.
+func test(gopath string, args ...string) error {
+	args = append([]string{"test"}, args...)
+	cmd := exec.Command("go", args...)
+	cmd.Env = append(cleanEnv(), "GOPATH="+gopath)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	verbosePrintf("go %s\n", args)
+
+	return cmd.Run()
+}
+
 // getVersion returns a version string, either from the file VERSION in the
 // current directory or from git.
 func getVersion() string {
@@ -218,6 +248,8 @@ func main() {
 		case "-t", "-tags", "--tags":
 			skipNext = true
 			buildTags = strings.Split(params[i+1], " ")
+		case "-T", "--test":
+			runTests = true
 		case "-h":
 			showUsage(os.Stdout)
 		default:
@@ -258,6 +290,17 @@ func main() {
 		die("copying files from %v to %v failed: %v\n", root, gopath, err)
 	}
 
+	defer func() {
+		if !keepGopath {
+			verbosePrintf("remove %v\n", gopath)
+			if err = os.RemoveAll(gopath); err != nil {
+				die("remove GOPATH at %s failed: %v\n", err)
+			}
+		} else {
+			fmt.Printf("leaving temporary GOPATH at %v\n", gopath)
+		}
+	}()
+
 	output := "restic"
 	if runtime.GOOS == "windows" {
 		output = "restic.exe"
@@ -276,15 +319,15 @@ func main() {
 
 	err = build(gopath, args...)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "build failed: %v\n", err)
+		die("build failed: %v\n", err)
 	}
 
-	if !keepGopath {
-		verbosePrintf("remove %v\n", gopath)
-		if err = os.RemoveAll(gopath); err != nil {
-			die("remove GOPATH at %s failed: %v\n", err)
+	if runTests {
+		verbosePrintf("running tests\n")
+
+		err = test(gopath, "github.com/restic/restic/...")
+		if err != nil {
+			die("running tests failed: %v\n", err)
 		}
-	} else {
-		fmt.Printf("leaving temporary GOPATH at %v\n", gopath)
 	}
 }

cache.go

@@ -235,6 +235,8 @@ func getWindowsCacheDir() (string, error) {
 		if err != nil {
 			return "", err
 		}
+
+		return cachedir, nil
 	}
 
 	if err != nil {

lock.go

@@ -207,17 +207,10 @@ func (l *Lock) Stale() bool {
 		return false
 	}
 
-	proc, err := os.FindProcess(l.PID)
-	defer proc.Release()
-	if err != nil {
-		debug.Log("Lock.Stale", "error searching for process %d: %v\n", l.PID, err)
-		return true
-	}
-
-	debug.Log("Lock.Stale", "sending SIGHUP to process %d\n", l.PID)
-	err = proc.Signal(syscall.SIGHUP)
-	if err != nil {
-		debug.Log("Lock.Stale", "signal error: %v, lock is probably stale\n", err)
+	// check if we can reach the process retaining the lock
+	exists := l.processExists()
+	if !exists {
+		debug.Log("Lock.Stale", "could not reach process, %d, lock is probably stale\n", l.PID)
 		return true
 	}
 

lock_test.go

@@ -134,7 +134,7 @@ var staleLockTests = []struct {
 		timestamp:        time.Now(),
 		stale:            true,
 		staleOnOtherHost: false,
-		pid:              os.Getpid() + 500,
+		pid:              os.Getpid() + 500000,
 	},
 }
 
@@ -204,7 +204,7 @@ func TestRemoveAllLocks(t *testing.T) {
 	id2, err := createFakeLock(repo, time.Now().Add(-time.Minute), os.Getpid())
 	OK(t, err)
 
-	id3, err := createFakeLock(repo, time.Now().Add(-time.Minute), os.Getpid()+500)
+	id3, err := createFakeLock(repo, time.Now().Add(-time.Minute), os.Getpid()+500000)
 	OK(t, err)
 
 	OK(t, restic.RemoveAllLocks(repo))

run_integration_tests.go

@@ -0,0 +1,186 @@
+// +build ignore
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+)
+
+type CIEnvironment interface {
+	Prepare()
+	RunTests()
+}
+
+type TravisEnvironment struct {
+	goxArch []string
+	goxOS   []string
+}
+
+func (env *TravisEnvironment) Prepare() {
+	msg("preparing environment for Travis CI\n")
+
+	run("go", "get", "github.com/mattn/goveralls")
+	run("go", "get", "github.com/mitchellh/gox")
+
+	if runtime.GOOS == "darwin" {
+		// install the libraries necessary for fuse
+		run("brew", "install", "caskroom/cask/brew-cask")
+		run("brew", "cask", "install", "osxfuse")
+	}
+
+	// only test cross compilation on linux with Travis
+	if runtime.GOOS == "linux" {
+		env.goxArch = []string{"386", "amd64"}
+		if !strings.HasPrefix(runtime.Version(), "go1.3") {
+			env.goxArch = append(env.goxArch, "arm")
+		}
+
+		env.goxOS = []string{"linux", "darwin", "freebsd", "openbsd", "windows"}
+	} else {
+		env.goxArch = []string{runtime.GOARCH}
+		env.goxOS = []string{runtime.GOOS}
+	}
+
+	msg("gox: OS %v, ARCH %v\n", env.goxOS, env.goxArch)
+	run("gox", "-build-toolchain",
+		"-os", strings.Join(env.goxOS, " "),
+		"-arch", strings.Join(env.goxArch, " "))
+}
+
+func (env *TravisEnvironment) RunTests() {
+	// run fuse tests on darwin
+	if runtime.GOOS != "darwin" {
+		msg("skip fuse integration tests on %v\n", runtime.GOOS)
+		os.Setenv("RESTIC_TEST_FUSE", "0")
+	}
+
+	// compile for all target architectures with tags
+	for _, tags := range []string{"release", "debug"} {
+		run("gox", "-verbose",
+			"-os", strings.Join(env.goxOS, " "),
+			"-arch", strings.Join(env.goxArch, " "),
+			"-tags", tags,
+			"./cmd/restic")
+	}
+
+	// run the build script
+	run("go", "run", "build.go")
+
+	// gather coverage information
+	run("go", "run", "run_tests.go", "all.cov")
+
+	runGofmt()
+}
+
+type AppveyorEnvironment struct{}
+
+func (env *AppveyorEnvironment) Prepare() {
+	msg("preparing environment for Appveyor CI\n")
+}
+
+func (env *AppveyorEnvironment) RunTests() {
+	run("go", "run", "build.go", "-v", "-T")
+}
+
+// findGoFiles returns a list of go source code file names below dir.
+func findGoFiles(dir string) (list []string, err error) {
+	err = filepath.Walk(dir, func(name string, fi os.FileInfo, err error) error {
+		if filepath.Base(name) == "Godeps" {
+			return filepath.SkipDir
+		}
+
+		if filepath.Ext(name) == ".go" {
+			relpath, err := filepath.Rel(dir, name)
+			if err != nil {
+				return err
+			}
+
+			list = append(list, relpath)
+		}
+
+		return err
+	})
+
+	return list, err
+}
+
+func msg(format string, args ...interface{}) {
+	fmt.Printf("CI: "+format, args...)
+}
+
+func runGofmt() {
+	dir, err := os.Getwd()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Getwd(): %v\n", err)
+		os.Exit(5)
+	}
+
+	files, err := findGoFiles(dir)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error finding Go files: %v\n", err)
+		os.Exit(4)
+	}
+
+	msg("runGofmt() with %d files\n", len(files))
+	args := append([]string{"-l"}, files...)
+	cmd := exec.Command("gofmt", args...)
+	cmd.Stderr = os.Stderr
+
+	buf, err := cmd.Output()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error running gofmt: %v", err)
+		fmt.Fprintf(os.Stderr, "output:\n%s\n", buf)
+		os.Exit(3)
+	}
+
+	if len(buf) > 0 {
+		fmt.Fprintf(os.Stderr, "not formatted with `gofmt`:\n")
+		fmt.Fprintln(os.Stderr, string(buf))
+		os.Exit(6)
+	}
+}
+
+func run(command string, args ...string) {
+	msg("run %v %v\n", command, strings.Join(args, " "))
+	cmd := exec.Command(command, args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err := cmd.Run()
+
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error running %v %v: %v",
+			command, strings.Join(args, " "), err)
+		os.Exit(3)
+	}
+}
+
+func isTravis() bool {
+	return os.Getenv("TRAVIS_BUILD_DIR") != ""
+}
+
+func isAppveyor() bool {
+	return runtime.GOOS == "windows"
+}
+
+func main() {
+	var env CIEnvironment
+
+	switch {
+	case isTravis():
+		env = &TravisEnvironment{}
+	case isAppveyor():
+		env = &AppveyorEnvironment{}
+	default:
+		fmt.Fprintln(os.Stderr, "unknown CI environment")
+		os.Exit(1)
+	}
+
+	for _, f := range []func(){env.Prepare, env.RunTests} {
+		f()
+	}
+}

test/helpers.go

@@ -2,7 +2,10 @@ package test_helper
 
 import (
 	"bytes"
+	"compress/bzip2"
+	"compress/gzip"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"math/rand"
 	"os"
@@ -89,14 +92,28 @@ func RandomReader(seed, size int) *bytes.Reader {
 
 // SetupTarTestFixture extracts the tarFile to outputDir.
 func SetupTarTestFixture(t testing.TB, outputDir, tarFile string) {
-	f, err := os.Open(tarFile)
-	defer f.Close()
+	input, err := os.Open(tarFile)
+	defer input.Close()
 	OK(t, err)
 
-	cmd := exec.Command("tar", "xzf", "-")
+	var rd io.Reader
+	switch filepath.Ext(tarFile) {
+	case ".gz":
+		r, err := gzip.NewReader(input)
+		OK(t, err)
+
+		defer r.Close()
+		rd = r
+	case ".bzip2":
+		rd = bzip2.NewReader(input)
+	default:
+		rd = input
+	}
+
+	cmd := exec.Command("tar", "xf", "-")
 	cmd.Dir = outputDir
 
-	cmd.Stdin = f
+	cmd.Stdin = rd
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr