diff --git a/integration/all.sh b/integration/all.sh index 12ef6eb1e..5d4b91478 100755 --- a/integration/all.sh +++ b/integration/all.sh @@ -3,5 +3,3 @@ ./test-http.sh || exit ./test-merge.sh || exit ./test-delupd.sh || exit -# ./test-folders.sh || exit -./test-reconnect.sh || exit diff --git a/integration/common_test.go b/integration/common_test.go new file mode 100644 index 000000000..d5b8318ea --- /dev/null +++ b/integration/common_test.go @@ -0,0 +1,262 @@ +// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// All rights reserved. Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +package integration_test + +import ( + "crypto/md5" + "crypto/rand" + "encoding/json" + "errors" + "fmt" + "io" + "log" + mr "math/rand" + "net/http" + "os" + "os/exec" + "path/filepath" + "time" +) + +type syncthingProcess struct { + log string + argv []string + port int + + cmd *exec.Cmd + logfd *os.File +} + +func (p *syncthingProcess) start() error { + if p.logfd == nil { + logfd, err := os.Create(p.log) + if err != nil { + return err + } + p.logfd = logfd + } + + cmd := exec.Command("../bin/syncthing", p.argv...) + cmd.Stdout = p.logfd + cmd.Stderr = p.logfd + cmd.Env = append(env, fmt.Sprintf("STPROFILER=:%d", p.port+1000)) + + err := cmd.Start() + if err != nil { + return err + } + p.cmd = cmd + return nil +} + +func (p *syncthingProcess) stop() { + p.cmd.Process.Kill() + p.cmd.Wait() +} + +func (p *syncthingProcess) peerCompletion() (map[string]int, error) { + resp, err := http.Get(fmt.Sprintf("http://localhost:%d/rest/debug/peerCompletion", p.port)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + comp := map[string]int{} + err = json.NewDecoder(resp.Body).Decode(&comp) + return comp, err +} + +type fileGenerator struct { + files int + maxexp int + srcname string +} + +func generateFiles(dir string, files, maxexp int, srcname string) error { + fd, err := os.Open(srcname) + if err != nil { + return err + } + + for i := 0; i < files; i++ { + n := randomName() + p0 := filepath.Join(dir, string(n[0]), n[0:2]) + err = os.MkdirAll(p0, 0755) + if err != nil { + log.Fatal(err) + } + + s := 1 << uint(mr.Intn(maxexp)) + a := 128 * 1024 + if a > s { + a = s + } + s += mr.Intn(a) + + src := io.LimitReader(&inifiteReader{fd}, int64(s)) + + p1 := filepath.Join(p0, n) + dst, err := os.Create(p1) + if err != nil { + return err + } + + _, err = io.Copy(dst, src) + if err != nil { + return err + } + + err = dst.Close() + if err != nil { + return err + } + + err = os.Chmod(p1, os.FileMode(mr.Intn(0777)|0400)) + if err != nil { + return err + } + + t := time.Now().Add(-time.Duration(mr.Intn(30*86400)) * time.Second) + err = os.Chtimes(p1, t, t) + if err != nil { + return err + } + } + + return nil +} + +func randomName() string { + var b [16]byte + rand.Reader.Read(b[:]) + return fmt.Sprintf("%x", b[:]) +} + +type inifiteReader struct { + rd io.ReadSeeker +} + +func (i *inifiteReader) Read(bs []byte) (int, error) { + n, err := i.rd.Read(bs) + if err == io.EOF { + err = nil + i.rd.Seek(0, 0) + } + return n, err +} + +// rm -rf +func removeAll(dirs ...string) error { + for _, dir := range dirs { + err := os.RemoveAll(dir) + if err != nil { + return err + } + } + return nil +} + +// Compare a number of directories. Returns nil if the contents are identical, +// otherwise an error describing the first found difference. +func compareDirectories(dirs ...string) error { + chans := make([]chan fileInfo, len(dirs)) + for i := range chans { + chans[i] = make(chan fileInfo) + } + abort := make(chan struct{}) + + for i := range dirs { + startWalker(dirs[i], chans[i], abort) + } + + res := make([]fileInfo, len(dirs)) + for { + numDone := 0 + for i := range chans { + fi, ok := <-chans[i] + if !ok { + numDone++ + } + res[i] = fi + } + + for i := 1; i < len(res); i++ { + if res[i] != res[0] { + close(abort) + return fmt.Errorf("Mismatch; %#v (%s) != %#v (%s)", res[i], dirs[i], res[0], dirs[0]) + } + } + + if numDone == len(dirs) { + return nil + } + } +} + +type fileInfo struct { + name string + mode os.FileMode + mod time.Time + hash [16]byte +} + +func startWalker(dir string, res chan<- fileInfo, abort <-chan struct{}) { + walker := func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + rn, _ := filepath.Rel(dir, path) + if rn == "." { + return nil + } + + var f fileInfo + if info.IsDir() { + f = fileInfo{ + name: rn, + mode: info.Mode(), + // hash and modtime zero for directories + } + } else { + f = fileInfo{ + name: rn, + mode: info.Mode(), + mod: info.ModTime(), + } + sum, err := md5file(path) + if err != nil { + return err + } + f.hash = sum + } + + select { + case res <- f: + return nil + case <-abort: + return errors.New("abort") + } + } + go func() { + filepath.Walk(dir, walker) + close(res) + }() +} + +func md5file(fname string) (hash [16]byte, err error) { + f, err := os.Open(fname) + if err != nil { + return + } + defer f.Close() + + h := md5.New() + io.Copy(h, f) + hb := h.Sum(nil) + copy(hash[:], hb) + + return +} diff --git a/integration/f1/config.xml b/integration/f1/config.xml index 111c1607f..5a3fc13c5 100644 --- a/integration/f1/config.xml +++ b/integration/f1/config.xml @@ -26,7 +26,7 @@ 5 10000 false - true + false -1 diff --git a/integration/f2/config.xml b/integration/f2/config.xml index fa47a3c39..010a09cb5 100644 --- a/integration/f2/config.xml +++ b/integration/f2/config.xml @@ -28,7 +28,7 @@ 5 10000 false - true + false -1 diff --git a/integration/reconnect_test.go b/integration/reconnect_test.go new file mode 100644 index 000000000..56e92973c --- /dev/null +++ b/integration/reconnect_test.go @@ -0,0 +1,144 @@ +// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// All rights reserved. Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +package integration_test + +import ( + "sync" + "testing" + "time" +) + +const ( + apiKey = "abc123" // Used when talking to the processes under test + id1 = "I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU" + id2 = "JMFJCXB-GZDE4BN-OCJE3VF-65GYZNU-AIVJRET-3J6HMRQ-AUQIGJO-FKNHMQU" +) + +var env = []string{ + "HOME=.", + "STTRACE=model", +} + +func TestRestartBothDuringTransfer(t *testing.T) { + // Give the receiver some time to rot with needed files but + // without any peer. This triggers + // https://github.com/syncthing/syncthing/issues/463 + testRestartDuringTransfer(t, true, true, 10*time.Second, 0) +} + +func TestRestartReceiverDuringTransfer(t *testing.T) { + testRestartDuringTransfer(t, false, true, 0, 0) +} + +func TestRestartSenderDuringTransfer(t *testing.T) { + testRestartDuringTransfer(t, true, false, 0, 0) +} + +func testRestartDuringTransfer(t *testing.T, restartSender, restartReceiver bool, senderDelay, receiverDelay time.Duration) { + if testing.Short() { + t.Skip("skipping integration test") + return + } + + t.Log("Cleaning...") + err := removeAll("s1", "s2", "f1/index", "f2/index") + if err != nil { + t.Fatal(err) + } + + t.Log("Generating files...") + err = generateFiles("s1", 1000, 20, "../bin/syncthing") + if err != nil { + t.Fatal(err) + } + + t.Log("Starting up...") + sender := syncthingProcess{ // id1 + log: "1.out", + argv: []string{"-home", "f1"}, + port: 8081, + } + err = sender.start() + if err != nil { + t.Fatal(err) + } + + receiver := syncthingProcess{ // id2 + log: "2.out", + argv: []string{"-home", "f2"}, + port: 8082, + } + err = receiver.start() + if err != nil { + t.Fatal(err) + } + + // Give them time to start up + time.Sleep(1 * time.Second) + + var prevComp int + for { + comp, err := sender.peerCompletion() + if err != nil { + sender.stop() + receiver.stop() + t.Fatal(err) + } + + curComp := comp[id2] + + if curComp == 100 { + sender.stop() + receiver.stop() + break + } + + if curComp > prevComp { + if restartReceiver { + t.Logf("Stopping receiver...") + receiver.stop() + } + + if restartSender { + t.Logf("Stopping sender...") + sender.stop() + } + + var wg sync.WaitGroup + + if restartReceiver { + wg.Add(1) + go func() { + time.Sleep(receiverDelay) + t.Logf("Starting receiver...") + receiver.start() + wg.Done() + }() + } + + if restartSender { + wg.Add(1) + go func() { + time.Sleep(senderDelay) + t.Logf("Starting sender...") + sender.start() + wg.Done() + }() + } + + wg.Wait() + + prevComp = curComp + } + + time.Sleep(1 * time.Second) + } + + t.Log("Comparing directories...") + err = compareDirectories("s1", "s2") + if err != nil { + t.Fatal(err) + } +} diff --git a/integration/test-reconnect.sh b/integration/test-reconnect.sh deleted file mode 100755 index 3d4ac0076..000000000 --- a/integration/test-reconnect.sh +++ /dev/null @@ -1,87 +0,0 @@ -#!/bin/bash - -# Copyright (C) 2014 Jakob Borg and other contributors. All rights reserved. -# Use of this source code is governed by an MIT-style license that can be -# found in the LICENSE file. - -id1=I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU -id2=JMFJCXB-GZDE4BN-OCJE3VF-65GYZNU-AIVJRET-3J6HMRQ-AUQIGJO-FKNHMQU - -go build json.go -go build md5r.go -go build genfiles.go - -start() { - echo "Starting..." - STTRACE=model,scanner STPROFILER=":9091" syncthing -home "f1" > 1.out 2>&1 & - STTRACE=model,scanner STPROFILER=":9092" syncthing -home "f2" > 2.out 2>&1 & - sleep 1 -} - -stop() { - echo "Stopping..." - for i in 1 2 ; do - curl -HX-API-Key:abc123 -X POST "http://localhost:808$i/rest/shutdown" - done - sleep 1 -} - -setup() { - echo "Setting up..." - rm -rf s? s??-? - rm -rf f?/*.idx.gz f?/index - mkdir -p s1 - pushd s1 >/dev/null - ../genfiles - ../md5r > ../md5-1 - popd >/dev/null -} - -testConvergence() { - torestart="$1" - prevcomp=0 - - while true ; do - sleep 5 - comp=$(curl -HX-API-Key:abc123 -s "http://localhost:8081/rest/debug/peerCompletion" | ./json "$id2") - comp=${comp:-0} - echo $comp / 100 - - if [[ $comp == 100 ]] ; then - echo Done - break - fi - - # Restart if the destination has made some progress - if [[ $comp -gt $prevcomp ]] ; then - prevcomp=$comp - curl -HX-API-Key:abc123 -X POST "http://localhost:$torestart/rest/restart" - fi - done - - echo "Verifying..." - - pushd s2 >/dev/null - ../md5r | grep -v .stversions > ../md5-2 - popd >/dev/null - - if ! cmp md5-1 md5-2 ; then - echo Repos differ - stop - exit 1 - fi -} - -echo Testing reconnects during pull where the source node restarts -setup -start -testConvergence 8081 -stop - -echo Testing reconnects during pull where the destination node restarts -setup -start -testConvergence 8082 -stop - -exit 0