octoleo/syncthing
1
0
Fork 0
mirror of https://github.com/octoleo/syncthing.git synced 2024-11-09 14:50:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactor and improve integration tests

Browse Source
This commit is contained in:
Jakob Borg 2014-07-11 11:31:16 +02:00
parent 620eeae4a7
commit f510f5f205
7 changed files with 232 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
integration/all.sh Executable file
View File

@ -0,0 +1,6 @@
#!/bin/sh
./test-http.sh || exit
./test-merge.sh || exit
./test-delupd.sh || exit
./test-folders.sh || exit

12
integration/f2/config.xml
View File

@ -1,14 +1,16 @@
<configuration version="2">
<repository id="default" directory="s2" ro="false" ignorePerms="false">
<node id="I6KAH7666SLLL5PFXSOAUFJCDZYAOMLEKCP2GB3BV5RQST3PSROA"></node>
<node id="JMFJCXBGZDE4BOCJE3VF65GYZNAIVJRET3J6HMRAUQIGJOFKNHMQ"></node>
<versioning></versioning>
<node id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU"></node>
<node id="JMFJCXB-GZDE4BN-OCJE3VF-65GYZNU-AIVJRET-3J6HMRQ-AUQIGJO-FKNHMQU"></node>
<versioning type="simple">
<param key="keep" val="5"></param>
</versioning>
<syncorder></syncorder>
</repository>
<node id="I6KAH7666SLLL5PFXSOAUFJCDZYAOMLEKCP2GB3BV5RQST3PSROA" name="f1">
<node id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU" name="f1">
<address>127.0.0.1:22001</address>
</node>
<node id="JMFJCXBGZDE4BOCJE3VF65GYZNAIVJRET3J6HMRAUQIGJOFKNHMQ" name="f2">
<node id="JMFJCXB-GZDE4BN-OCJE3VF-65GYZNU-AIVJRET-3J6HMRQ-AUQIGJO-FKNHMQU" name="f2">
<address>127.0.0.1:22002</address>
</node>
<gui enabled="true" tls="false">

10
integration/h3/config.xml
View File

@ -1,15 +1,17 @@
<configuration version="2">
<repository id="default" directory="s3" ro="false" ignorePerms="false">
<node id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU"></node>
<repository id="s23" directory="s23-3" ro="false" ignorePerms="false">
<node id="JMFJCXB-GZDE4BN-OCJE3VF-65GYZNU-AIVJRET-3J6HMRQ-AUQIGJO-FKNHMQU"></node>
<node id="373HSRP-QLPNLIE-JYKZVQF-P4PKZ63-R2ZE6K3-YD442U2-JHBGBQG-WWXAHAU"></node>
<versioning></versioning>
<syncorder></syncorder>
</repository>
<repository id="s23" directory="s23-3" ro="false" ignorePerms="false">
<repository id="default" directory="s3" ro="false" ignorePerms="false">
<node id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU"></node>
<node id="JMFJCXB-GZDE4BN-OCJE3VF-65GYZNU-AIVJRET-3J6HMRQ-AUQIGJO-FKNHMQU"></node>
<node id="373HSRP-QLPNLIE-JYKZVQF-P4PKZ63-R2ZE6K3-YD442U2-JHBGBQG-WWXAHAU"></node>
<versioning></versioning>
<versioning type="simple">
<param key="keep" val="5"></param>
</versioning>
<syncorder></syncorder>
</repository>
<node id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU" name="s1">

148
integration/test-delupd.sh Executable file
View File

@ -0,0 +1,148 @@
#!/bin/bash
# Copyright (C) 2014 Jakob Borg and other contributors. All rights reserved.
# Use of this source code is governed by an MIT-style license that can be
# found in the LICENSE file.
iterations=${1:-5}
id1=I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU
id2=JMFJCXB-GZDE4BN-OCJE3VF-65GYZNU-AIVJRET-3J6HMRQ-AUQIGJO-FKNHMQU
id3=373HSRP-QLPNLIE-JYKZVQF-P4PKZ63-R2ZE6K3-YD442U2-JHBGBQG-WWXAHAU
go build genfiles.go
go build md5r.go
go build json.go
start() {
echo "Starting..."
for i in 1 2 3 ; do
STTRACE=files,model,puller,versioner STPROFILER=":909$i" syncthing -home "h$i" > "$i.out" 2>&1 &
done
}
stop() {
for i in 1 2 3 ; do
curl -HX-API-Key:abc123 -X POST "http://localhost:808$i/rest/shutdown"
done
exit $1
}
testConvergence() {
while true ; do
sleep 5
s1comp=$(curl -HX-API-Key:abc123 -s "http://localhost:8082/rest/connections" | ./json "$id1/Completion")
s2comp=$(curl -HX-API-Key:abc123 -s "http://localhost:8083/rest/connections" | ./json "$id2/Completion")
s3comp=$(curl -HX-API-Key:abc123 -s "http://localhost:8081/rest/connections" | ./json "$id3/Completion")
s1comp=${s1comp:-0}
s2comp=${s2comp:-0}
s3comp=${s3comp:-0}
tot=$(($s1comp + $s2comp + $s3comp))
echo $tot / 300
if [[ $tot == 300 ]] ; then
break
fi
done
echo "Verifying..."
cp md5-1 md5-tot
cp md5-12-2 md5-12-tot
cp md5-23-3 md5-23-tot
for i in 1 2 3 12-1 12-2 23-2 23-3; do
pushd "s$i" >/dev/null
../md5r -l | sort | grep -v .stversions > ../md5-$i
popd >/dev/null
done
ok=0
for i in 1 2 3 ; do
if ! cmp "md5-$i" md5-tot >/dev/null ; then
echo "Fail: instance $i unconverged for default"
else
ok=$(($ok + 1))
echo "OK: instance $i converged for default"
fi
done
for i in 12-1 12-2 ; do
if ! cmp "md5-$i" md5-12-tot >/dev/null ; then
echo "Fail: instance $i unconverged for s12"
else
ok=$(($ok + 1))
echo "OK: instance $i converged for s12"
fi
done
for i in 23-2 23-3 ; do
if ! cmp "md5-$i" md5-23-tot >/dev/null ; then
echo "Fail: instance $i unconverged for s23"
else
ok=$(($ok + 1))
echo "OK: instance $i converged for s23"
fi
done
if [[ $ok != 7 ]] ; then
stop 1
fi
}
alterFiles() {
pkill -STOP syncthing
for i in 1 12-2 23-3 ; do
# Delete some files
pushd "s$i" >/dev/null
nfiles=$(find . -type f | wc -l)
if [[ $nfiles -ge 300 ]] ; then
todelete=$(( $nfiles - 300 ))
echo " $i: deleting $todelete files..."
find . -type f \
| grep -v large \
| sort -k 1.16 \
| head -n "$todelete" \
| xargs rm -f
fi
# Create some new files and alter existing ones
echo " $i: random nonoverlapping"
../genfiles -maxexp 22 -files 200
echo " $i: append to large file"
dd if=large-$i bs=1024k count=4 >> large-$i 2>/dev/null
../md5r -l | sort | grep -v .stversions > ../md5-$i
popd >/dev/null
done
pkill -CONT syncthing
}
rm -rf h?/*.idx.gz h?/index
rm -rf s? s??-?
mkdir s1 s2 s3 s12-1 s12-2 s23-2 s23-3
echo "Setting up files..."
for i in 1 12-2 23-3; do
pushd "s$i" >/dev/null
echo " $i: random nonoverlapping"
../genfiles -maxexp 22 -files 400
popd >/dev/null
done
echo "MD5-summing..."
for i in 1 12-2 23-3 ; do
pushd "s$i" >/dev/null
../md5r -l | sort > ../md5-$i
popd >/dev/null
done
start
testConvergence
for ((t = 1; t <= $iterations; t++)) ; do
echo "Add and remove random files ($t / $iterations)..."
alterFiles
echo "Waiting..."
sleep 30
testConvergence
done
stop 0

27
integration/folders.sh → integration/test-folders.sh
View File

@ -6,8 +6,8 @@
iterations=${1:-5}
id1=I6KAH7666SLLL5PFXSOAUFJCDZYAOMLEKCP2GB3BV5RQST3PSROA
id2=JMFJCXBGZDE4BOCJE3VF65GYZNAIVJRET3J6HMRAUQIGJOFKNHMQ
id1=I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU
id2=JMFJCXB-GZDE4BN-OCJE3VF-65GYZNU-AIVJRET-3J6HMRQ-AUQIGJO-FKNHMQU
go build json.go
@ -15,6 +15,7 @@ start() {
echo "Starting..."
STTRACE=model,scanner STPROFILER=":9091" syncthing -home "f1" > 1.out 2>&1 &
STTRACE=model,scanner STPROFILER=":9092" syncthing -home "f2" > 2.out 2>&1 &
sleep 1
}
stop() {
@ -28,11 +29,17 @@ setup() {
echo "Setting up dirs..."
mkdir -p s1
pushd s1 >/dev/null
rmdir */*[02468] 2>/dev/null
rm -r */*[02468] 2>/dev/null
rm -rf *2
for ((i = 0; i < 1000; i++)) ; do
mkdir -p $RANDOM/$RANDOM
for ((i = 0; i < 500; i++)) ; do
mkdir -p "$RANDOM/$RANDOM"
done
for ((i = 0; i < 500; i++)) ; do
d="$RANDOM/$RANDOM"
mkdir -p "$d"
touch "$d/foo"
done
../md5r -d | grep -v ' . ' > ../dirs-1
popd >/dev/null
}
@ -57,12 +64,8 @@ testConvergence() {
echo "Verifying..."
pushd s1 >/dev/null
../md5r -d | grep -v ' . ' > ../dirs-1
popd >/dev/null
pushd s2 >/dev/null
../md5r -d | grep -v ' . ' > ../dirs-2
../md5r -d | grep -v ' . ' | grep -v .stversions > ../dirs-2
popd >/dev/null
if ! cmp dirs-1 dirs-2 ; then
@ -73,12 +76,12 @@ testConvergence() {
}
rm -rf s? s??-?
rm -f f?/*.idx.gz
rm -rf f?/*.idx.gz f?/index
setup
start
for ((j = 0; j < 10; j++)) ; do
for ((j = 0; j < iterations; j++)) ; do
echo "#$j..."
testConvergence
setup

38
integration/test-http.sh Executable file
View File

@ -0,0 +1,38 @@
#!/bin/bash
# Copyright (C) 2014 Jakob Borg and other contributors. All rights reserved.
# Use of this source code is governed by an MIT-style license that can be
# found in the LICENSE file.
id1=I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU
id2=JMFJCXB-GZDE4BN-OCJE3VF-65GYZNU-AIVJRET-3J6HMRQ-AUQIGJO-FKNHMQU
id3=373HSRP-QLPNLIE-JYKZVQF-P4PKZ63-R2ZE6K3-YD442U2-JHBGBQG-WWXAHAU
stop() {
echo Stopping
curl -s -o/dev/null -HX-API-Key:abc123 -X POST http://localhost:8081/rest/shutdown
curl -s -o/dev/null -HX-API-Key:abc123 -X POST http://localhost:8082/rest/shutdown
exit $1
}
echo Building
go build http.go
echo Starting
rm -rf s1 s2 h1/index h2/index
syncthing -home h1 > 1.out 2>&1 &
syncthing -home h2 > 2.out 2>&1 &
sleep 1
echo Fetching CSRF tokens
curl -s -o /dev/null http://testuser:testpass@localhost:8081/index.html
curl -s -o /dev/null http://localhost:8082/index.html
sleep 1
echo Testing
./http -target localhost:8081 -user testuser -pass testpass -csrf h1/csrftokens.txt || stop 1
./http -target localhost:8081 -api abc123 || stop 1
./http -target localhost:8082 -csrf h2/csrftokens.txt || stop 1
./http -target localhost:8082 -api abc123 || stop 1
stop 0

42
integration/test.sh → integration/test-merge.sh
View File

@ -13,23 +13,12 @@ id3=373HSRP-QLPNLIE-JYKZVQF-P4PKZ63-R2ZE6K3-YD442U2-JHBGBQG-WWXAHAU
go build genfiles.go
go build md5r.go
go build json.go
go build http.go
start() {
echo "Starting..."
for i in 1 2 3 4 ; do
STTRACE=files,model,puller STPROFILER=":909$i" syncthing -home "h$i" > "$i.out" 2>&1 &
STTRACE=files,model,puller,versioner STPROFILER=":909$i" syncthing -home "h$i" > "$i.out" 2>&1 &
done
# Test REST API
sleep 2
curl -s -o /dev/null http://testuser:testpass@localhost:8081/index.html
curl -s -o /dev/null http://localhost:8082/index.html
sleep 1
./http -target localhost:8081 -user testuser -pass testpass -csrf h1/csrftokens.txt || stop 1
./http -target localhost:8081 -api abc123 || stop 1
./http -target localhost:8082 -csrf h2/csrftokens.txt || stop 1
./http -target localhost:8082 -api abc123 || stop 1
}
stop() {
@ -41,9 +30,9 @@ stop() {
clean() {
if [[ $(uname -s) == "Linux" ]] ; then
grep -v utf8-nfd
grep -v .stversions | grep -v utf8-nfd
else
cat
grep -v .stversions
fi
}
@ -107,27 +96,20 @@ testConvergence() {
alterFiles() {
pkill -STOP syncthing
# Create some new files and alter existing ones
for i in 1 2 3 12-1 12-2 23-2 23-3 ; do
pushd "s$i" >/dev/null
nfiles=$(find . -type f | wc -l)
if [[ $nfiles > 2000 ]] ; then
todelete=$(( $nfiles - 2000 ))
echo "Deleting $todelete files..."
find . -type f \
| grep -v large \
| sort -k 1.16 \
| head -n "$todelete" \
| xargs rm -f
fi
../genfiles -maxexp 22 -files 600
echo " $i: random nonoverlapping"
../genfiles -maxexp 22 -files 200
echo " $i: append to large file"
dd if=/dev/urandom bs=1024k count=4 >> large-$i 2>/dev/null
dd if=large-$i bs=1024k count=4 >> large-$i 2>/dev/null
../md5r -l > ../md5-tmp
(grep -v large ../md5-tmp ; grep "large-$i" ../md5-tmp) | grep -v '/.syncthing.' > ../md5-$i
popd >/dev/null
done
pkill -CONT syncthing
}
@ -139,11 +121,11 @@ for i in 1 2 3 12-1 12-2 23-2 23-3; do
mkdir "s$i"
pushd "s$i" >/dev/null
echo " $i: random nonoverlapping"
../genfiles -maxexp 22 -files 600
../genfiles -maxexp 22 -files 200
echo " $i: empty file"
touch "empty-$i"
echo " $i: large file"
dd if=/dev/urandom of=large-$i bs=1024k count=55 2>/dev/null
dd if=/dev/urandom of=large-$i bs=1024k count=15 2>/dev/null
echo " $i: weird encodings"
echo somedata > "$(echo -e utf8-nfc-\\xc3\\xad)-$i"
echo somedata > "$(echo -e utf8-nfd-i\\xcc\\x81)-$i"
@ -166,7 +148,7 @@ start
testConvergence
for ((t = 1; t <= $iterations; t++)) ; do
echo "Add and remove random files ($t / $iterations)..."
echo "Add and alter random files ($t / $iterations)..."
alterFiles
echo "Waiting..."