mirror of
https://github.com/octoleo/syncthing.git
synced 2024-09-10 17:09:01 +00:00
REBASE!
This commit is contained in:
commit
bfe935b5ab
1
.gitignore
vendored
Normal file
1
.gitignore
vendored
Normal file
@ -0,0 +1 @@
|
||||
syncthing
|
19
LICENSE
Normal file
19
LICENSE
Normal file
@ -0,0 +1,19 @@
|
||||
Copyright (C) 2013 Jakob Borg
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
this software and associated documentation files (the "Software"), to deal in
|
||||
the Software without restriction, including without limitation the rights to
|
||||
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
|
||||
of the Software, and to permit persons to whom the Software is furnished to do
|
||||
so, subject to the following conditions:
|
||||
|
||||
- The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
161
README.md
Normal file
161
README.md
Normal file
@ -0,0 +1,161 @@
|
||||
syncthing
|
||||
=========
|
||||
|
||||
This is `syncthing`, an open BitTorrent Sync alternative. It is
|
||||
currently far from ready for mass consumption, but it is a usable proof
|
||||
of concept and tech demo. The following are the project goals:
|
||||
|
||||
1. Define an open, secure, language neutral protocol usable for
|
||||
efficient synchronization of a file repository between an arbitrary
|
||||
number of nodes. This is the [Block Exchange
|
||||
Protocol](https://github.com/calmh/syncthing/blob/master/protocol/PROTOCOL.md)
|
||||
(BEP).
|
||||
|
||||
2. Provide the reference implementation to demonstrate the usability of
|
||||
said protocol. This is the `syncthing` utility.
|
||||
|
||||
The two are evolving together; the protocol is not to be considered
|
||||
stable until syncthing 1.0 is released, at which point it is locked down
|
||||
for incompatible changes.
|
||||
|
||||
Syncthing does not use the BitTorrent protocol. The reasons for this are
|
||||
1) we don't know if BitTorrent Sync does either, so there's nothing to
|
||||
be compatible with, 2) BitTorrent includes a lot of functionality for
|
||||
making sure large swarms of selfish agents behave and somehow work
|
||||
towards a common goal. Here we have a much smaller swarm of cooperative
|
||||
agents and a simpler approach will suffice.
|
||||
|
||||
Features
|
||||
--------
|
||||
|
||||
The following features are _currently implemented and working_:
|
||||
|
||||
* The formation of a cluster of nodes, certificate authenticated and
|
||||
communicating over TLS over TCP.
|
||||
|
||||
* Synchronization of a single directory among the cluster nodes.
|
||||
|
||||
* Change detection by periodic scanning of the local repository.
|
||||
|
||||
* Static configuration of cluster nodes.
|
||||
|
||||
* Automatic discovery of cluster nodes on the local network. See
|
||||
[discover.go](https://github.com/calmh/syncthing/blob/master/discover/discover.go)
|
||||
for the protocol specification.
|
||||
|
||||
* Handling of deleted files. Deletes can be propagated or ignored per
|
||||
client.
|
||||
|
||||
The following features are _not yet implemented but planned_:
|
||||
|
||||
* Syncing multiple directories from the same syncthing instance.
|
||||
|
||||
* Change detection by listening to file system notifications instead of
|
||||
periodic scanning.
|
||||
|
||||
* HTTP GUI.
|
||||
|
||||
The following features are _not implemented but may be implemented_ in
|
||||
the future:
|
||||
|
||||
* Automatic remote node discovery using a DHT. This is not technically
|
||||
very difficult but requires one or more globally reachable root
|
||||
nodes. This is open for discussion -- perhaps we can piggyback on an
|
||||
existing DHT, or root nodes need to be established in some other
|
||||
manner.
|
||||
|
||||
* Automatic NAT handling via UPNP. Required for the above, not very
|
||||
useful without it.
|
||||
|
||||
* Conflict resolution. Currently whichever file has the newest
|
||||
modification time "wins". The correct behavior in the face of
|
||||
conflicts is open for discussion.
|
||||
|
||||
Security
|
||||
--------
|
||||
|
||||
Security is one of the primary project goals. This means that it should
|
||||
not be possible for an attacker to join a cluster uninvited, and it
|
||||
should not be possible to extract private information from intercepted
|
||||
traffic. Currently this is implemented as follows.
|
||||
|
||||
All traffic is protected by TLS. To prevent uninvited nodes from joining
|
||||
a cluster, the certificate fingerprint of each node is compared to a
|
||||
preset list of acceptable nodes at connection establishment. The
|
||||
fingerprint is computed as the SHA-1 hash of the certificate and
|
||||
displayed in BASE32 encoding to form a compact yet convenient string.
|
||||
Currently SHA-1 is deemed secure against preimage attacks.
|
||||
|
||||
Usage
|
||||
=====
|
||||
|
||||
`go get github.com/calmh/syncthing`
|
||||
|
||||
Check out the options:
|
||||
|
||||
```
|
||||
$ syncthing --help
|
||||
Usage:
|
||||
syncthing [options]
|
||||
|
||||
...
|
||||
```
|
||||
|
||||
Run syncthing to let it create it's config directory and certificate:
|
||||
|
||||
```
|
||||
$ syncthing
|
||||
11:34:13 tls.go:61: OK: wrote cert.pem
|
||||
11:34:13 tls.go:67: OK: wrote key.pem
|
||||
11:34:13 main.go:66: INFO: My ID: NCTBZAAHXR6ZZP3D7SL3DLYFFQERMW4Q
|
||||
11:34:13 main.go:90: FATAL: No config file
|
||||
```
|
||||
|
||||
Take note of the "My ID: ..." line. Perform the same operation on
|
||||
another computer (or the same computer but with a different `--home` for
|
||||
testing) to create another node. Take note of that ID as well, and
|
||||
create a config file `~/.syncthing/syncthing.ini` looking something like
|
||||
this:
|
||||
|
||||
```
|
||||
[repository]
|
||||
dir = /Users/jb/Synced
|
||||
|
||||
[nodes]
|
||||
NCTBZAAHXR6ZZP3D7SL3DLYFFQERMW4Q = 172.16.32.1:22000 192.23.34.56:22000
|
||||
CUGAE43Y5N64CRJU26YFH6MTWPSBLSUL = dynamic
|
||||
```
|
||||
|
||||
This assumes that the first node is reachable on either of the two
|
||||
addresses listed (perhaps one internal and one port-forwarded external)
|
||||
and that the other node is not normally reachable from the outside. Save
|
||||
this config file, identically, to both nodes. If both nodes are running
|
||||
on the same network, you can set all addresses to 'dynamic' and they
|
||||
will find each other by local node discovery.
|
||||
|
||||
Start syncthing on both nodes. If you're running both on the same
|
||||
computer, one needs a different repository directory (in the config
|
||||
file) and listening port (set as a command line paramter). For the
|
||||
cautious, one side can be set to be read only.
|
||||
|
||||
```
|
||||
$ syncthing --ro
|
||||
13:30:55 main.go:102: INFO: My ID: NCTBZAAHXR6ZZP3D7SL3DLYFFQERMW4Q
|
||||
13:30:55 main.go:149: INFO: Initial repository scan in progress
|
||||
13:30:59 main.go:153: INFO: Listening for incoming connections
|
||||
13:30:59 main.go:157: INFO: Attempting to connect to other nodes
|
||||
13:30:59 main.go:247: INFO: Starting local discovery
|
||||
13:30:59 main.go:165: OK: Ready to synchronize
|
||||
13:31:04 discover.go:113: INFO: Discovered node CUGAE43Y5N64CRJU26YFH6MTWPSBLSUL at 172.16.32.24:23456
|
||||
13:31:14 main.go:296: OK: Connected to node CUGAE43Y5N64CRJU26YFH6MTWPSBLSUL
|
||||
13:31:19 main.go:345: INFO: Transferred 139 KiB in (14 KiB/s), 139 KiB out (14 KiB/s)
|
||||
...
|
||||
```
|
||||
You should see the synchronization start and then finish a short while
|
||||
later. Add nodes to taste.
|
||||
|
||||
License
|
||||
=======
|
||||
|
||||
MIT
|
||||
|
67
blocks.go
Normal file
67
blocks.go
Normal file
@ -0,0 +1,67 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"io"
|
||||
)
|
||||
|
||||
type BlockList []Block
|
||||
|
||||
type Block struct {
|
||||
Offset uint64
|
||||
Length uint32
|
||||
Hash []byte
|
||||
}
|
||||
|
||||
// Blocks returns the blockwise hash of the reader.
|
||||
func Blocks(r io.Reader, blocksize int) (BlockList, error) {
|
||||
var blocks BlockList
|
||||
var offset uint64
|
||||
for {
|
||||
lr := &io.LimitedReader{r, int64(blocksize)}
|
||||
hf := sha256.New()
|
||||
n, err := io.Copy(hf, lr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if n == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
b := Block{
|
||||
Offset: offset,
|
||||
Length: uint32(n),
|
||||
Hash: hf.Sum(nil),
|
||||
}
|
||||
blocks = append(blocks, b)
|
||||
offset += uint64(n)
|
||||
}
|
||||
|
||||
return blocks, nil
|
||||
}
|
||||
|
||||
// To returns the list of blocks necessary to transform src into dst.
|
||||
// Both block lists must have been created with the same block size.
|
||||
func (src BlockList) To(tgt BlockList) (have, need BlockList) {
|
||||
if len(tgt) == 0 && len(src) != 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if len(tgt) != 0 && len(src) == 0 {
|
||||
// Copy the entire file
|
||||
return nil, tgt
|
||||
}
|
||||
|
||||
for i := range tgt {
|
||||
if i >= len(src) || bytes.Compare(tgt[i].Hash, src[i].Hash) != 0 {
|
||||
// Copy differing block
|
||||
need = append(need, tgt[i])
|
||||
} else {
|
||||
have = append(have, tgt[i])
|
||||
}
|
||||
}
|
||||
|
||||
return have, need
|
||||
}
|
115
blocks_test.go
Normal file
115
blocks_test.go
Normal file
@ -0,0 +1,115 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"testing"
|
||||
)
|
||||
|
||||
var blocksTestData = []struct {
|
||||
data []byte
|
||||
blocksize int
|
||||
hash []string
|
||||
}{
|
||||
{[]byte(""), 1024, []string{}},
|
||||
{[]byte("contents"), 1024, []string{
|
||||
"d1b2a59fbea7e20077af9f91b27e95e865061b270be03ff539ab3b73587882e8"}},
|
||||
{[]byte("contents"), 9, []string{
|
||||
"d1b2a59fbea7e20077af9f91b27e95e865061b270be03ff539ab3b73587882e8"}},
|
||||
{[]byte("contents"), 8, []string{
|
||||
"d1b2a59fbea7e20077af9f91b27e95e865061b270be03ff539ab3b73587882e8"}},
|
||||
{[]byte("contents"), 7, []string{
|
||||
"ed7002b439e9ac845f22357d822bac1444730fbdb6016d3ec9432297b9ec9f73",
|
||||
"043a718774c572bd8a25adbeb1bfcd5c0256ae11cecf9f9c3f925d0e52beaf89"},
|
||||
},
|
||||
{[]byte("contents"), 3, []string{
|
||||
"1143da2bc54c495c4be31d3868785d39ffdfd56df5668f0645d8f14d47647952",
|
||||
"e4432baa90819aaef51d2a7f8e148bf7e679610f3173752fabb4dcb2d0f418d3",
|
||||
"44ad63f60af0f6db6fdde6d5186ef78176367df261fa06be3079b6c80c8adba4"},
|
||||
},
|
||||
{[]byte("conconts"), 3, []string{
|
||||
"1143da2bc54c495c4be31d3868785d39ffdfd56df5668f0645d8f14d47647952",
|
||||
"1143da2bc54c495c4be31d3868785d39ffdfd56df5668f0645d8f14d47647952",
|
||||
"44ad63f60af0f6db6fdde6d5186ef78176367df261fa06be3079b6c80c8adba4"},
|
||||
},
|
||||
{[]byte("contenten"), 3, []string{
|
||||
"1143da2bc54c495c4be31d3868785d39ffdfd56df5668f0645d8f14d47647952",
|
||||
"e4432baa90819aaef51d2a7f8e148bf7e679610f3173752fabb4dcb2d0f418d3",
|
||||
"e4432baa90819aaef51d2a7f8e148bf7e679610f3173752fabb4dcb2d0f418d3"},
|
||||
},
|
||||
}
|
||||
|
||||
func TestBlocks(t *testing.T) {
|
||||
for _, test := range blocksTestData {
|
||||
buf := bytes.NewBuffer(test.data)
|
||||
blocks, err := Blocks(buf, test.blocksize)
|
||||
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if l := len(blocks); l != len(test.hash) {
|
||||
t.Fatalf("Incorrect number of blocks %d != %d", l, len(test.hash))
|
||||
} else {
|
||||
i := 0
|
||||
for off := uint64(0); off < uint64(len(test.data)); off += uint64(test.blocksize) {
|
||||
if blocks[i].Offset != off {
|
||||
t.Errorf("Incorrect offset for block %d: %d != %d", i, blocks[i].Offset, off)
|
||||
}
|
||||
|
||||
bs := test.blocksize
|
||||
if rem := len(test.data) - int(off); bs > rem {
|
||||
bs = rem
|
||||
}
|
||||
if int(blocks[i].Length) != bs {
|
||||
t.Errorf("Incorrect length for block %d: %d != %d", i, blocks[i].Length, bs)
|
||||
}
|
||||
if h := fmt.Sprintf("%x", blocks[i].Hash); h != test.hash[i] {
|
||||
t.Errorf("Incorrect block hash %q != %q", h, test.hash[i])
|
||||
}
|
||||
|
||||
i++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var diffTestData = []struct {
|
||||
a string
|
||||
b string
|
||||
s int
|
||||
d []Block
|
||||
}{
|
||||
{"contents", "contents", 1024, []Block{}},
|
||||
{"", "", 1024, []Block{}},
|
||||
{"contents", "contents", 3, []Block{}},
|
||||
{"contents", "cantents", 3, []Block{{0, 3, nil}}},
|
||||
{"contents", "contants", 3, []Block{{3, 3, nil}}},
|
||||
{"contents", "cantants", 3, []Block{{0, 3, nil}, {3, 3, nil}}},
|
||||
{"contents", "", 3, nil},
|
||||
{"", "contents", 3, []Block{{0, 3, nil}, {3, 3, nil}, {6, 2, nil}}},
|
||||
{"con", "contents", 3, []Block{{3, 3, nil}, {6, 2, nil}}},
|
||||
{"contents", "con", 3, nil},
|
||||
{"contents", "cont", 3, []Block{{3, 1, nil}}},
|
||||
{"cont", "contents", 3, []Block{{3, 3, nil}, {6, 2, nil}}},
|
||||
}
|
||||
|
||||
func TestDiff(t *testing.T) {
|
||||
for i, test := range diffTestData {
|
||||
a, _ := Blocks(bytes.NewBufferString(test.a), test.s)
|
||||
b, _ := Blocks(bytes.NewBufferString(test.b), test.s)
|
||||
_, d := a.To(b)
|
||||
if len(d) != len(test.d) {
|
||||
t.Fatalf("Incorrect length for diff %d; %d != %d", i, len(d), len(test.d))
|
||||
} else {
|
||||
for j := range test.d {
|
||||
if d[j].Offset != test.d[j].Offset {
|
||||
t.Errorf("Incorrect offset for diff %d block %d; %d != %d", i, j, d[j].Offset, test.d[j].Offset)
|
||||
}
|
||||
if d[j].Length != test.d[j].Length {
|
||||
t.Errorf("Incorrect length for diff %d block %d; %d != %d", i, j, d[j].Length, test.d[j].Length)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
26
buffers/buffers.go
Normal file
26
buffers/buffers.go
Normal file
@ -0,0 +1,26 @@
|
||||
package buffers
|
||||
|
||||
var buffers = make(chan []byte, 32)
|
||||
|
||||
func Get(size int) []byte {
|
||||
var buf []byte
|
||||
select {
|
||||
case buf = <-buffers:
|
||||
default:
|
||||
}
|
||||
if len(buf) < size {
|
||||
return make([]byte, size)
|
||||
}
|
||||
return buf[:size]
|
||||
}
|
||||
|
||||
func Put(buf []byte) {
|
||||
if cap(buf) == 0 {
|
||||
return
|
||||
}
|
||||
buf = buf[:cap(buf)]
|
||||
select {
|
||||
case buffers <- buf:
|
||||
default:
|
||||
}
|
||||
}
|
121
discover/discover.go
Normal file
121
discover/discover.go
Normal file
@ -0,0 +1,121 @@
|
||||
/*
|
||||
This is the local node discovery protocol. In principle we might be better
|
||||
served by something more standardized, such as mDNS / DNS-SD. In practice, this
|
||||
was much easier and quicker to get up and running.
|
||||
|
||||
The mode of operation is to periodically (currently once every 30 seconds)
|
||||
transmit a broadcast UDP packet to the well known port number 21025. The packet
|
||||
has the following format:
|
||||
|
||||
0 1 2 3
|
||||
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
|
||||
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|
||||
| Magic Number |
|
||||
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|
||||
| Port Number | Length of NodeID |
|
||||
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|
||||
/ /
|
||||
\ NodeID (variable length) \
|
||||
/ /
|
||||
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|
||||
|
||||
The sending node's address is not encoded -- it is taken to be the source
|
||||
address of the announcement. Every time such a packet is received, a local
|
||||
table that maps NodeID to Address is updated. When the local node wants to
|
||||
connect to another node with the address specification 'dynamic', this table is
|
||||
consulted.
|
||||
*/
|
||||
package discover
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"net"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Discoverer struct {
|
||||
MyID string
|
||||
ListenPort int
|
||||
BroadcastIntv time.Duration
|
||||
|
||||
conn *net.UDPConn
|
||||
registry map[string]string
|
||||
registryLock sync.RWMutex
|
||||
}
|
||||
|
||||
func NewDiscoverer(id string, port int) (*Discoverer, error) {
|
||||
local4 := &net.UDPAddr{IP: net.IP{0, 0, 0, 0}, Port: 21025}
|
||||
conn, err := net.ListenUDP("udp4", local4)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
disc := &Discoverer{
|
||||
MyID: id,
|
||||
ListenPort: port,
|
||||
BroadcastIntv: 30 * time.Second,
|
||||
conn: conn,
|
||||
registry: make(map[string]string),
|
||||
}
|
||||
|
||||
go disc.sendAnnouncements()
|
||||
go disc.recvAnnouncements()
|
||||
|
||||
return disc, nil
|
||||
}
|
||||
|
||||
func (d *Discoverer) sendAnnouncements() {
|
||||
remote4 := &net.UDPAddr{IP: net.IP{255, 255, 255, 255}, Port: 21025}
|
||||
|
||||
idbs := []byte(d.MyID)
|
||||
buf := make([]byte, 4+4+4+len(idbs))
|
||||
|
||||
binary.BigEndian.PutUint32(buf, uint32(0x121025))
|
||||
binary.BigEndian.PutUint16(buf[4:], uint16(d.ListenPort))
|
||||
binary.BigEndian.PutUint16(buf[6:], uint16(len(idbs)))
|
||||
copy(buf[8:], idbs)
|
||||
|
||||
for {
|
||||
_, _, err := d.conn.WriteMsgUDP(buf, nil, remote4)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
time.Sleep(d.BroadcastIntv)
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Discoverer) recvAnnouncements() {
|
||||
var buf = make([]byte, 1024)
|
||||
for {
|
||||
_, addr, err := d.conn.ReadFromUDP(buf)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
magic := binary.BigEndian.Uint32(buf)
|
||||
if magic != 0x121025 {
|
||||
continue
|
||||
}
|
||||
port := binary.BigEndian.Uint16(buf[4:])
|
||||
l := binary.BigEndian.Uint16(buf[6:])
|
||||
idbs := buf[8 : l+8]
|
||||
id := string(idbs)
|
||||
|
||||
if id != d.MyID {
|
||||
nodeAddr := fmt.Sprintf("%s:%d", addr.IP.String(), port)
|
||||
d.registryLock.Lock()
|
||||
if d.registry[id] != nodeAddr {
|
||||
d.registry[id] = nodeAddr
|
||||
}
|
||||
d.registryLock.Unlock()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Discoverer) Lookup(node string) (string, bool) {
|
||||
d.registryLock.Lock()
|
||||
defer d.registryLock.Unlock()
|
||||
addr, ok := d.registry[node]
|
||||
return addr, ok
|
||||
}
|
72
logger.go
Normal file
72
logger.go
Normal file
@ -0,0 +1,72 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
)
|
||||
|
||||
var debugEnabled = true
|
||||
var logger = log.New(os.Stderr, "", log.Lshortfile|log.Ltime)
|
||||
|
||||
func debugln(vals ...interface{}) {
|
||||
if debugEnabled {
|
||||
s := fmt.Sprintln(vals...)
|
||||
logger.Output(2, "DEBUG: "+s)
|
||||
}
|
||||
}
|
||||
|
||||
func debugf(format string, vals ...interface{}) {
|
||||
if debugEnabled {
|
||||
s := fmt.Sprintf(format, vals...)
|
||||
logger.Output(2, "DEBUG: "+s)
|
||||
}
|
||||
}
|
||||
|
||||
func infoln(vals ...interface{}) {
|
||||
s := fmt.Sprintln(vals...)
|
||||
logger.Output(2, "INFO: "+s)
|
||||
}
|
||||
|
||||
func infof(format string, vals ...interface{}) {
|
||||
s := fmt.Sprintf(format, vals...)
|
||||
logger.Output(2, "INFO: "+s)
|
||||
}
|
||||
|
||||
func okln(vals ...interface{}) {
|
||||
s := fmt.Sprintln(vals...)
|
||||
logger.Output(2, "OK: "+s)
|
||||
}
|
||||
|
||||
func okf(format string, vals ...interface{}) {
|
||||
s := fmt.Sprintf(format, vals...)
|
||||
logger.Output(2, "OK: "+s)
|
||||
}
|
||||
|
||||
func warnln(vals ...interface{}) {
|
||||
s := fmt.Sprintln(vals...)
|
||||
logger.Output(2, "WARNING: "+s)
|
||||
}
|
||||
|
||||
func warnf(format string, vals ...interface{}) {
|
||||
s := fmt.Sprintf(format, vals...)
|
||||
logger.Output(2, "WARNING: "+s)
|
||||
}
|
||||
|
||||
func fatalln(vals ...interface{}) {
|
||||
s := fmt.Sprintln(vals...)
|
||||
logger.Output(2, "FATAL: "+s)
|
||||
os.Exit(3)
|
||||
}
|
||||
|
||||
func fatalf(format string, vals ...interface{}) {
|
||||
s := fmt.Sprintf(format, vals...)
|
||||
logger.Output(2, "FATAL: "+s)
|
||||
os.Exit(3)
|
||||
}
|
||||
|
||||
func fatalErr(err error) {
|
||||
if err != nil {
|
||||
fatalf(err.Error())
|
||||
}
|
||||
}
|
352
main.go
Normal file
352
main.go
Normal file
@ -0,0 +1,352 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha1"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
_ "net/http/pprof"
|
||||
"os"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/calmh/ini"
|
||||
"github.com/calmh/syncthing/discover"
|
||||
"github.com/calmh/syncthing/protocol"
|
||||
docopt "github.com/docopt/docopt.go"
|
||||
)
|
||||
|
||||
const (
|
||||
confDirName = ".syncthing"
|
||||
confFileName = "syncthing.ini"
|
||||
usage = `Usage:
|
||||
syncthing [options]
|
||||
|
||||
Options:
|
||||
-l <addr> Listening address [default: :22000]
|
||||
-p <addr> Enable HTTP profiler on addr
|
||||
--home <path> Home directory
|
||||
--delete Delete files that were deleted on a peer node
|
||||
--ro Local repository is read only
|
||||
--scan-intv <s> Repository scan interval, in seconds [default: 60]
|
||||
--conn-intv <s> Node reconnect interval, in seconds [default: 15]
|
||||
--no-stats Don't print transfer statistics
|
||||
|
||||
Help Options:
|
||||
-h, --help Show this help
|
||||
--version Show version
|
||||
|
||||
Debug Options:
|
||||
--trace-file Trace file operations
|
||||
--trace-net Trace network operations
|
||||
--trace-idx Trace sent indexes
|
||||
`
|
||||
)
|
||||
|
||||
var (
|
||||
config ini.Config
|
||||
nodeAddrs = make(map[string][]string)
|
||||
)
|
||||
|
||||
// Options
|
||||
var (
|
||||
confDir = path.Join(getHomeDir(), confDirName)
|
||||
addr string
|
||||
prof string
|
||||
readOnly bool
|
||||
scanIntv int
|
||||
connIntv int
|
||||
traceNet bool
|
||||
traceFile bool
|
||||
traceIdx bool
|
||||
printStats bool
|
||||
doDelete bool
|
||||
)
|
||||
|
||||
func main() {
|
||||
// Useful for debugging; to be adjusted.
|
||||
log.SetFlags(log.Ltime | log.Lshortfile)
|
||||
|
||||
arguments, _ := docopt.Parse(usage, nil, true, "syncthing 0.1", false)
|
||||
|
||||
addr = arguments["-l"].(string)
|
||||
prof, _ = arguments["-p"].(string)
|
||||
readOnly, _ = arguments["--ro"].(bool)
|
||||
|
||||
if arguments["--home"] != nil {
|
||||
confDir, _ = arguments["--home"].(string)
|
||||
}
|
||||
|
||||
scanIntv, _ = strconv.Atoi(arguments["--scan-intv"].(string))
|
||||
if scanIntv == 0 {
|
||||
fatalln("Invalid --scan-intv")
|
||||
}
|
||||
|
||||
connIntv, _ = strconv.Atoi(arguments["--conn-intv"].(string))
|
||||
if connIntv == 0 {
|
||||
fatalln("Invalid --conn-intv")
|
||||
}
|
||||
|
||||
doDelete = arguments["--delete"].(bool)
|
||||
traceFile = arguments["--trace-file"].(bool)
|
||||
traceNet = arguments["--trace-net"].(bool)
|
||||
traceIdx = arguments["--trace-idx"].(bool)
|
||||
printStats = !arguments["--no-stats"].(bool)
|
||||
|
||||
// Ensure that our home directory exists and that we have a certificate and key.
|
||||
|
||||
ensureDir(confDir)
|
||||
cert, err := loadCert(confDir)
|
||||
if err != nil {
|
||||
newCertificate(confDir)
|
||||
cert, err = loadCert(confDir)
|
||||
fatalErr(err)
|
||||
}
|
||||
|
||||
myID := string(certId(cert.Certificate[0]))
|
||||
infoln("My ID:", myID)
|
||||
|
||||
if prof != "" {
|
||||
okln("Profiler listening on", prof)
|
||||
go func() {
|
||||
http.ListenAndServe(prof, nil)
|
||||
}()
|
||||
}
|
||||
|
||||
// The TLS configuration is used for both the listening socket and outgoing
|
||||
// connections.
|
||||
|
||||
cfg := &tls.Config{
|
||||
ClientAuth: tls.RequestClientCert,
|
||||
ServerName: "syncthing",
|
||||
NextProtos: []string{"bep/1.0"},
|
||||
InsecureSkipVerify: true,
|
||||
Certificates: []tls.Certificate{cert},
|
||||
}
|
||||
|
||||
// Load the configuration file, if it exists.
|
||||
|
||||
cf, err := os.Open(path.Join(confDir, confFileName))
|
||||
if err != nil {
|
||||
fatalln("No config file")
|
||||
config = ini.Config{}
|
||||
}
|
||||
config = ini.Parse(cf)
|
||||
cf.Close()
|
||||
|
||||
var dir = config.Get("repository", "dir")
|
||||
|
||||
// Create a map of desired node connections based on the configuration file
|
||||
// directives.
|
||||
|
||||
for nodeID, addrs := range config.OptionMap("nodes") {
|
||||
addrs := strings.Fields(addrs)
|
||||
nodeAddrs[nodeID] = addrs
|
||||
}
|
||||
|
||||
m := NewModel(dir)
|
||||
|
||||
// Walk the repository and update the local model before establishing any
|
||||
// connections to other nodes.
|
||||
|
||||
infoln("Iniial repository scan in progress")
|
||||
loadIndex(m)
|
||||
updateLocalModel(m)
|
||||
|
||||
// Routine to listen for incoming connections
|
||||
infoln("Listening for incoming connections")
|
||||
go listen(myID, addr, m, cfg)
|
||||
|
||||
// Routine to connect out to configured nodes
|
||||
infoln("Attempting to connect to other nodes")
|
||||
go connect(myID, addr, nodeAddrs, m, cfg)
|
||||
|
||||
// Routine to pull blocks from other nodes to synchronize the local
|
||||
// repository. Does not run when we are in read only (publish only) mode.
|
||||
if !readOnly {
|
||||
infoln("Cleaning out incomplete synchronizations")
|
||||
CleanTempFiles(dir)
|
||||
okln("Ready to synchronize")
|
||||
m.Start()
|
||||
}
|
||||
|
||||
// Periodically scan the repository and update the local model.
|
||||
// XXX: Should use some fsnotify mechanism.
|
||||
go func() {
|
||||
for {
|
||||
time.Sleep(time.Duration(scanIntv) * time.Second)
|
||||
updateLocalModel(m)
|
||||
}
|
||||
}()
|
||||
|
||||
select {}
|
||||
}
|
||||
|
||||
func listen(myID string, addr string, m *Model, cfg *tls.Config) {
|
||||
l, err := tls.Listen("tcp", addr, cfg)
|
||||
fatalErr(err)
|
||||
|
||||
listen:
|
||||
for {
|
||||
conn, err := l.Accept()
|
||||
if err != nil {
|
||||
warnln(err)
|
||||
continue
|
||||
}
|
||||
|
||||
if traceNet {
|
||||
debugln("NET: Connect from", conn.RemoteAddr())
|
||||
}
|
||||
|
||||
tc := conn.(*tls.Conn)
|
||||
err = tc.Handshake()
|
||||
if err != nil {
|
||||
warnln(err)
|
||||
tc.Close()
|
||||
continue
|
||||
}
|
||||
|
||||
remoteID := certId(tc.ConnectionState().PeerCertificates[0].Raw)
|
||||
|
||||
if remoteID == myID {
|
||||
warnf("Connect from myself (%s) - should not happen", remoteID)
|
||||
conn.Close()
|
||||
continue
|
||||
}
|
||||
|
||||
if m.ConnectedTo(remoteID) {
|
||||
warnf("Connect from connected node (%s)", remoteID)
|
||||
}
|
||||
|
||||
for nodeID := range nodeAddrs {
|
||||
if nodeID == remoteID {
|
||||
nc := protocol.NewConnection(remoteID, conn, conn, m)
|
||||
m.AddNode(nc)
|
||||
okln("Connected to nodeID", remoteID, "(in)")
|
||||
continue listen
|
||||
}
|
||||
}
|
||||
|
||||
warnln("Connect from unknown node", remoteID)
|
||||
conn.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func connect(myID string, addr string, nodeAddrs map[string][]string, m *Model, cfg *tls.Config) {
|
||||
_, portstr, err := net.SplitHostPort(addr)
|
||||
fatalErr(err)
|
||||
port, _ := strconv.Atoi(portstr)
|
||||
|
||||
infoln("Starting local discovery")
|
||||
disc, err := discover.NewDiscoverer(myID, port)
|
||||
if err != nil {
|
||||
warnln("No local discovery possible")
|
||||
}
|
||||
|
||||
for {
|
||||
nextNode:
|
||||
for nodeID, addrs := range nodeAddrs {
|
||||
if nodeID == myID {
|
||||
continue
|
||||
}
|
||||
if m.ConnectedTo(nodeID) {
|
||||
continue
|
||||
}
|
||||
for _, addr := range addrs {
|
||||
if addr == "dynamic" {
|
||||
var ok bool
|
||||
if disc != nil {
|
||||
addr, ok = disc.Lookup(nodeID)
|
||||
}
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if traceNet {
|
||||
debugln("NET: Dial", nodeID, addr)
|
||||
}
|
||||
conn, err := tls.Dial("tcp", addr, cfg)
|
||||
if err != nil {
|
||||
if traceNet {
|
||||
debugln("NET:", err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
remoteID := certId(conn.ConnectionState().PeerCertificates[0].Raw)
|
||||
if remoteID != nodeID {
|
||||
warnln("Unexpected nodeID", remoteID, "!=", nodeID)
|
||||
conn.Close()
|
||||
continue
|
||||
}
|
||||
|
||||
nc := protocol.NewConnection(nodeID, conn, conn, m)
|
||||
okln("Connected to node", remoteID, "(out)")
|
||||
m.AddNode(nc)
|
||||
if traceNet {
|
||||
t0 := time.Now()
|
||||
nc.Ping()
|
||||
timing("NET: Ping reply", t0)
|
||||
}
|
||||
continue nextNode
|
||||
}
|
||||
}
|
||||
|
||||
time.Sleep(time.Duration(connIntv) * time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
func updateLocalModel(m *Model) {
|
||||
files := Walk(m.Dir(), m)
|
||||
m.ReplaceLocal(files)
|
||||
saveIndex(m)
|
||||
}
|
||||
|
||||
func saveIndex(m *Model) {
|
||||
fname := fmt.Sprintf("%x.idx", sha1.Sum([]byte(m.Dir())))
|
||||
idxf, err := os.Create(path.Join(confDir, fname))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
protocol.WriteIndex(idxf, m.ProtocolIndex())
|
||||
idxf.Close()
|
||||
}
|
||||
|
||||
func loadIndex(m *Model) {
|
||||
fname := fmt.Sprintf("%x.idx", sha1.Sum([]byte(m.Dir())))
|
||||
idxf, err := os.Open(path.Join(confDir, fname))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer idxf.Close()
|
||||
|
||||
idx, err := protocol.ReadIndex(idxf)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
m.SeedIndex(idx)
|
||||
}
|
||||
|
||||
func ensureDir(dir string) {
|
||||
fi, err := os.Stat(dir)
|
||||
if os.IsNotExist(err) {
|
||||
err := os.MkdirAll(dir, 0700)
|
||||
fatalErr(err)
|
||||
} else if fi.Mode()&0077 != 0 {
|
||||
err := os.Chmod(dir, 0700)
|
||||
fatalErr(err)
|
||||
}
|
||||
}
|
||||
|
||||
func getHomeDir() string {
|
||||
home := os.Getenv("HOME")
|
||||
if home == "" {
|
||||
fatalln("No home directory?")
|
||||
}
|
||||
return home
|
||||
}
|
373
model.go
Normal file
373
model.go
Normal file
@ -0,0 +1,373 @@
|
||||
package main
|
||||
|
||||
/*
|
||||
|
||||
Locking
|
||||
=======
|
||||
|
||||
The model has read and write locks. These must be acquired as appropriate by
|
||||
public methods. To prevent deadlock situations, private methods should never
|
||||
acquire locks, but document what locks they require.
|
||||
|
||||
TODO(jb): Keep global and per node transfer and performance statistics.
|
||||
|
||||
*/
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/calmh/syncthing/buffers"
|
||||
"github.com/calmh/syncthing/protocol"
|
||||
)
|
||||
|
||||
type Model struct {
|
||||
sync.RWMutex
|
||||
dir string
|
||||
updated int64
|
||||
global map[string]File // the latest version of each file as it exists in the cluster
|
||||
local map[string]File // the files we currently have locally on disk
|
||||
remote map[string]map[string]File
|
||||
need map[string]bool // the files we need to update
|
||||
nodes map[string]*protocol.Connection
|
||||
}
|
||||
|
||||
const (
|
||||
RemoteFetchers = 4
|
||||
FlagDeleted = 1 << 12
|
||||
)
|
||||
|
||||
func NewModel(dir string) *Model {
|
||||
m := &Model{
|
||||
dir: dir,
|
||||
global: make(map[string]File),
|
||||
local: make(map[string]File),
|
||||
remote: make(map[string]map[string]File),
|
||||
need: make(map[string]bool),
|
||||
nodes: make(map[string]*protocol.Connection),
|
||||
}
|
||||
|
||||
return m
|
||||
}
|
||||
|
||||
func (m *Model) Start() {
|
||||
go m.puller()
|
||||
}
|
||||
|
||||
func (m *Model) Index(nodeID string, fs []protocol.FileInfo) {
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
|
||||
if traceNet {
|
||||
debugf("NET IDX(in): %s: %d files", nodeID, len(fs))
|
||||
}
|
||||
|
||||
m.remote[nodeID] = make(map[string]File)
|
||||
for _, f := range fs {
|
||||
if f.Flags&FlagDeleted != 0 && !doDelete {
|
||||
// Files marked as deleted do not even enter the model
|
||||
continue
|
||||
}
|
||||
mf := File{
|
||||
Name: f.Name,
|
||||
Flags: f.Flags,
|
||||
Modified: int64(f.Modified),
|
||||
}
|
||||
var offset uint64
|
||||
for _, b := range f.Blocks {
|
||||
mf.Blocks = append(mf.Blocks, Block{
|
||||
Offset: offset,
|
||||
Length: b.Length,
|
||||
Hash: b.Hash,
|
||||
})
|
||||
offset += uint64(b.Length)
|
||||
}
|
||||
m.remote[nodeID][f.Name] = mf
|
||||
}
|
||||
|
||||
m.recomputeGlobal()
|
||||
m.recomputeNeed()
|
||||
}
|
||||
|
||||
func (m *Model) SeedIndex(fs []protocol.FileInfo) {
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
|
||||
m.local = make(map[string]File)
|
||||
for _, f := range fs {
|
||||
mf := File{
|
||||
Name: f.Name,
|
||||
Flags: f.Flags,
|
||||
Modified: int64(f.Modified),
|
||||
}
|
||||
var offset uint64
|
||||
for _, b := range f.Blocks {
|
||||
mf.Blocks = append(mf.Blocks, Block{
|
||||
Offset: offset,
|
||||
Length: b.Length,
|
||||
Hash: b.Hash,
|
||||
})
|
||||
offset += uint64(b.Length)
|
||||
}
|
||||
m.local[f.Name] = mf
|
||||
}
|
||||
|
||||
m.recomputeGlobal()
|
||||
m.recomputeNeed()
|
||||
}
|
||||
|
||||
func (m *Model) Close(node string) {
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
|
||||
if traceNet {
|
||||
debugf("NET CLOSE: %s", node)
|
||||
}
|
||||
|
||||
delete(m.remote, node)
|
||||
delete(m.nodes, node)
|
||||
|
||||
m.recomputeGlobal()
|
||||
m.recomputeNeed()
|
||||
}
|
||||
|
||||
func (m *Model) Request(nodeID, name string, offset uint64, size uint32, hash []byte) ([]byte, error) {
|
||||
if traceNet && nodeID != "<local>" {
|
||||
debugf("NET REQ(in): %s: %q o=%d s=%d h=%x", nodeID, name, offset, size, hash)
|
||||
}
|
||||
fn := path.Join(m.dir, name)
|
||||
fd, err := os.Open(fn) // XXX: Inefficient, should cache fd?
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer fd.Close()
|
||||
|
||||
buf := buffers.Get(int(size))
|
||||
_, err = fd.ReadAt(buf, int64(offset))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return buf, nil
|
||||
}
|
||||
|
||||
func (m *Model) RequestGlobal(nodeID, name string, offset uint64, size uint32, hash []byte) ([]byte, error) {
|
||||
m.RLock()
|
||||
nc := m.nodes[nodeID]
|
||||
m.RUnlock()
|
||||
|
||||
if traceNet {
|
||||
debugf("NET REQ(out): %s: %q o=%d s=%d h=%x", nodeID, name, offset, size, hash)
|
||||
}
|
||||
|
||||
return nc.Request(name, offset, size, hash)
|
||||
}
|
||||
|
||||
func (m *Model) ReplaceLocal(fs []File) {
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
|
||||
var updated bool
|
||||
var newLocal = make(map[string]File)
|
||||
|
||||
for _, f := range fs {
|
||||
newLocal[f.Name] = f
|
||||
if ef := m.local[f.Name]; ef.Modified != f.Modified {
|
||||
updated = true
|
||||
}
|
||||
}
|
||||
|
||||
if m.markDeletedLocals(newLocal) {
|
||||
updated = true
|
||||
}
|
||||
|
||||
if len(newLocal) != len(m.local) {
|
||||
updated = true
|
||||
}
|
||||
|
||||
if updated {
|
||||
m.local = newLocal
|
||||
m.recomputeGlobal()
|
||||
m.recomputeNeed()
|
||||
m.updated = time.Now().Unix()
|
||||
go m.broadcastIndex()
|
||||
}
|
||||
}
|
||||
|
||||
// Must be called with the read lock held.
|
||||
func (m *Model) broadcastIndex() {
|
||||
idx := m.protocolIndex()
|
||||
for _, node := range m.nodes {
|
||||
if traceNet {
|
||||
debugf("NET IDX(out): %s: %d files", node.ID, len(idx))
|
||||
}
|
||||
node.Index(idx)
|
||||
}
|
||||
}
|
||||
|
||||
// markDeletedLocals sets the deleted flag on files that have gone missing locally.
|
||||
// Must be called with the write lock held.
|
||||
func (m *Model) markDeletedLocals(newLocal map[string]File) bool {
|
||||
// For every file in the existing local table, check if they are also
|
||||
// present in the new local table. If they are not, check that we already
|
||||
// had the newest version available according to the global table and if so
|
||||
// note the file as having been deleted.
|
||||
var updated bool
|
||||
for n, f := range m.local {
|
||||
if _, ok := newLocal[n]; !ok {
|
||||
if gf := m.global[n]; gf.Modified <= f.Modified {
|
||||
if f.Flags&FlagDeleted == 0 {
|
||||
f.Flags = FlagDeleted
|
||||
f.Modified = f.Modified + 1
|
||||
f.Blocks = nil
|
||||
updated = true
|
||||
}
|
||||
newLocal[n] = f
|
||||
}
|
||||
}
|
||||
}
|
||||
return updated
|
||||
}
|
||||
|
||||
func (m *Model) UpdateLocal(f File) {
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
|
||||
if ef, ok := m.local[f.Name]; !ok || ef.Modified != f.Modified {
|
||||
m.local[f.Name] = f
|
||||
m.recomputeGlobal()
|
||||
m.recomputeNeed()
|
||||
m.updated = time.Now().Unix()
|
||||
go m.broadcastIndex()
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Model) Dir() string {
|
||||
m.RLock()
|
||||
defer m.RUnlock()
|
||||
return m.dir
|
||||
}
|
||||
|
||||
func (m *Model) HaveFiles() []File {
|
||||
m.RLock()
|
||||
defer m.RUnlock()
|
||||
var files []File
|
||||
for _, file := range m.local {
|
||||
files = append(files, file)
|
||||
}
|
||||
return files
|
||||
}
|
||||
|
||||
func (m *Model) LocalFile(name string) (File, bool) {
|
||||
m.RLock()
|
||||
defer m.RUnlock()
|
||||
f, ok := m.local[name]
|
||||
return f, ok
|
||||
}
|
||||
|
||||
func (m *Model) GlobalFile(name string) (File, bool) {
|
||||
m.RLock()
|
||||
defer m.RUnlock()
|
||||
f, ok := m.global[name]
|
||||
return f, ok
|
||||
}
|
||||
|
||||
// Must be called with the write lock held.
|
||||
func (m *Model) recomputeGlobal() {
|
||||
var newGlobal = make(map[string]File)
|
||||
|
||||
for n, f := range m.local {
|
||||
newGlobal[n] = f
|
||||
}
|
||||
|
||||
for _, fs := range m.remote {
|
||||
for n, f := range fs {
|
||||
if cf, ok := newGlobal[n]; !ok || cf.Modified < f.Modified {
|
||||
newGlobal[n] = f
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
m.global = newGlobal
|
||||
}
|
||||
|
||||
// Must be called with the write lock held.
|
||||
func (m *Model) recomputeNeed() {
|
||||
m.need = make(map[string]bool)
|
||||
for n, f := range m.global {
|
||||
hf, ok := m.local[n]
|
||||
if !ok || f.Modified > hf.Modified {
|
||||
m.need[n] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Must be called with the read lock held.
|
||||
func (m *Model) whoHas(name string) []string {
|
||||
var remote []string
|
||||
|
||||
gf := m.global[name]
|
||||
for node, files := range m.remote {
|
||||
if file, ok := files[name]; ok && file.Modified == gf.Modified {
|
||||
remote = append(remote, node)
|
||||
}
|
||||
}
|
||||
|
||||
return remote
|
||||
}
|
||||
|
||||
func (m *Model) ConnectedTo(nodeID string) bool {
|
||||
m.RLock()
|
||||
defer m.RUnlock()
|
||||
_, ok := m.nodes[nodeID]
|
||||
return ok
|
||||
}
|
||||
|
||||
func (m *Model) ProtocolIndex() []protocol.FileInfo {
|
||||
m.RLock()
|
||||
defer m.RUnlock()
|
||||
return m.protocolIndex()
|
||||
}
|
||||
|
||||
// Must be called with the read lock held.
|
||||
func (m *Model) protocolIndex() []protocol.FileInfo {
|
||||
var index []protocol.FileInfo
|
||||
for _, f := range m.local {
|
||||
mf := protocol.FileInfo{
|
||||
Name: f.Name,
|
||||
Flags: f.Flags,
|
||||
Modified: int64(f.Modified),
|
||||
}
|
||||
for _, b := range f.Blocks {
|
||||
mf.Blocks = append(mf.Blocks, protocol.BlockInfo{
|
||||
Length: b.Length,
|
||||
Hash: b.Hash,
|
||||
})
|
||||
}
|
||||
if traceIdx {
|
||||
var flagComment string
|
||||
if mf.Flags&FlagDeleted != 0 {
|
||||
flagComment = " (deleted)"
|
||||
}
|
||||
debugf("IDX: %q m=%d f=%o%s (%d blocks)", mf.Name, mf.Modified, mf.Flags, flagComment, len(mf.Blocks))
|
||||
}
|
||||
index = append(index, mf)
|
||||
}
|
||||
return index
|
||||
}
|
||||
|
||||
func (m *Model) AddNode(node *protocol.Connection) {
|
||||
m.Lock()
|
||||
m.nodes[node.ID] = node
|
||||
m.Unlock()
|
||||
m.RLock()
|
||||
idx := m.protocolIndex()
|
||||
m.RUnlock()
|
||||
|
||||
if traceNet {
|
||||
debugf("NET IDX(out): %s: %d files", node.ID, len(idx))
|
||||
}
|
||||
node.Index(idx)
|
||||
}
|
212
model_puller.go
Normal file
212
model_puller.go
Normal file
@ -0,0 +1,212 @@
|
||||
package main
|
||||
|
||||
/*
|
||||
|
||||
Locking
|
||||
=======
|
||||
|
||||
These methods are never called from the outside so don't follow the locking
|
||||
policy in model.go. Instead, appropriate locks are acquired when needed and
|
||||
held for as short a time as possible.
|
||||
|
||||
TODO(jb): Refactor this into smaller and cleaner pieces.
|
||||
|
||||
TODO(jb): Some kind of coalescing / rate limiting of index sending, so we don't
|
||||
send hundreds of index updates in a short period if time when deleting files
|
||||
etc.
|
||||
|
||||
*/
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/calmh/syncthing/buffers"
|
||||
)
|
||||
|
||||
func (m *Model) pullFile(name string) error {
|
||||
m.RLock()
|
||||
var localFile = m.local[name]
|
||||
var globalFile = m.global[name]
|
||||
m.RUnlock()
|
||||
|
||||
filename := path.Join(m.dir, name)
|
||||
sdir := path.Dir(filename)
|
||||
|
||||
_, err := os.Stat(sdir)
|
||||
if err != nil && os.IsNotExist(err) {
|
||||
os.MkdirAll(sdir, 0777)
|
||||
}
|
||||
|
||||
tmpFilename := tempName(filename, globalFile.Modified)
|
||||
tmpFile, err := os.Create(tmpFilename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tmpFile.Close()
|
||||
|
||||
contentChan := make(chan content, 32)
|
||||
var applyDone sync.WaitGroup
|
||||
applyDone.Add(1)
|
||||
go func() {
|
||||
applyContent(contentChan, tmpFile)
|
||||
applyDone.Done()
|
||||
}()
|
||||
|
||||
local, remote := localFile.Blocks.To(globalFile.Blocks)
|
||||
var fetchDone sync.WaitGroup
|
||||
|
||||
// One local copy routing
|
||||
|
||||
fetchDone.Add(1)
|
||||
go func() {
|
||||
for _, block := range local {
|
||||
data, err := m.Request("<local>", name, block.Offset, block.Length, block.Hash)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
contentChan <- content{
|
||||
offset: int64(block.Offset),
|
||||
data: data,
|
||||
}
|
||||
}
|
||||
fetchDone.Done()
|
||||
}()
|
||||
|
||||
// N remote copy routines
|
||||
|
||||
m.RLock()
|
||||
var nodeIDs = m.whoHas(name)
|
||||
m.RUnlock()
|
||||
var remoteBlocksChan = make(chan Block)
|
||||
go func() {
|
||||
for _, block := range remote {
|
||||
remoteBlocksChan <- block
|
||||
}
|
||||
close(remoteBlocksChan)
|
||||
}()
|
||||
|
||||
// XXX: This should be rewritten into something nicer that takes differing
|
||||
// peer performance into account.
|
||||
|
||||
for i := 0; i < RemoteFetchers; i++ {
|
||||
for _, nodeID := range nodeIDs {
|
||||
fetchDone.Add(1)
|
||||
go func(nodeID string) {
|
||||
for block := range remoteBlocksChan {
|
||||
data, err := m.RequestGlobal(nodeID, name, block.Offset, block.Length, block.Hash)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
contentChan <- content{
|
||||
offset: int64(block.Offset),
|
||||
data: data,
|
||||
}
|
||||
}
|
||||
fetchDone.Done()
|
||||
}(nodeID)
|
||||
}
|
||||
}
|
||||
|
||||
fetchDone.Wait()
|
||||
close(contentChan)
|
||||
applyDone.Wait()
|
||||
|
||||
rf, err := os.Open(tmpFilename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rf.Close()
|
||||
|
||||
writtenBlocks, err := Blocks(rf, BlockSize)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(writtenBlocks) != len(globalFile.Blocks) {
|
||||
return fmt.Errorf("%s: incorrect number of blocks after sync", tmpFilename)
|
||||
}
|
||||
for i := range writtenBlocks {
|
||||
if bytes.Compare(writtenBlocks[i].Hash, globalFile.Blocks[i].Hash) != 0 {
|
||||
return fmt.Errorf("%s: hash mismatch after sync\n %v\n %v", tmpFilename, writtenBlocks[i], globalFile.Blocks[i])
|
||||
}
|
||||
}
|
||||
|
||||
err = os.Chtimes(tmpFilename, time.Unix(globalFile.Modified, 0), time.Unix(globalFile.Modified, 0))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = os.Rename(tmpFilename, filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Model) puller() {
|
||||
for {
|
||||
for {
|
||||
var n string
|
||||
var f File
|
||||
|
||||
m.RLock()
|
||||
for n = range m.need {
|
||||
break // just pick first name
|
||||
}
|
||||
if len(n) != 0 {
|
||||
f = m.global[n]
|
||||
}
|
||||
m.RUnlock()
|
||||
|
||||
if len(n) == 0 {
|
||||
// we got nothing
|
||||
break
|
||||
}
|
||||
|
||||
var err error
|
||||
if f.Flags&FlagDeleted == 0 {
|
||||
if traceFile {
|
||||
debugf("FILE: Pull %q", n)
|
||||
}
|
||||
err = m.pullFile(n)
|
||||
} else {
|
||||
if traceFile {
|
||||
debugf("FILE: Remove %q", n)
|
||||
}
|
||||
// Cheerfully ignore errors here
|
||||
_ = os.Remove(path.Join(m.dir, n))
|
||||
}
|
||||
if err == nil {
|
||||
m.UpdateLocal(f)
|
||||
} else {
|
||||
warnln(err)
|
||||
}
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
type content struct {
|
||||
offset int64
|
||||
data []byte
|
||||
}
|
||||
|
||||
func applyContent(cc <-chan content, dst io.WriterAt) error {
|
||||
var err error
|
||||
|
||||
for c := range cc {
|
||||
_, err = dst.WriteAt(c.data, c.offset)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
buffers.Put(c.data)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
275
model_test.go
Normal file
275
model_test.go
Normal file
@ -0,0 +1,275 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/calmh/syncthing/protocol"
|
||||
)
|
||||
|
||||
func TestNewModel(t *testing.T) {
|
||||
m := NewModel("foo")
|
||||
|
||||
if m == nil {
|
||||
t.Fatalf("NewModel returned nil")
|
||||
}
|
||||
|
||||
if len(m.need) > 0 {
|
||||
t.Errorf("New model should have no Need")
|
||||
}
|
||||
|
||||
if len(m.local) > 0 {
|
||||
t.Errorf("New model should have no Have")
|
||||
}
|
||||
}
|
||||
|
||||
var testDataExpected = map[string]File{
|
||||
"foo": File{
|
||||
Name: "foo",
|
||||
Flags: 0644,
|
||||
Modified: 1384244572,
|
||||
Blocks: []Block{{Offset: 0x0, Length: 0x7, Hash: []uint8{0xae, 0xc0, 0x70, 0x64, 0x5f, 0xe5, 0x3e, 0xe3, 0xb3, 0x76, 0x30, 0x59, 0x37, 0x61, 0x34, 0xf0, 0x58, 0xcc, 0x33, 0x72, 0x47, 0xc9, 0x78, 0xad, 0xd1, 0x78, 0xb6, 0xcc, 0xdf, 0xb0, 0x1, 0x9f}}},
|
||||
},
|
||||
"bar": File{
|
||||
Name: "bar",
|
||||
Flags: 0644,
|
||||
Modified: 1384244579,
|
||||
Blocks: []Block{{Offset: 0x0, Length: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}},
|
||||
},
|
||||
"baz/quux": File{
|
||||
Name: "baz/quux",
|
||||
Flags: 0644,
|
||||
Modified: 1384244676,
|
||||
Blocks: []Block{{Offset: 0x0, Length: 0x9, Hash: []uint8{0xc1, 0x54, 0xd9, 0x4e, 0x94, 0xba, 0x72, 0x98, 0xa6, 0xad, 0xb0, 0x52, 0x3a, 0xfe, 0x34, 0xd1, 0xb6, 0xa5, 0x81, 0xd6, 0xb8, 0x93, 0xa7, 0x63, 0xd4, 0x5d, 0xdc, 0x5e, 0x20, 0x9d, 0xcb, 0x83}}},
|
||||
},
|
||||
}
|
||||
|
||||
func TestUpdateLocal(t *testing.T) {
|
||||
m := NewModel("foo")
|
||||
fs := Walk("testdata", m)
|
||||
m.ReplaceLocal(fs)
|
||||
|
||||
if len(m.need) > 0 {
|
||||
t.Fatalf("Model with only local data should have no need")
|
||||
}
|
||||
|
||||
if l1, l2 := len(m.local), len(testDataExpected); l1 != l2 {
|
||||
t.Fatalf("Model len(local) incorrect, %d != %d", l1, l2)
|
||||
}
|
||||
if l1, l2 := len(m.global), len(testDataExpected); l1 != l2 {
|
||||
t.Fatalf("Model len(global) incorrect, %d != %d", l1, l2)
|
||||
}
|
||||
for name, file := range testDataExpected {
|
||||
if f, ok := m.local[name]; ok {
|
||||
if !reflect.DeepEqual(f, file) {
|
||||
t.Errorf("Incorrect local\n%v !=\n%v\nfor file %q", f, file, name)
|
||||
}
|
||||
} else {
|
||||
t.Errorf("Missing file %q in local table", name)
|
||||
}
|
||||
if f, ok := m.global[name]; ok {
|
||||
if !reflect.DeepEqual(f, file) {
|
||||
t.Errorf("Incorrect global\n%v !=\n%v\nfor file %q", f, file, name)
|
||||
}
|
||||
} else {
|
||||
t.Errorf("Missing file %q in global table", name)
|
||||
}
|
||||
}
|
||||
|
||||
for _, f := range fs {
|
||||
if hf, ok := m.local[f.Name]; !ok || hf.Modified != f.Modified {
|
||||
t.Fatalf("Incorrect local for %q", f.Name)
|
||||
}
|
||||
if cf, ok := m.global[f.Name]; !ok || cf.Modified != f.Modified {
|
||||
t.Fatalf("Incorrect global for %q", f.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteUpdateExisting(t *testing.T) {
|
||||
m := NewModel("foo")
|
||||
fs := Walk("testdata", m)
|
||||
m.ReplaceLocal(fs)
|
||||
|
||||
newFile := protocol.FileInfo{
|
||||
Name: "foo",
|
||||
Modified: time.Now().Unix(),
|
||||
Blocks: []protocol.BlockInfo{{100, []byte("some hash bytes")}},
|
||||
}
|
||||
m.Index(string("42"), []protocol.FileInfo{newFile})
|
||||
|
||||
if l := len(m.need); l != 1 {
|
||||
t.Errorf("Model missing Need for one file (%d != 1)", l)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteAddNew(t *testing.T) {
|
||||
m := NewModel("foo")
|
||||
fs := Walk("testdata", m)
|
||||
m.ReplaceLocal(fs)
|
||||
|
||||
newFile := protocol.FileInfo{
|
||||
Name: "a new file",
|
||||
Modified: time.Now().Unix(),
|
||||
Blocks: []protocol.BlockInfo{{100, []byte("some hash bytes")}},
|
||||
}
|
||||
m.Index(string("42"), []protocol.FileInfo{newFile})
|
||||
|
||||
if l1, l2 := len(m.need), 1; l1 != l2 {
|
||||
t.Errorf("Model len(m.need) incorrect (%d != %d)", l1, l2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteUpdateOld(t *testing.T) {
|
||||
m := NewModel("foo")
|
||||
fs := Walk("testdata", m)
|
||||
m.ReplaceLocal(fs)
|
||||
|
||||
oldTimeStamp := int64(1234)
|
||||
newFile := protocol.FileInfo{
|
||||
Name: "foo",
|
||||
Modified: oldTimeStamp,
|
||||
Blocks: []protocol.BlockInfo{{100, []byte("some hash bytes")}},
|
||||
}
|
||||
m.Index(string("42"), []protocol.FileInfo{newFile})
|
||||
|
||||
if l1, l2 := len(m.need), 0; l1 != l2 {
|
||||
t.Errorf("Model len(need) incorrect (%d != %d)", l1, l2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelete(t *testing.T) {
|
||||
m := NewModel("foo")
|
||||
fs := Walk("testdata", m)
|
||||
m.ReplaceLocal(fs)
|
||||
|
||||
if l1, l2 := len(m.local), len(fs); l1 != l2 {
|
||||
t.Errorf("Model len(local) incorrect (%d != %d)", l1, l2)
|
||||
}
|
||||
if l1, l2 := len(m.global), len(fs); l1 != l2 {
|
||||
t.Errorf("Model len(global) incorrect (%d != %d)", l1, l2)
|
||||
}
|
||||
|
||||
ot := time.Now().Unix()
|
||||
newFile := File{
|
||||
Name: "a new file",
|
||||
Modified: ot,
|
||||
Blocks: []Block{{0, 100, []byte("some hash bytes")}},
|
||||
}
|
||||
m.UpdateLocal(newFile)
|
||||
|
||||
if l1, l2 := len(m.local), len(fs)+1; l1 != l2 {
|
||||
t.Errorf("Model len(local) incorrect (%d != %d)", l1, l2)
|
||||
}
|
||||
if l1, l2 := len(m.global), len(fs)+1; l1 != l2 {
|
||||
t.Errorf("Model len(global) incorrect (%d != %d)", l1, l2)
|
||||
}
|
||||
|
||||
// The deleted file is kept in the local and global tables and marked as deleted.
|
||||
|
||||
m.ReplaceLocal(fs)
|
||||
|
||||
if l1, l2 := len(m.local), len(fs)+1; l1 != l2 {
|
||||
t.Errorf("Model len(local) incorrect (%d != %d)", l1, l2)
|
||||
}
|
||||
if l1, l2 := len(m.global), len(fs)+1; l1 != l2 {
|
||||
t.Errorf("Model len(global) incorrect (%d != %d)", l1, l2)
|
||||
}
|
||||
|
||||
if m.local["a new file"].Flags&(1<<12) == 0 {
|
||||
t.Error("Unexpected deleted flag = 0 in local table")
|
||||
}
|
||||
if len(m.local["a new file"].Blocks) != 0 {
|
||||
t.Error("Unexpected non-zero blocks for deleted file in local")
|
||||
}
|
||||
if ft := m.local["a new file"].Modified; ft != ot+1 {
|
||||
t.Errorf("Unexpected time %d != %d for deleted file in local", ft, ot+1)
|
||||
}
|
||||
|
||||
if m.global["a new file"].Flags&(1<<12) == 0 {
|
||||
t.Error("Unexpected deleted flag = 0 in global table")
|
||||
}
|
||||
if len(m.global["a new file"].Blocks) != 0 {
|
||||
t.Error("Unexpected non-zero blocks for deleted file in global")
|
||||
}
|
||||
if ft := m.local["a new file"].Modified; ft != ot+1 {
|
||||
t.Errorf("Unexpected time %d != %d for deleted file in local", ft, ot+1)
|
||||
}
|
||||
|
||||
// Another update should change nothing
|
||||
|
||||
m.ReplaceLocal(fs)
|
||||
|
||||
if l1, l2 := len(m.local), len(fs)+1; l1 != l2 {
|
||||
t.Errorf("Model len(local) incorrect (%d != %d)", l1, l2)
|
||||
}
|
||||
if l1, l2 := len(m.global), len(fs)+1; l1 != l2 {
|
||||
t.Errorf("Model len(global) incorrect (%d != %d)", l1, l2)
|
||||
}
|
||||
|
||||
if m.local["a new file"].Flags&(1<<12) == 0 {
|
||||
t.Error("Unexpected deleted flag = 0 in local table")
|
||||
}
|
||||
if len(m.local["a new file"].Blocks) != 0 {
|
||||
t.Error("Unexpected non-zero blocks for deleted file in local")
|
||||
}
|
||||
if ft := m.local["a new file"].Modified; ft != ot+1 {
|
||||
t.Errorf("Unexpected time %d != %d for deleted file in local", ft, ot+1)
|
||||
}
|
||||
|
||||
if m.global["a new file"].Flags&(1<<12) == 0 {
|
||||
t.Error("Unexpected deleted flag = 0 in global table")
|
||||
}
|
||||
if len(m.global["a new file"].Blocks) != 0 {
|
||||
t.Error("Unexpected non-zero blocks for deleted file in global")
|
||||
}
|
||||
if ft := m.local["a new file"].Modified; ft != ot+1 {
|
||||
t.Errorf("Unexpected time %d != %d for deleted file in local", ft, ot+1)
|
||||
}
|
||||
}
|
||||
|
||||
func TestForgetNode(t *testing.T) {
|
||||
m := NewModel("foo")
|
||||
fs := Walk("testdata", m)
|
||||
m.ReplaceLocal(fs)
|
||||
|
||||
if l1, l2 := len(m.local), len(fs); l1 != l2 {
|
||||
t.Errorf("Model len(local) incorrect (%d != %d)", l1, l2)
|
||||
}
|
||||
if l1, l2 := len(m.global), len(fs); l1 != l2 {
|
||||
t.Errorf("Model len(global) incorrect (%d != %d)", l1, l2)
|
||||
}
|
||||
if l1, l2 := len(m.need), 0; l1 != l2 {
|
||||
t.Errorf("Model len(need) incorrect (%d != %d)", l1, l2)
|
||||
}
|
||||
|
||||
newFile := protocol.FileInfo{
|
||||
Name: "new file",
|
||||
Modified: time.Now().Unix(),
|
||||
Blocks: []protocol.BlockInfo{{100, []byte("some hash bytes")}},
|
||||
}
|
||||
m.Index(string("42"), []protocol.FileInfo{newFile})
|
||||
|
||||
if l1, l2 := len(m.local), len(fs); l1 != l2 {
|
||||
t.Errorf("Model len(local) incorrect (%d != %d)", l1, l2)
|
||||
}
|
||||
if l1, l2 := len(m.global), len(fs)+1; l1 != l2 {
|
||||
t.Errorf("Model len(global) incorrect (%d != %d)", l1, l2)
|
||||
}
|
||||
if l1, l2 := len(m.need), 1; l1 != l2 {
|
||||
t.Errorf("Model len(need) incorrect (%d != %d)", l1, l2)
|
||||
}
|
||||
|
||||
m.Close(string("42"))
|
||||
|
||||
if l1, l2 := len(m.local), len(fs); l1 != l2 {
|
||||
t.Errorf("Model len(local) incorrect (%d != %d)", l1, l2)
|
||||
}
|
||||
if l1, l2 := len(m.global), len(fs); l1 != l2 {
|
||||
t.Errorf("Model len(global) incorrect (%d != %d)", l1, l2)
|
||||
}
|
||||
if l1, l2 := len(m.need), 0; l1 != l2 {
|
||||
t.Errorf("Model len(need) incorrect (%d != %d)", l1, l2)
|
||||
}
|
||||
}
|
208
protocol/PROTOCOL.md
Normal file
208
protocol/PROTOCOL.md
Normal file
@ -0,0 +1,208 @@
|
||||
Block Exchange Protocol v1.0
|
||||
============================
|
||||
|
||||
Introduction and Definitions
|
||||
----------------------------
|
||||
|
||||
The BEP is used between two or more _nodes_ thus forming a _cluster_.
|
||||
Each node has a _repository_ of files described by the _local model_,
|
||||
containing modifications times and block hashes. The local model is sent
|
||||
to the other nodes in the cluster. The union of all files in the local
|
||||
models, with files selected for most recent modification time, forms the
|
||||
_global model_. Each node strives to get it's repository in synch with
|
||||
the global model by requesting missing blocks from the other nodes.
|
||||
|
||||
Transport and Authentication
|
||||
----------------------------
|
||||
|
||||
The BEP itself does not provide retransmissions, compression, encryption
|
||||
nor authentication. It is expected that this is performed at lower
|
||||
layers of the networking stack. A typical deployment stack should be
|
||||
similar to the following:
|
||||
|
||||
|-----------------------------|
|
||||
| Block Exchange Protocol |
|
||||
|-----------------------------|
|
||||
| Compression (RFC 1951) |
|
||||
|-----------------------------|
|
||||
| Encryption & Auth (TLS 1.0) |
|
||||
|-----------------------------|
|
||||
| TCP |
|
||||
|-----------------------------|
|
||||
v v
|
||||
|
||||
The exact nature of the authentication is up to the application.
|
||||
Possibilities include certificates signed by a common trusted CA,
|
||||
preshared certificates, preshared certificate fingerprints or
|
||||
certificate pinning combined with some out of band first verification.
|
||||
|
||||
There is no required order or synchronization among BEP messages - any
|
||||
message type may be sent at any time and the sender need not await a
|
||||
response to one message before sending another. Responses must however
|
||||
be sent in the same order as the requests are received.
|
||||
|
||||
Compression is started directly after a successfull TLS handshake,
|
||||
before the first message is sent. The compression is flushed at each
|
||||
message boundary.
|
||||
|
||||
Messages
|
||||
--------
|
||||
|
||||
Every message starts with one 32 bit word indicating the message version
|
||||
and type. For BEP v1.0 the Version field is set to zero. Future versions
|
||||
with incompatible message formats will increment the Version field. The
|
||||
reserved bits must be set to zero.
|
||||
|
||||
0 1 2 3
|
||||
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
|
||||
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|
||||
| Ver=0 | Message ID | Type | Reserved |
|
||||
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|
||||
|
||||
All data following the message header is in XDR (RFC 1014) encoding.
|
||||
The actual data types in use by BEP, in XDR naming convention, are:
|
||||
|
||||
- unsigned int -- unsigned 32 bit integer
|
||||
- hyper -- signed 64 bit integer
|
||||
- unsigned hyper -- signed 64 bit integer
|
||||
- opaque<> -- variable length opaque data
|
||||
- string<> -- variable length string
|
||||
|
||||
The encoding of opaque<> and string<> are identical, the distinction is
|
||||
solely in interpretation. Opaque data should not be interpreted as such,
|
||||
but can be compared bytewise to other opaque data. All strings use the
|
||||
UTF-8 encoding.
|
||||
|
||||
### Index (Type = 1)
|
||||
|
||||
The Index message defines the contents of the senders repository. A Index
|
||||
message is sent by each peer immediately upon connection and whenever the
|
||||
local repository contents changes. However, if a peer has no data to
|
||||
advertise (the repository is empty, or it is set to only import data) it
|
||||
is allowed but not required to send an empty Index message (a file list of
|
||||
zero length). If the repository contents change from non-empty to empty,
|
||||
an empty Index message must be sent. There is no response to the Index
|
||||
message.
|
||||
|
||||
struct IndexMessage {
|
||||
FileInfo Files<>;
|
||||
}
|
||||
|
||||
struct FileInfo {
|
||||
string Name<>;
|
||||
unsigned int Flags;
|
||||
hyper Modified;
|
||||
BlockInfo Blocks<>;
|
||||
}
|
||||
|
||||
struct BlockInfo {
|
||||
unsigned int Length;
|
||||
opaque Hash<>
|
||||
}
|
||||
|
||||
The file name is the part relative to the repository root. The
|
||||
modification time is expressed as the number of seconds since the Unix
|
||||
Epoch. The hash algorithm is implied by the hash length. Currently, the
|
||||
hash must be 32 bytes long and computed by SHA256.
|
||||
|
||||
The flags field is made up of the following single bit flags:
|
||||
|
||||
0 1 2 3
|
||||
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
|
||||
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|
||||
| Reserved |D| Unix Perm. & Mode |
|
||||
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|
||||
|
||||
- The lower 12 bits hold the common Unix permission and mode bits.
|
||||
|
||||
- Bit 19 ("D") is set when the file has been deleted. The block list
|
||||
shall contain zero blocks and the modification time indicates the
|
||||
time of deletion or, if deletion time is not reliably determinable,
|
||||
one second past the last know modification time.
|
||||
|
||||
- Bit 0 through 18 are reserved for future use and shall be set to
|
||||
zero.
|
||||
|
||||
### Request (Type = 2)
|
||||
|
||||
The Request message expresses the desire to receive a data block
|
||||
corresponding to a part of a certain file in the peer's repository.
|
||||
|
||||
The requested block must correspond exactly to one block seen in the
|
||||
peer's Index message. The hash field must be set to the expected value by
|
||||
the sender. The receiver may validate that this is actually the case
|
||||
before transmitting data. Each Request message must be met with a Response
|
||||
message.
|
||||
|
||||
struct RequestMessage {
|
||||
string Name<>;
|
||||
unsigned hyper Offset;
|
||||
unsigned int Length;
|
||||
opaque Hash<>;
|
||||
}
|
||||
|
||||
The hash algorithm is implied by the hash length. Currently, the hash
|
||||
must be 32 bytes long and computed by SHA256.
|
||||
|
||||
The Message ID in the header must set to a unique value to be able to
|
||||
correlate the request with the response message.
|
||||
|
||||
### Response (Type = 3)
|
||||
|
||||
The Response message is sent in response to a Request message. In case the
|
||||
requested data was not available (an outdated block was requested, or
|
||||
the file has been deleted), the Data field is empty.
|
||||
|
||||
struct ResponseMessage {
|
||||
opaque Data<>
|
||||
}
|
||||
|
||||
The Message ID in the header is used to correlate requests and
|
||||
responses.
|
||||
|
||||
### Ping (Type = 4)
|
||||
|
||||
The Ping message is used to determine that a connection is alive, and to
|
||||
keep connections alive through state tracking network elements such as
|
||||
firewalls and NAT gateways. The Ping message has no contents.
|
||||
|
||||
struct PingMessage {
|
||||
}
|
||||
|
||||
### Pong (Type = 5)
|
||||
|
||||
The Pong message is sent in response to a Ping. The Pong message has no
|
||||
contents, but copies the Message ID from the Ping.
|
||||
|
||||
struct PongMessage {
|
||||
}
|
||||
|
||||
Example Exchange
|
||||
----------------
|
||||
|
||||
A B
|
||||
1. Index-> <-Index
|
||||
2. Request->
|
||||
3. Request->
|
||||
4. Request->
|
||||
5. Request->
|
||||
6. <-Response
|
||||
7. <-Response
|
||||
8. <-Response
|
||||
9. <-Response
|
||||
10. Index->
|
||||
...
|
||||
11. Ping->
|
||||
12. <-Pong
|
||||
|
||||
The connection is established and at 1. both peers send Index records.
|
||||
The Index records are received and both peers recompute their knowledge
|
||||
of the data in the cluster. In this example, peer A has four missing or
|
||||
outdated blocks. At 2 through 5 peer A sends requests for these blocks.
|
||||
The requests are received by peer B, who retrieves the data from the
|
||||
repository and transmits Response records (6 through 9). Node A updates
|
||||
their repository contents and transmits an updated Index message (10).
|
||||
Both peers enter idle state after 10. At some later time 11, peer A
|
||||
determines that it has not seen data from B for some time and sends a
|
||||
Ping request. A response is sent at 12.
|
||||
|
119
protocol/marshal.go
Normal file
119
protocol/marshal.go
Normal file
@ -0,0 +1,119 @@
|
||||
package protocol
|
||||
|
||||
import (
|
||||
"io"
|
||||
|
||||
"github.com/calmh/syncthing/buffers"
|
||||
)
|
||||
|
||||
func pad(l int) int {
|
||||
d := l % 4
|
||||
if d == 0 {
|
||||
return 0
|
||||
}
|
||||
return 4 - d
|
||||
}
|
||||
|
||||
var padBytes = []byte{0, 0, 0}
|
||||
|
||||
type marshalWriter struct {
|
||||
w io.Writer
|
||||
tot int
|
||||
err error
|
||||
}
|
||||
|
||||
func (w *marshalWriter) writeString(s string) {
|
||||
w.writeBytes([]byte(s))
|
||||
}
|
||||
|
||||
func (w *marshalWriter) writeBytes(bs []byte) {
|
||||
if w.err != nil {
|
||||
return
|
||||
}
|
||||
w.writeUint32(uint32(len(bs)))
|
||||
if w.err != nil {
|
||||
return
|
||||
}
|
||||
_, w.err = w.w.Write(bs)
|
||||
if p := pad(len(bs)); p > 0 {
|
||||
w.w.Write(padBytes[:p])
|
||||
}
|
||||
w.tot += len(bs) + pad(len(bs))
|
||||
}
|
||||
|
||||
func (w *marshalWriter) writeUint32(v uint32) {
|
||||
if w.err != nil {
|
||||
return
|
||||
}
|
||||
var b [4]byte
|
||||
b[0] = byte(v >> 24)
|
||||
b[1] = byte(v >> 16)
|
||||
b[2] = byte(v >> 8)
|
||||
b[3] = byte(v)
|
||||
_, w.err = w.w.Write(b[:])
|
||||
w.tot += 4
|
||||
}
|
||||
|
||||
func (w *marshalWriter) writeUint64(v uint64) {
|
||||
if w.err != nil {
|
||||
return
|
||||
}
|
||||
var b [8]byte
|
||||
b[0] = byte(v >> 56)
|
||||
b[1] = byte(v >> 48)
|
||||
b[2] = byte(v >> 40)
|
||||
b[3] = byte(v >> 32)
|
||||
b[4] = byte(v >> 24)
|
||||
b[5] = byte(v >> 16)
|
||||
b[6] = byte(v >> 8)
|
||||
b[7] = byte(v)
|
||||
_, w.err = w.w.Write(b[:])
|
||||
w.tot += 8
|
||||
}
|
||||
|
||||
type marshalReader struct {
|
||||
r io.Reader
|
||||
tot int
|
||||
err error
|
||||
}
|
||||
|
||||
func (r *marshalReader) readString() string {
|
||||
bs := r.readBytes()
|
||||
defer buffers.Put(bs)
|
||||
return string(bs)
|
||||
}
|
||||
|
||||
func (r *marshalReader) readBytes() []byte {
|
||||
if r.err != nil {
|
||||
return nil
|
||||
}
|
||||
l := int(r.readUint32())
|
||||
if r.err != nil {
|
||||
return nil
|
||||
}
|
||||
b := buffers.Get(l + pad(l))
|
||||
_, r.err = io.ReadFull(r.r, b)
|
||||
r.tot += int(l + pad(l))
|
||||
return b[:l]
|
||||
}
|
||||
|
||||
func (r *marshalReader) readUint32() uint32 {
|
||||
if r.err != nil {
|
||||
return 0
|
||||
}
|
||||
var b [4]byte
|
||||
_, r.err = io.ReadFull(r.r, b[:])
|
||||
r.tot += 4
|
||||
return uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24
|
||||
}
|
||||
|
||||
func (r *marshalReader) readUint64() uint64 {
|
||||
if r.err != nil {
|
||||
return 0
|
||||
}
|
||||
var b [8]byte
|
||||
_, r.err = io.ReadFull(r.r, b[:])
|
||||
r.tot += 8
|
||||
return uint64(b[7]) | uint64(b[6])<<8 | uint64(b[5])<<16 | uint64(b[4])<<24 |
|
||||
uint64(b[3])<<32 | uint64(b[2])<<40 | uint64(b[1])<<48 | uint64(b[0])<<56
|
||||
}
|
106
protocol/messages.go
Normal file
106
protocol/messages.go
Normal file
@ -0,0 +1,106 @@
|
||||
package protocol
|
||||
|
||||
import "io"
|
||||
|
||||
type request struct {
|
||||
name string
|
||||
offset uint64
|
||||
size uint32
|
||||
hash []byte
|
||||
}
|
||||
|
||||
type header struct {
|
||||
version int
|
||||
msgID int
|
||||
msgType int
|
||||
}
|
||||
|
||||
func encodeHeader(h header) uint32 {
|
||||
return uint32(h.version&0xf)<<28 +
|
||||
uint32(h.msgID&0xfff)<<16 +
|
||||
uint32(h.msgType&0xff)<<8
|
||||
}
|
||||
|
||||
func decodeHeader(u uint32) header {
|
||||
return header{
|
||||
version: int(u>>28) & 0xf,
|
||||
msgID: int(u>>16) & 0xfff,
|
||||
msgType: int(u>>8) & 0xff,
|
||||
}
|
||||
}
|
||||
|
||||
func (w *marshalWriter) writeHeader(h header) {
|
||||
w.writeUint32(encodeHeader(h))
|
||||
}
|
||||
|
||||
func (w *marshalWriter) writeIndex(idx []FileInfo) {
|
||||
w.writeUint32(uint32(len(idx)))
|
||||
for _, f := range idx {
|
||||
w.writeString(f.Name)
|
||||
w.writeUint32(f.Flags)
|
||||
w.writeUint64(uint64(f.Modified))
|
||||
w.writeUint32(uint32(len(f.Blocks)))
|
||||
for _, b := range f.Blocks {
|
||||
w.writeUint32(b.Length)
|
||||
w.writeBytes(b.Hash)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func WriteIndex(w io.Writer, idx []FileInfo) (int, error) {
|
||||
mw := marshalWriter{w, 0, nil}
|
||||
mw.writeIndex(idx)
|
||||
return mw.tot, mw.err
|
||||
}
|
||||
|
||||
func (w *marshalWriter) writeRequest(r request) {
|
||||
w.writeString(r.name)
|
||||
w.writeUint64(r.offset)
|
||||
w.writeUint32(r.size)
|
||||
w.writeBytes(r.hash)
|
||||
}
|
||||
|
||||
func (w *marshalWriter) writeResponse(data []byte) {
|
||||
w.writeBytes(data)
|
||||
}
|
||||
|
||||
func (r *marshalReader) readHeader() header {
|
||||
return decodeHeader(r.readUint32())
|
||||
}
|
||||
|
||||
func (r *marshalReader) readIndex() []FileInfo {
|
||||
nfiles := r.readUint32()
|
||||
files := make([]FileInfo, nfiles)
|
||||
for i := range files {
|
||||
files[i].Name = r.readString()
|
||||
files[i].Flags = r.readUint32()
|
||||
files[i].Modified = int64(r.readUint64())
|
||||
nblocks := r.readUint32()
|
||||
blocks := make([]BlockInfo, nblocks)
|
||||
for j := range blocks {
|
||||
blocks[j].Length = r.readUint32()
|
||||
blocks[j].Hash = r.readBytes()
|
||||
}
|
||||
files[i].Blocks = blocks
|
||||
}
|
||||
return files
|
||||
}
|
||||
|
||||
func ReadIndex(r io.Reader) ([]FileInfo, error) {
|
||||
mr := marshalReader{r, 0, nil}
|
||||
idx := mr.readIndex()
|
||||
return idx, mr.err
|
||||
}
|
||||
|
||||
func (r *marshalReader) readRequest() request {
|
||||
var req request
|
||||
req.name = r.readString()
|
||||
req.offset = r.readUint64()
|
||||
req.size = r.readUint32()
|
||||
req.hash = r.readBytes()
|
||||
return req
|
||||
}
|
||||
|
||||
func (r *marshalReader) readResponse() []byte {
|
||||
return r.readBytes()
|
||||
}
|
115
protocol/messages_test.go
Normal file
115
protocol/messages_test.go
Normal file
@ -0,0 +1,115 @@
|
||||
package protocol
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io/ioutil"
|
||||
"reflect"
|
||||
"testing"
|
||||
"testing/quick"
|
||||
)
|
||||
|
||||
func TestIndex(t *testing.T) {
|
||||
idx := []FileInfo{
|
||||
{
|
||||
"Foo",
|
||||
0755,
|
||||
1234567890,
|
||||
[]BlockInfo{
|
||||
{12345678, []byte("hash hash hash")},
|
||||
{23456781, []byte("ash hash hashh")},
|
||||
{34567812, []byte("sh hash hashha")},
|
||||
},
|
||||
}, {
|
||||
"Quux/Quux",
|
||||
0644,
|
||||
2345678901,
|
||||
[]BlockInfo{
|
||||
{45678123, []byte("4321 hash hash hash")},
|
||||
{56781234, []byte("3214 ash hash hashh")},
|
||||
{67812345, []byte("2143 sh hash hashha")},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
var buf = new(bytes.Buffer)
|
||||
var wr = marshalWriter{buf, 0, nil}
|
||||
wr.writeIndex(idx)
|
||||
|
||||
var rd = marshalReader{buf, 0, nil}
|
||||
var idx2 = rd.readIndex()
|
||||
|
||||
if !reflect.DeepEqual(idx, idx2) {
|
||||
t.Errorf("Index marshal error:\n%#v\n%#v\n", idx, idx2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequest(t *testing.T) {
|
||||
f := func(name string, offset uint64, size uint32, hash []byte) bool {
|
||||
var buf = new(bytes.Buffer)
|
||||
var req = request{name, offset, size, hash}
|
||||
var wr = marshalWriter{buf, 0, nil}
|
||||
wr.writeRequest(req)
|
||||
var rd = marshalReader{buf, 0, nil}
|
||||
var req2 = rd.readRequest()
|
||||
return req.name == req2.name &&
|
||||
req.offset == req2.offset &&
|
||||
req.size == req2.size &&
|
||||
bytes.Compare(req.hash, req2.hash) == 0
|
||||
}
|
||||
if err := quick.Check(f, nil); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResponse(t *testing.T) {
|
||||
f := func(data []byte) bool {
|
||||
var buf = new(bytes.Buffer)
|
||||
var wr = marshalWriter{buf, 0, nil}
|
||||
wr.writeResponse(data)
|
||||
var rd = marshalReader{buf, 0, nil}
|
||||
var read = rd.readResponse()
|
||||
return bytes.Compare(read, data) == 0
|
||||
}
|
||||
if err := quick.Check(f, nil); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkWriteIndex(b *testing.B) {
|
||||
idx := []FileInfo{
|
||||
{
|
||||
"Foo",
|
||||
0777,
|
||||
1234567890,
|
||||
[]BlockInfo{
|
||||
{12345678, []byte("hash hash hash")},
|
||||
{23456781, []byte("ash hash hashh")},
|
||||
{34567812, []byte("sh hash hashha")},
|
||||
},
|
||||
}, {
|
||||
"Quux/Quux",
|
||||
0644,
|
||||
2345678901,
|
||||
[]BlockInfo{
|
||||
{45678123, []byte("4321 hash hash hash")},
|
||||
{56781234, []byte("3214 ash hash hashh")},
|
||||
{67812345, []byte("2143 sh hash hashha")},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
var wr = marshalWriter{ioutil.Discard, 0, nil}
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
wr.writeIndex(idx)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkWriteRequest(b *testing.B) {
|
||||
var req = request{"blah blah", 1231323, 13123123, []byte("hash hash hash")}
|
||||
var wr = marshalWriter{ioutil.Discard, 0, nil}
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
wr.writeRequest(req)
|
||||
}
|
||||
}
|
239
protocol/protocol.go
Normal file
239
protocol/protocol.go
Normal file
@ -0,0 +1,239 @@
|
||||
package protocol
|
||||
|
||||
import (
|
||||
"compress/flate"
|
||||
"errors"
|
||||
"io"
|
||||
"sync"
|
||||
|
||||
"github.com/calmh/syncthing/buffers"
|
||||
)
|
||||
|
||||
const (
|
||||
messageTypeReserved = iota
|
||||
messageTypeIndex
|
||||
messageTypeRequest
|
||||
messageTypeResponse
|
||||
messageTypePing
|
||||
messageTypePong
|
||||
)
|
||||
|
||||
var ErrClosed = errors.New("Connection closed")
|
||||
|
||||
type FileInfo struct {
|
||||
Name string
|
||||
Flags uint32
|
||||
Modified int64
|
||||
Blocks []BlockInfo
|
||||
}
|
||||
|
||||
type BlockInfo struct {
|
||||
Length uint32
|
||||
Hash []byte
|
||||
}
|
||||
|
||||
type Model interface {
|
||||
// An index was received from the peer node
|
||||
Index(nodeID string, files []FileInfo)
|
||||
// A request was made by the peer node
|
||||
Request(nodeID, name string, offset uint64, size uint32, hash []byte) ([]byte, error)
|
||||
// The peer node closed the connection
|
||||
Close(nodeID string)
|
||||
}
|
||||
|
||||
type Connection struct {
|
||||
receiver Model
|
||||
reader io.Reader
|
||||
mreader *marshalReader
|
||||
writer io.Writer
|
||||
mwriter *marshalWriter
|
||||
wLock sync.RWMutex
|
||||
closed bool
|
||||
closedLock sync.RWMutex
|
||||
awaiting map[int]chan interface{}
|
||||
nextId int
|
||||
ID string
|
||||
}
|
||||
|
||||
func NewConnection(nodeID string, reader io.Reader, writer io.Writer, receiver Model) *Connection {
|
||||
flrd := flate.NewReader(reader)
|
||||
flwr, err := flate.NewWriter(writer, flate.BestSpeed)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
c := Connection{
|
||||
receiver: receiver,
|
||||
reader: flrd,
|
||||
mreader: &marshalReader{flrd, 0, nil},
|
||||
writer: flwr,
|
||||
mwriter: &marshalWriter{flwr, 0, nil},
|
||||
awaiting: make(map[int]chan interface{}),
|
||||
ID: nodeID,
|
||||
}
|
||||
|
||||
go c.readerLoop()
|
||||
|
||||
return &c
|
||||
}
|
||||
|
||||
// Index writes the list of file information to the connected peer node
|
||||
func (c *Connection) Index(idx []FileInfo) {
|
||||
c.wLock.Lock()
|
||||
defer c.wLock.Unlock()
|
||||
|
||||
c.mwriter.writeHeader(header{0, c.nextId, messageTypeIndex})
|
||||
c.nextId = (c.nextId + 1) & 0xfff
|
||||
c.mwriter.writeIndex(idx)
|
||||
c.flush()
|
||||
}
|
||||
|
||||
// Request returns the bytes for the specified block after fetching them from the connected peer.
|
||||
func (c *Connection) Request(name string, offset uint64, size uint32, hash []byte) ([]byte, error) {
|
||||
c.wLock.Lock()
|
||||
rc := make(chan interface{})
|
||||
c.awaiting[c.nextId] = rc
|
||||
c.mwriter.writeHeader(header{0, c.nextId, messageTypeRequest})
|
||||
c.mwriter.writeRequest(request{name, offset, size, hash})
|
||||
c.flush()
|
||||
c.nextId = (c.nextId + 1) & 0xfff
|
||||
c.wLock.Unlock()
|
||||
|
||||
// Reading something that might be nil from a possibly closed channel...
|
||||
// r0<~
|
||||
|
||||
var data []byte
|
||||
i, ok := <-rc
|
||||
if ok {
|
||||
if d, ok := i.([]byte); ok {
|
||||
data = d
|
||||
}
|
||||
}
|
||||
|
||||
var err error
|
||||
i, ok = <-rc
|
||||
if ok {
|
||||
if e, ok := i.(error); ok {
|
||||
err = e
|
||||
}
|
||||
}
|
||||
return data, err
|
||||
}
|
||||
|
||||
func (c *Connection) Ping() bool {
|
||||
c.wLock.Lock()
|
||||
rc := make(chan interface{})
|
||||
c.awaiting[c.nextId] = rc
|
||||
c.mwriter.writeHeader(header{0, c.nextId, messageTypePing})
|
||||
c.flush()
|
||||
c.nextId = (c.nextId + 1) & 0xfff
|
||||
c.wLock.Unlock()
|
||||
|
||||
_, ok := <-rc
|
||||
return ok
|
||||
}
|
||||
|
||||
func (c *Connection) Stop() {
|
||||
}
|
||||
|
||||
type flusher interface {
|
||||
Flush() error
|
||||
}
|
||||
|
||||
func (c *Connection) flush() {
|
||||
if f, ok := c.writer.(flusher); ok {
|
||||
f.Flush()
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Connection) close() {
|
||||
c.closedLock.Lock()
|
||||
c.closed = true
|
||||
c.closedLock.Unlock()
|
||||
c.wLock.Lock()
|
||||
for _, ch := range c.awaiting {
|
||||
close(ch)
|
||||
}
|
||||
c.awaiting = nil
|
||||
c.wLock.Unlock()
|
||||
c.receiver.Close(c.ID)
|
||||
}
|
||||
|
||||
func (c *Connection) isClosed() bool {
|
||||
c.closedLock.RLock()
|
||||
defer c.closedLock.RUnlock()
|
||||
return c.closed
|
||||
}
|
||||
|
||||
func (c *Connection) readerLoop() {
|
||||
for !c.isClosed() {
|
||||
hdr := c.mreader.readHeader()
|
||||
if c.mreader.err != nil {
|
||||
c.close()
|
||||
break
|
||||
}
|
||||
|
||||
switch hdr.msgType {
|
||||
case messageTypeIndex:
|
||||
files := c.mreader.readIndex()
|
||||
if c.mreader.err != nil {
|
||||
c.close()
|
||||
} else {
|
||||
c.receiver.Index(c.ID, files)
|
||||
}
|
||||
|
||||
case messageTypeRequest:
|
||||
c.processRequest(hdr.msgID)
|
||||
|
||||
case messageTypeResponse:
|
||||
data := c.mreader.readResponse()
|
||||
|
||||
if c.mreader.err != nil {
|
||||
c.close()
|
||||
} else {
|
||||
c.wLock.RLock()
|
||||
rc, ok := c.awaiting[hdr.msgID]
|
||||
c.wLock.RUnlock()
|
||||
|
||||
if ok {
|
||||
rc <- data
|
||||
rc <- c.mreader.err
|
||||
delete(c.awaiting, hdr.msgID)
|
||||
close(rc)
|
||||
}
|
||||
}
|
||||
|
||||
case messageTypePing:
|
||||
c.wLock.Lock()
|
||||
c.mwriter.writeUint32(encodeHeader(header{0, hdr.msgID, messageTypePong}))
|
||||
c.flush()
|
||||
c.wLock.Unlock()
|
||||
|
||||
case messageTypePong:
|
||||
c.wLock.Lock()
|
||||
if rc, ok := c.awaiting[hdr.msgID]; ok {
|
||||
rc <- true
|
||||
close(rc)
|
||||
delete(c.awaiting, hdr.msgID)
|
||||
}
|
||||
c.wLock.Unlock()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Connection) processRequest(msgID int) {
|
||||
req := c.mreader.readRequest()
|
||||
if c.mreader.err != nil {
|
||||
c.close()
|
||||
} else {
|
||||
go func() {
|
||||
data, _ := c.receiver.Request(c.ID, req.name, req.offset, req.size, req.hash)
|
||||
c.wLock.Lock()
|
||||
c.mwriter.writeUint32(encodeHeader(header{0, msgID, messageTypeResponse}))
|
||||
c.mwriter.writeResponse(data)
|
||||
buffers.Put(data)
|
||||
c.flush()
|
||||
c.wLock.Unlock()
|
||||
}()
|
||||
}
|
||||
}
|
37
protocol/protocol_test.go
Normal file
37
protocol/protocol_test.go
Normal file
@ -0,0 +1,37 @@
|
||||
package protocol
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"testing/quick"
|
||||
)
|
||||
|
||||
func TestHeaderFunctions(t *testing.T) {
|
||||
f := func(ver, id, typ int) bool {
|
||||
ver = int(uint(ver) % 16)
|
||||
id = int(uint(id) % 4096)
|
||||
typ = int(uint(typ) % 256)
|
||||
h0 := header{ver, id, typ}
|
||||
h1 := decodeHeader(encodeHeader(h0))
|
||||
return h0 == h1
|
||||
}
|
||||
if err := quick.Check(f, nil); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPad(t *testing.T) {
|
||||
tests := [][]int{
|
||||
{0, 0},
|
||||
{1, 3},
|
||||
{2, 2},
|
||||
{3, 1},
|
||||
{4, 0},
|
||||
{32, 0},
|
||||
{33, 3},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
if p := pad(tc[0]); p != tc[1] {
|
||||
t.Errorf("Incorrect padding for %d bytes, %d != %d", tc[0], p, tc[1])
|
||||
}
|
||||
}
|
||||
}
|
11
syncthing.ini
Normal file
11
syncthing.ini
Normal file
@ -0,0 +1,11 @@
|
||||
[repository]
|
||||
dir = /Users/jb/Synced
|
||||
|
||||
# The nodes section lists the nodes that make up the cluster. The format is
|
||||
# <certificate id> = <space separated list of addresses>
|
||||
# The special address "dynamic" means that outbound connections will not be
|
||||
# attempted, but inbound connections are accepted.
|
||||
|
||||
[nodes]
|
||||
ITZXTZ7A32DWV3NLNR5W4M3CHVBW56NA = 172.16.32.1:22000 192.23.34.56:22000
|
||||
CUGAE43Y5N64CRJU26YFH6MTWPSBLSUL = dynamic
|
1
testdata/bar
vendored
Normal file
1
testdata/bar
vendored
Normal file
@ -0,0 +1 @@
|
||||
foobarbaz
|
1
testdata/baz/quux
vendored
Normal file
1
testdata/baz/quux
vendored
Normal file
@ -0,0 +1 @@
|
||||
baazquux
|
1
testdata/foo
vendored
Normal file
1
testdata/foo
vendored
Normal file
@ -0,0 +1 @@
|
||||
foobar
|
68
tls.go
Normal file
68
tls.go
Normal file
@ -0,0 +1,68 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/sha1"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/base32"
|
||||
"encoding/pem"
|
||||
"math/big"
|
||||
"os"
|
||||
"path"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
tlsRSABits = 2048
|
||||
tlsName = "syncthing"
|
||||
)
|
||||
|
||||
func loadCert(dir string) (tls.Certificate, error) {
|
||||
return tls.LoadX509KeyPair(path.Join(dir, "cert.pem"), path.Join(dir, "key.pem"))
|
||||
}
|
||||
|
||||
func certId(bs []byte) string {
|
||||
hf := sha1.New()
|
||||
hf.Write(bs)
|
||||
id := hf.Sum(nil)
|
||||
return base32.StdEncoding.EncodeToString(id)
|
||||
}
|
||||
|
||||
func newCertificate(dir string) {
|
||||
priv, err := rsa.GenerateKey(rand.Reader, tlsRSABits)
|
||||
fatalErr(err)
|
||||
|
||||
notBefore := time.Now()
|
||||
notAfter := time.Date(2049, 12, 31, 23, 59, 59, 0, time.UTC)
|
||||
|
||||
template := x509.Certificate{
|
||||
SerialNumber: new(big.Int).SetInt64(0),
|
||||
Subject: pkix.Name{
|
||||
CommonName: tlsName,
|
||||
},
|
||||
NotBefore: notBefore,
|
||||
NotAfter: notAfter,
|
||||
|
||||
KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
|
||||
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
||||
BasicConstraintsValid: true,
|
||||
}
|
||||
|
||||
derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)
|
||||
fatalErr(err)
|
||||
|
||||
certOut, err := os.Create(path.Join(dir, "cert.pem"))
|
||||
fatalErr(err)
|
||||
pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
|
||||
certOut.Close()
|
||||
okln("wrote cert.pem")
|
||||
|
||||
keyOut, err := os.OpenFile(path.Join(dir, "key.pem"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
|
||||
fatalErr(err)
|
||||
pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)})
|
||||
keyOut.Close()
|
||||
okln("wrote key.pem")
|
||||
}
|
7
util.go
Normal file
7
util.go
Normal file
@ -0,0 +1,7 @@
|
||||
package main
|
||||
|
||||
import "time"
|
||||
|
||||
func timing(name string, t0 time.Time) {
|
||||
debugf("%s: %.02f ms", name, time.Since(t0).Seconds()*1000)
|
||||
}
|
117
walk.go
Normal file
117
walk.go
Normal file
@ -0,0 +1,117 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const BlockSize = 128 * 1024
|
||||
|
||||
type File struct {
|
||||
Name string
|
||||
Flags uint32
|
||||
Modified int64
|
||||
Blocks BlockList
|
||||
}
|
||||
|
||||
func (f File) Dump() {
|
||||
fmt.Printf("%s\n", f.Name)
|
||||
for _, b := range f.Blocks {
|
||||
fmt.Printf(" %dB @ %d: %x\n", b.Length, b.Offset, b.Hash)
|
||||
}
|
||||
fmt.Println()
|
||||
}
|
||||
|
||||
func isTempName(name string) bool {
|
||||
return strings.HasPrefix(path.Base(name), ".syncthing.")
|
||||
}
|
||||
|
||||
func tempName(name string, modified int64) string {
|
||||
tdir := path.Dir(name)
|
||||
tname := fmt.Sprintf(".syncthing.%s.%d", path.Base(name), modified)
|
||||
return path.Join(tdir, tname)
|
||||
}
|
||||
|
||||
func genWalker(base string, res *[]File, model *Model) filepath.WalkFunc {
|
||||
return func(p string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if isTempName(p) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if info.Mode()&os.ModeType == 0 {
|
||||
rn, err := filepath.Rel(base, p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fi, err := os.Stat(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
modified := fi.ModTime().Unix()
|
||||
|
||||
hf, ok := model.LocalFile(rn)
|
||||
if ok && hf.Modified == modified {
|
||||
// No change
|
||||
*res = append(*res, hf)
|
||||
} else {
|
||||
if traceFile {
|
||||
debugf("FILE: Hash %q", p)
|
||||
}
|
||||
fd, err := os.Open(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer fd.Close()
|
||||
|
||||
blocks, err := Blocks(fd, BlockSize)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f := File{
|
||||
Name: rn,
|
||||
Flags: uint32(info.Mode()),
|
||||
Modified: modified,
|
||||
Blocks: blocks,
|
||||
}
|
||||
*res = append(*res, f)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func Walk(dir string, model *Model) []File {
|
||||
var files []File
|
||||
fn := genWalker(dir, &files, model)
|
||||
err := filepath.Walk(dir, fn)
|
||||
if err != nil {
|
||||
warnln(err)
|
||||
}
|
||||
return files
|
||||
}
|
||||
|
||||
func cleanTempFile(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.Mode()&os.ModeType == 0 && isTempName(path) {
|
||||
if traceFile {
|
||||
debugf("FILE: Remove %q", path)
|
||||
}
|
||||
os.Remove(path)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func CleanTempFiles(dir string) {
|
||||
filepath.Walk(dir, cleanTempFile)
|
||||
}
|
42
walk_test.go
Normal file
42
walk_test.go
Normal file
@ -0,0 +1,42 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
var testdata = []struct {
|
||||
name string
|
||||
size int
|
||||
hash string
|
||||
}{
|
||||
{"bar", 10, "2f72cc11a6fcd0271ecef8c61056ee1eb1243be3805bf9a9df98f92f7636b05c"},
|
||||
{"baz/quux", 9, "c154d94e94ba7298a6adb0523afe34d1b6a581d6b893a763d45ddc5e209dcb83"},
|
||||
{"foo", 7, "aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f"},
|
||||
}
|
||||
|
||||
func TestWalk(t *testing.T) {
|
||||
m := new(Model)
|
||||
files := Walk("testdata", m)
|
||||
|
||||
if l1, l2 := len(files), len(testdata); l1 != l2 {
|
||||
t.Fatalf("Incorrect number of walked files %d != %d", l1, l2)
|
||||
}
|
||||
|
||||
for i := range testdata {
|
||||
if n1, n2 := testdata[i].name, files[i].Name; n1 != n2 {
|
||||
t.Errorf("Incorrect file name %q != %q for case #%d", n1, n2, i)
|
||||
}
|
||||
|
||||
if h1, h2 := fmt.Sprintf("%x", files[i].Blocks[0].Hash), testdata[i].hash; h1 != h2 {
|
||||
t.Errorf("Incorrect hash %q != %q for case #%d", h1, h2, i)
|
||||
}
|
||||
|
||||
t0 := time.Date(2010, 1, 1, 0, 0, 0, 0, time.UTC).Unix()
|
||||
t1 := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC).Unix()
|
||||
if mt := files[i].Modified; mt < t0 || mt > t1 {
|
||||
t.Errorf("Unrealistic modtime %d for test %d", mt, i)
|
||||
}
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue
Block a user