From f28367bcfc94f2648c99ca05403993c5d56ec3cd Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Mon, 22 Sep 2014 21:42:11 +0200 Subject: [PATCH 01/94] Move top level packages to internal. --- .gitignore | 1 + common_test.go | 76 ++++ counting.go | 64 +++ debug.go | 17 + doc.go | 6 + header.go | 45 ++ message.go | 139 ++++++ message_xdr.go | 964 +++++++++++++++++++++++++++++++++++++++++ nativemodel_darwin.go | 42 ++ nativemodel_unix.go | 33 ++ nativemodel_windows.go | 72 +++ nodeid.go | 159 +++++++ nodeid_test.go | 78 ++++ protocol.go | 640 +++++++++++++++++++++++++++ protocol_test.go | 383 ++++++++++++++++ wireformat.go | 58 +++ 16 files changed, 2777 insertions(+) create mode 100644 .gitignore create mode 100644 common_test.go create mode 100644 counting.go create mode 100644 debug.go create mode 100644 doc.go create mode 100644 header.go create mode 100644 message.go create mode 100644 message_xdr.go create mode 100644 nativemodel_darwin.go create mode 100644 nativemodel_unix.go create mode 100644 nativemodel_windows.go create mode 100644 nodeid.go create mode 100644 nodeid_test.go create mode 100644 protocol.go create mode 100644 protocol_test.go create mode 100644 wireformat.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..2211df63d --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.txt diff --git a/common_test.go b/common_test.go new file mode 100644 index 000000000..9d387825c --- /dev/null +++ b/common_test.go @@ -0,0 +1,76 @@ +// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// All rights reserved. Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +package protocol + +import ( + "io" + "time" +) + +type TestModel struct { + data []byte + repo string + name string + offset int64 + size int + closedCh chan bool +} + +func newTestModel() *TestModel { + return &TestModel{ + closedCh: make(chan bool), + } +} + +func (t *TestModel) Index(nodeID NodeID, repo string, files []FileInfo) { +} + +func (t *TestModel) IndexUpdate(nodeID NodeID, repo string, files []FileInfo) { +} + +func (t *TestModel) Request(nodeID NodeID, repo, name string, offset int64, size int) ([]byte, error) { + t.repo = repo + t.name = name + t.offset = offset + t.size = size + return t.data, nil +} + +func (t *TestModel) Close(nodeID NodeID, err error) { + close(t.closedCh) +} + +func (t *TestModel) ClusterConfig(nodeID NodeID, config ClusterConfigMessage) { +} + +func (t *TestModel) isClosed() bool { + select { + case <-t.closedCh: + return true + case <-time.After(1 * time.Second): + return false // Timeout + } +} + +type ErrPipe struct { + io.PipeWriter + written int + max int + err error + closed bool +} + +func (e *ErrPipe) Write(data []byte) (int, error) { + if e.closed { + return 0, e.err + } + if e.written+len(data) > e.max { + n, _ := e.PipeWriter.Write(data[:e.max-e.written]) + e.PipeWriter.CloseWithError(e.err) + e.closed = true + return n, e.err + } + return e.PipeWriter.Write(data) +} diff --git a/counting.go b/counting.go new file mode 100644 index 000000000..512774fba --- /dev/null +++ b/counting.go @@ -0,0 +1,64 @@ +// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// All rights reserved. Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +package protocol + +import ( + "io" + "sync/atomic" + "time" +) + +type countingReader struct { + io.Reader + tot uint64 // bytes + last int64 // unix nanos +} + +var ( + totalIncoming uint64 + totalOutgoing uint64 +) + +func (c *countingReader) Read(bs []byte) (int, error) { + n, err := c.Reader.Read(bs) + atomic.AddUint64(&c.tot, uint64(n)) + atomic.AddUint64(&totalIncoming, uint64(n)) + atomic.StoreInt64(&c.last, time.Now().UnixNano()) + return n, err +} + +func (c *countingReader) Tot() uint64 { + return atomic.LoadUint64(&c.tot) +} + +func (c *countingReader) Last() time.Time { + return time.Unix(0, atomic.LoadInt64(&c.last)) +} + +type countingWriter struct { + io.Writer + tot uint64 // bytes + last int64 // unix nanos +} + +func (c *countingWriter) Write(bs []byte) (int, error) { + n, err := c.Writer.Write(bs) + atomic.AddUint64(&c.tot, uint64(n)) + atomic.AddUint64(&totalOutgoing, uint64(n)) + atomic.StoreInt64(&c.last, time.Now().UnixNano()) + return n, err +} + +func (c *countingWriter) Tot() uint64 { + return atomic.LoadUint64(&c.tot) +} + +func (c *countingWriter) Last() time.Time { + return time.Unix(0, atomic.LoadInt64(&c.last)) +} + +func TotalInOut() (uint64, uint64) { + return atomic.LoadUint64(&totalIncoming), atomic.LoadUint64(&totalOutgoing) +} diff --git a/debug.go b/debug.go new file mode 100644 index 000000000..6c586b90e --- /dev/null +++ b/debug.go @@ -0,0 +1,17 @@ +// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// All rights reserved. Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +package protocol + +import ( + "os" + "strings" + + "github.com/syncthing/syncthing/internal/logger" +) + +var ( + debug = strings.Contains(os.Getenv("STTRACE"), "protocol") || os.Getenv("STTRACE") == "all" + l = logger.DefaultLogger +) diff --git a/doc.go b/doc.go new file mode 100644 index 000000000..8c6b524e6 --- /dev/null +++ b/doc.go @@ -0,0 +1,6 @@ +// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// All rights reserved. Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +// Package protocol implements the Block Exchange Protocol. +package protocol diff --git a/header.go b/header.go new file mode 100644 index 000000000..6fd2ebb8f --- /dev/null +++ b/header.go @@ -0,0 +1,45 @@ +// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// All rights reserved. Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +package protocol + +import "github.com/calmh/xdr" + +type header struct { + version int + msgID int + msgType int + compression bool +} + +func (h header) encodeXDR(xw *xdr.Writer) (int, error) { + u := encodeHeader(h) + return xw.WriteUint32(u) +} + +func (h *header) decodeXDR(xr *xdr.Reader) error { + u := xr.ReadUint32() + *h = decodeHeader(u) + return xr.Error() +} + +func encodeHeader(h header) uint32 { + var isComp uint32 + if h.compression { + isComp = 1 << 0 // the zeroth bit is the compression bit + } + return uint32(h.version&0xf)<<28 + + uint32(h.msgID&0xfff)<<16 + + uint32(h.msgType&0xff)<<8 + + isComp +} + +func decodeHeader(u uint32) header { + return header{ + version: int(u>>28) & 0xf, + msgID: int(u>>16) & 0xfff, + msgType: int(u>>8) & 0xff, + compression: u&1 == 1, + } +} diff --git a/message.go b/message.go new file mode 100644 index 000000000..779817a7b --- /dev/null +++ b/message.go @@ -0,0 +1,139 @@ +// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// All rights reserved. Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +package protocol + +import "fmt" + +type IndexMessage struct { + Repository string // max:64 + Files []FileInfo +} + +type FileInfo struct { + Name string // max:8192 + Flags uint32 + Modified int64 + Version uint64 + LocalVersion uint64 + Blocks []BlockInfo +} + +func (f FileInfo) String() string { + return fmt.Sprintf("File{Name:%q, Flags:0%o, Modified:%d, Version:%d, Size:%d, Blocks:%v}", + f.Name, f.Flags, f.Modified, f.Version, f.Size(), f.Blocks) +} + +func (f FileInfo) Size() (bytes int64) { + if IsDeleted(f.Flags) || IsDirectory(f.Flags) { + return 128 + } + for _, b := range f.Blocks { + bytes += int64(b.Size) + } + return +} + +func (f FileInfo) IsDeleted() bool { + return IsDeleted(f.Flags) +} + +func (f FileInfo) IsInvalid() bool { + return IsInvalid(f.Flags) +} + +// Used for unmarshalling a FileInfo structure but skipping the actual block list +type FileInfoTruncated struct { + Name string // max:8192 + Flags uint32 + Modified int64 + Version uint64 + LocalVersion uint64 + NumBlocks uint32 +} + +// Returns a statistical guess on the size, not the exact figure +func (f FileInfoTruncated) Size() int64 { + if IsDeleted(f.Flags) || IsDirectory(f.Flags) { + return 128 + } + if f.NumBlocks < 2 { + return BlockSize / 2 + } else { + return int64(f.NumBlocks-1)*BlockSize + BlockSize/2 + } +} + +func (f FileInfoTruncated) IsDeleted() bool { + return IsDeleted(f.Flags) +} + +func (f FileInfoTruncated) IsInvalid() bool { + return IsInvalid(f.Flags) +} + +type FileIntf interface { + Size() int64 + IsDeleted() bool + IsInvalid() bool +} + +type BlockInfo struct { + Offset int64 // noencode (cache only) + Size uint32 + Hash []byte // max:64 +} + +func (b BlockInfo) String() string { + return fmt.Sprintf("Block{%d/%d/%x}", b.Offset, b.Size, b.Hash) +} + +type RequestMessage struct { + Repository string // max:64 + Name string // max:8192 + Offset uint64 + Size uint32 +} + +type ResponseMessage struct { + Data []byte +} + +type ClusterConfigMessage struct { + ClientName string // max:64 + ClientVersion string // max:64 + Repositories []Repository // max:64 + Options []Option // max:64 +} + +func (o *ClusterConfigMessage) GetOption(key string) string { + for _, option := range o.Options { + if option.Key == key { + return option.Value + } + } + return "" +} + +type Repository struct { + ID string // max:64 + Nodes []Node // max:64 +} + +type Node struct { + ID []byte // max:32 + Flags uint32 + MaxLocalVersion uint64 +} + +type Option struct { + Key string // max:64 + Value string // max:1024 +} + +type CloseMessage struct { + Reason string // max:1024 +} + +type EmptyMessage struct{} diff --git a/message_xdr.go b/message_xdr.go new file mode 100644 index 000000000..11a11da34 --- /dev/null +++ b/message_xdr.go @@ -0,0 +1,964 @@ +// ************************************************************ +// This file is automatically generated by genxdr. Do not edit. +// ************************************************************ + +package protocol + +import ( + "bytes" + "io" + + "github.com/calmh/xdr" +) + +/* + +IndexMessage Structure: + + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Length of Repository | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Repository (variable length) \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Number of Files | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Zero or more FileInfo Structures \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + +struct IndexMessage { + string Repository<64>; + FileInfo Files<>; +} + +*/ + +func (o IndexMessage) EncodeXDR(w io.Writer) (int, error) { + var xw = xdr.NewWriter(w) + return o.encodeXDR(xw) +} + +func (o IndexMessage) MarshalXDR() []byte { + return o.AppendXDR(make([]byte, 0, 128)) +} + +func (o IndexMessage) AppendXDR(bs []byte) []byte { + var aw = xdr.AppendWriter(bs) + var xw = xdr.NewWriter(&aw) + o.encodeXDR(xw) + return []byte(aw) +} + +func (o IndexMessage) encodeXDR(xw *xdr.Writer) (int, error) { + if len(o.Repository) > 64 { + return xw.Tot(), xdr.ErrElementSizeExceeded + } + xw.WriteString(o.Repository) + xw.WriteUint32(uint32(len(o.Files))) + for i := range o.Files { + _, err := o.Files[i].encodeXDR(xw) + if err != nil { + return xw.Tot(), err + } + } + return xw.Tot(), xw.Error() +} + +func (o *IndexMessage) DecodeXDR(r io.Reader) error { + xr := xdr.NewReader(r) + return o.decodeXDR(xr) +} + +func (o *IndexMessage) UnmarshalXDR(bs []byte) error { + var br = bytes.NewReader(bs) + var xr = xdr.NewReader(br) + return o.decodeXDR(xr) +} + +func (o *IndexMessage) decodeXDR(xr *xdr.Reader) error { + o.Repository = xr.ReadStringMax(64) + _FilesSize := int(xr.ReadUint32()) + o.Files = make([]FileInfo, _FilesSize) + for i := range o.Files { + (&o.Files[i]).decodeXDR(xr) + } + return xr.Error() +} + +/* + +FileInfo Structure: + + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Length of Name | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Name (variable length) \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Flags | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| | ++ Modified (64 bits) + +| | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| | ++ Version (64 bits) + +| | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| | ++ Local Version (64 bits) + +| | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Number of Blocks | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Zero or more BlockInfo Structures \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + +struct FileInfo { + string Name<8192>; + unsigned int Flags; + hyper Modified; + unsigned hyper Version; + unsigned hyper LocalVersion; + BlockInfo Blocks<>; +} + +*/ + +func (o FileInfo) EncodeXDR(w io.Writer) (int, error) { + var xw = xdr.NewWriter(w) + return o.encodeXDR(xw) +} + +func (o FileInfo) MarshalXDR() []byte { + return o.AppendXDR(make([]byte, 0, 128)) +} + +func (o FileInfo) AppendXDR(bs []byte) []byte { + var aw = xdr.AppendWriter(bs) + var xw = xdr.NewWriter(&aw) + o.encodeXDR(xw) + return []byte(aw) +} + +func (o FileInfo) encodeXDR(xw *xdr.Writer) (int, error) { + if len(o.Name) > 8192 { + return xw.Tot(), xdr.ErrElementSizeExceeded + } + xw.WriteString(o.Name) + xw.WriteUint32(o.Flags) + xw.WriteUint64(uint64(o.Modified)) + xw.WriteUint64(o.Version) + xw.WriteUint64(o.LocalVersion) + xw.WriteUint32(uint32(len(o.Blocks))) + for i := range o.Blocks { + _, err := o.Blocks[i].encodeXDR(xw) + if err != nil { + return xw.Tot(), err + } + } + return xw.Tot(), xw.Error() +} + +func (o *FileInfo) DecodeXDR(r io.Reader) error { + xr := xdr.NewReader(r) + return o.decodeXDR(xr) +} + +func (o *FileInfo) UnmarshalXDR(bs []byte) error { + var br = bytes.NewReader(bs) + var xr = xdr.NewReader(br) + return o.decodeXDR(xr) +} + +func (o *FileInfo) decodeXDR(xr *xdr.Reader) error { + o.Name = xr.ReadStringMax(8192) + o.Flags = xr.ReadUint32() + o.Modified = int64(xr.ReadUint64()) + o.Version = xr.ReadUint64() + o.LocalVersion = xr.ReadUint64() + _BlocksSize := int(xr.ReadUint32()) + o.Blocks = make([]BlockInfo, _BlocksSize) + for i := range o.Blocks { + (&o.Blocks[i]).decodeXDR(xr) + } + return xr.Error() +} + +/* + +FileInfoTruncated Structure: + + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Length of Name | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Name (variable length) \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Flags | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| | ++ Modified (64 bits) + +| | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| | ++ Version (64 bits) + +| | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| | ++ Local Version (64 bits) + +| | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Num Blocks | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + +struct FileInfoTruncated { + string Name<8192>; + unsigned int Flags; + hyper Modified; + unsigned hyper Version; + unsigned hyper LocalVersion; + unsigned int NumBlocks; +} + +*/ + +func (o FileInfoTruncated) EncodeXDR(w io.Writer) (int, error) { + var xw = xdr.NewWriter(w) + return o.encodeXDR(xw) +} + +func (o FileInfoTruncated) MarshalXDR() []byte { + return o.AppendXDR(make([]byte, 0, 128)) +} + +func (o FileInfoTruncated) AppendXDR(bs []byte) []byte { + var aw = xdr.AppendWriter(bs) + var xw = xdr.NewWriter(&aw) + o.encodeXDR(xw) + return []byte(aw) +} + +func (o FileInfoTruncated) encodeXDR(xw *xdr.Writer) (int, error) { + if len(o.Name) > 8192 { + return xw.Tot(), xdr.ErrElementSizeExceeded + } + xw.WriteString(o.Name) + xw.WriteUint32(o.Flags) + xw.WriteUint64(uint64(o.Modified)) + xw.WriteUint64(o.Version) + xw.WriteUint64(o.LocalVersion) + xw.WriteUint32(o.NumBlocks) + return xw.Tot(), xw.Error() +} + +func (o *FileInfoTruncated) DecodeXDR(r io.Reader) error { + xr := xdr.NewReader(r) + return o.decodeXDR(xr) +} + +func (o *FileInfoTruncated) UnmarshalXDR(bs []byte) error { + var br = bytes.NewReader(bs) + var xr = xdr.NewReader(br) + return o.decodeXDR(xr) +} + +func (o *FileInfoTruncated) decodeXDR(xr *xdr.Reader) error { + o.Name = xr.ReadStringMax(8192) + o.Flags = xr.ReadUint32() + o.Modified = int64(xr.ReadUint64()) + o.Version = xr.ReadUint64() + o.LocalVersion = xr.ReadUint64() + o.NumBlocks = xr.ReadUint32() + return xr.Error() +} + +/* + +BlockInfo Structure: + + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Size | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Length of Hash | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Hash (variable length) \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + +struct BlockInfo { + unsigned int Size; + opaque Hash<64>; +} + +*/ + +func (o BlockInfo) EncodeXDR(w io.Writer) (int, error) { + var xw = xdr.NewWriter(w) + return o.encodeXDR(xw) +} + +func (o BlockInfo) MarshalXDR() []byte { + return o.AppendXDR(make([]byte, 0, 128)) +} + +func (o BlockInfo) AppendXDR(bs []byte) []byte { + var aw = xdr.AppendWriter(bs) + var xw = xdr.NewWriter(&aw) + o.encodeXDR(xw) + return []byte(aw) +} + +func (o BlockInfo) encodeXDR(xw *xdr.Writer) (int, error) { + xw.WriteUint32(o.Size) + if len(o.Hash) > 64 { + return xw.Tot(), xdr.ErrElementSizeExceeded + } + xw.WriteBytes(o.Hash) + return xw.Tot(), xw.Error() +} + +func (o *BlockInfo) DecodeXDR(r io.Reader) error { + xr := xdr.NewReader(r) + return o.decodeXDR(xr) +} + +func (o *BlockInfo) UnmarshalXDR(bs []byte) error { + var br = bytes.NewReader(bs) + var xr = xdr.NewReader(br) + return o.decodeXDR(xr) +} + +func (o *BlockInfo) decodeXDR(xr *xdr.Reader) error { + o.Size = xr.ReadUint32() + o.Hash = xr.ReadBytesMax(64) + return xr.Error() +} + +/* + +RequestMessage Structure: + + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Length of Repository | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Repository (variable length) \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Length of Name | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Name (variable length) \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| | ++ Offset (64 bits) + +| | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Size | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + +struct RequestMessage { + string Repository<64>; + string Name<8192>; + unsigned hyper Offset; + unsigned int Size; +} + +*/ + +func (o RequestMessage) EncodeXDR(w io.Writer) (int, error) { + var xw = xdr.NewWriter(w) + return o.encodeXDR(xw) +} + +func (o RequestMessage) MarshalXDR() []byte { + return o.AppendXDR(make([]byte, 0, 128)) +} + +func (o RequestMessage) AppendXDR(bs []byte) []byte { + var aw = xdr.AppendWriter(bs) + var xw = xdr.NewWriter(&aw) + o.encodeXDR(xw) + return []byte(aw) +} + +func (o RequestMessage) encodeXDR(xw *xdr.Writer) (int, error) { + if len(o.Repository) > 64 { + return xw.Tot(), xdr.ErrElementSizeExceeded + } + xw.WriteString(o.Repository) + if len(o.Name) > 8192 { + return xw.Tot(), xdr.ErrElementSizeExceeded + } + xw.WriteString(o.Name) + xw.WriteUint64(o.Offset) + xw.WriteUint32(o.Size) + return xw.Tot(), xw.Error() +} + +func (o *RequestMessage) DecodeXDR(r io.Reader) error { + xr := xdr.NewReader(r) + return o.decodeXDR(xr) +} + +func (o *RequestMessage) UnmarshalXDR(bs []byte) error { + var br = bytes.NewReader(bs) + var xr = xdr.NewReader(br) + return o.decodeXDR(xr) +} + +func (o *RequestMessage) decodeXDR(xr *xdr.Reader) error { + o.Repository = xr.ReadStringMax(64) + o.Name = xr.ReadStringMax(8192) + o.Offset = xr.ReadUint64() + o.Size = xr.ReadUint32() + return xr.Error() +} + +/* + +ResponseMessage Structure: + + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Length of Data | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Data (variable length) \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + +struct ResponseMessage { + opaque Data<>; +} + +*/ + +func (o ResponseMessage) EncodeXDR(w io.Writer) (int, error) { + var xw = xdr.NewWriter(w) + return o.encodeXDR(xw) +} + +func (o ResponseMessage) MarshalXDR() []byte { + return o.AppendXDR(make([]byte, 0, 128)) +} + +func (o ResponseMessage) AppendXDR(bs []byte) []byte { + var aw = xdr.AppendWriter(bs) + var xw = xdr.NewWriter(&aw) + o.encodeXDR(xw) + return []byte(aw) +} + +func (o ResponseMessage) encodeXDR(xw *xdr.Writer) (int, error) { + xw.WriteBytes(o.Data) + return xw.Tot(), xw.Error() +} + +func (o *ResponseMessage) DecodeXDR(r io.Reader) error { + xr := xdr.NewReader(r) + return o.decodeXDR(xr) +} + +func (o *ResponseMessage) UnmarshalXDR(bs []byte) error { + var br = bytes.NewReader(bs) + var xr = xdr.NewReader(br) + return o.decodeXDR(xr) +} + +func (o *ResponseMessage) decodeXDR(xr *xdr.Reader) error { + o.Data = xr.ReadBytes() + return xr.Error() +} + +/* + +ClusterConfigMessage Structure: + + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Length of Client Name | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Client Name (variable length) \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Length of Client Version | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Client Version (variable length) \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Number of Repositories | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Zero or more Repository Structures \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Number of Options | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Zero or more Option Structures \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + +struct ClusterConfigMessage { + string ClientName<64>; + string ClientVersion<64>; + Repository Repositories<64>; + Option Options<64>; +} + +*/ + +func (o ClusterConfigMessage) EncodeXDR(w io.Writer) (int, error) { + var xw = xdr.NewWriter(w) + return o.encodeXDR(xw) +} + +func (o ClusterConfigMessage) MarshalXDR() []byte { + return o.AppendXDR(make([]byte, 0, 128)) +} + +func (o ClusterConfigMessage) AppendXDR(bs []byte) []byte { + var aw = xdr.AppendWriter(bs) + var xw = xdr.NewWriter(&aw) + o.encodeXDR(xw) + return []byte(aw) +} + +func (o ClusterConfigMessage) encodeXDR(xw *xdr.Writer) (int, error) { + if len(o.ClientName) > 64 { + return xw.Tot(), xdr.ErrElementSizeExceeded + } + xw.WriteString(o.ClientName) + if len(o.ClientVersion) > 64 { + return xw.Tot(), xdr.ErrElementSizeExceeded + } + xw.WriteString(o.ClientVersion) + if len(o.Repositories) > 64 { + return xw.Tot(), xdr.ErrElementSizeExceeded + } + xw.WriteUint32(uint32(len(o.Repositories))) + for i := range o.Repositories { + _, err := o.Repositories[i].encodeXDR(xw) + if err != nil { + return xw.Tot(), err + } + } + if len(o.Options) > 64 { + return xw.Tot(), xdr.ErrElementSizeExceeded + } + xw.WriteUint32(uint32(len(o.Options))) + for i := range o.Options { + _, err := o.Options[i].encodeXDR(xw) + if err != nil { + return xw.Tot(), err + } + } + return xw.Tot(), xw.Error() +} + +func (o *ClusterConfigMessage) DecodeXDR(r io.Reader) error { + xr := xdr.NewReader(r) + return o.decodeXDR(xr) +} + +func (o *ClusterConfigMessage) UnmarshalXDR(bs []byte) error { + var br = bytes.NewReader(bs) + var xr = xdr.NewReader(br) + return o.decodeXDR(xr) +} + +func (o *ClusterConfigMessage) decodeXDR(xr *xdr.Reader) error { + o.ClientName = xr.ReadStringMax(64) + o.ClientVersion = xr.ReadStringMax(64) + _RepositoriesSize := int(xr.ReadUint32()) + if _RepositoriesSize > 64 { + return xdr.ErrElementSizeExceeded + } + o.Repositories = make([]Repository, _RepositoriesSize) + for i := range o.Repositories { + (&o.Repositories[i]).decodeXDR(xr) + } + _OptionsSize := int(xr.ReadUint32()) + if _OptionsSize > 64 { + return xdr.ErrElementSizeExceeded + } + o.Options = make([]Option, _OptionsSize) + for i := range o.Options { + (&o.Options[i]).decodeXDR(xr) + } + return xr.Error() +} + +/* + +Repository Structure: + + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Length of ID | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ ID (variable length) \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Number of Nodes | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Zero or more Node Structures \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + +struct Repository { + string ID<64>; + Node Nodes<64>; +} + +*/ + +func (o Repository) EncodeXDR(w io.Writer) (int, error) { + var xw = xdr.NewWriter(w) + return o.encodeXDR(xw) +} + +func (o Repository) MarshalXDR() []byte { + return o.AppendXDR(make([]byte, 0, 128)) +} + +func (o Repository) AppendXDR(bs []byte) []byte { + var aw = xdr.AppendWriter(bs) + var xw = xdr.NewWriter(&aw) + o.encodeXDR(xw) + return []byte(aw) +} + +func (o Repository) encodeXDR(xw *xdr.Writer) (int, error) { + if len(o.ID) > 64 { + return xw.Tot(), xdr.ErrElementSizeExceeded + } + xw.WriteString(o.ID) + if len(o.Nodes) > 64 { + return xw.Tot(), xdr.ErrElementSizeExceeded + } + xw.WriteUint32(uint32(len(o.Nodes))) + for i := range o.Nodes { + _, err := o.Nodes[i].encodeXDR(xw) + if err != nil { + return xw.Tot(), err + } + } + return xw.Tot(), xw.Error() +} + +func (o *Repository) DecodeXDR(r io.Reader) error { + xr := xdr.NewReader(r) + return o.decodeXDR(xr) +} + +func (o *Repository) UnmarshalXDR(bs []byte) error { + var br = bytes.NewReader(bs) + var xr = xdr.NewReader(br) + return o.decodeXDR(xr) +} + +func (o *Repository) decodeXDR(xr *xdr.Reader) error { + o.ID = xr.ReadStringMax(64) + _NodesSize := int(xr.ReadUint32()) + if _NodesSize > 64 { + return xdr.ErrElementSizeExceeded + } + o.Nodes = make([]Node, _NodesSize) + for i := range o.Nodes { + (&o.Nodes[i]).decodeXDR(xr) + } + return xr.Error() +} + +/* + +Node Structure: + + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Length of ID | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ ID (variable length) \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Flags | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| | ++ Max Local Version (64 bits) + +| | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + +struct Node { + opaque ID<32>; + unsigned int Flags; + unsigned hyper MaxLocalVersion; +} + +*/ + +func (o Node) EncodeXDR(w io.Writer) (int, error) { + var xw = xdr.NewWriter(w) + return o.encodeXDR(xw) +} + +func (o Node) MarshalXDR() []byte { + return o.AppendXDR(make([]byte, 0, 128)) +} + +func (o Node) AppendXDR(bs []byte) []byte { + var aw = xdr.AppendWriter(bs) + var xw = xdr.NewWriter(&aw) + o.encodeXDR(xw) + return []byte(aw) +} + +func (o Node) encodeXDR(xw *xdr.Writer) (int, error) { + if len(o.ID) > 32 { + return xw.Tot(), xdr.ErrElementSizeExceeded + } + xw.WriteBytes(o.ID) + xw.WriteUint32(o.Flags) + xw.WriteUint64(o.MaxLocalVersion) + return xw.Tot(), xw.Error() +} + +func (o *Node) DecodeXDR(r io.Reader) error { + xr := xdr.NewReader(r) + return o.decodeXDR(xr) +} + +func (o *Node) UnmarshalXDR(bs []byte) error { + var br = bytes.NewReader(bs) + var xr = xdr.NewReader(br) + return o.decodeXDR(xr) +} + +func (o *Node) decodeXDR(xr *xdr.Reader) error { + o.ID = xr.ReadBytesMax(32) + o.Flags = xr.ReadUint32() + o.MaxLocalVersion = xr.ReadUint64() + return xr.Error() +} + +/* + +Option Structure: + + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Length of Key | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Key (variable length) \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Length of Value | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Value (variable length) \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + +struct Option { + string Key<64>; + string Value<1024>; +} + +*/ + +func (o Option) EncodeXDR(w io.Writer) (int, error) { + var xw = xdr.NewWriter(w) + return o.encodeXDR(xw) +} + +func (o Option) MarshalXDR() []byte { + return o.AppendXDR(make([]byte, 0, 128)) +} + +func (o Option) AppendXDR(bs []byte) []byte { + var aw = xdr.AppendWriter(bs) + var xw = xdr.NewWriter(&aw) + o.encodeXDR(xw) + return []byte(aw) +} + +func (o Option) encodeXDR(xw *xdr.Writer) (int, error) { + if len(o.Key) > 64 { + return xw.Tot(), xdr.ErrElementSizeExceeded + } + xw.WriteString(o.Key) + if len(o.Value) > 1024 { + return xw.Tot(), xdr.ErrElementSizeExceeded + } + xw.WriteString(o.Value) + return xw.Tot(), xw.Error() +} + +func (o *Option) DecodeXDR(r io.Reader) error { + xr := xdr.NewReader(r) + return o.decodeXDR(xr) +} + +func (o *Option) UnmarshalXDR(bs []byte) error { + var br = bytes.NewReader(bs) + var xr = xdr.NewReader(br) + return o.decodeXDR(xr) +} + +func (o *Option) decodeXDR(xr *xdr.Reader) error { + o.Key = xr.ReadStringMax(64) + o.Value = xr.ReadStringMax(1024) + return xr.Error() +} + +/* + +CloseMessage Structure: + + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Length of Reason | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Reason (variable length) \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + +struct CloseMessage { + string Reason<1024>; +} + +*/ + +func (o CloseMessage) EncodeXDR(w io.Writer) (int, error) { + var xw = xdr.NewWriter(w) + return o.encodeXDR(xw) +} + +func (o CloseMessage) MarshalXDR() []byte { + return o.AppendXDR(make([]byte, 0, 128)) +} + +func (o CloseMessage) AppendXDR(bs []byte) []byte { + var aw = xdr.AppendWriter(bs) + var xw = xdr.NewWriter(&aw) + o.encodeXDR(xw) + return []byte(aw) +} + +func (o CloseMessage) encodeXDR(xw *xdr.Writer) (int, error) { + if len(o.Reason) > 1024 { + return xw.Tot(), xdr.ErrElementSizeExceeded + } + xw.WriteString(o.Reason) + return xw.Tot(), xw.Error() +} + +func (o *CloseMessage) DecodeXDR(r io.Reader) error { + xr := xdr.NewReader(r) + return o.decodeXDR(xr) +} + +func (o *CloseMessage) UnmarshalXDR(bs []byte) error { + var br = bytes.NewReader(bs) + var xr = xdr.NewReader(br) + return o.decodeXDR(xr) +} + +func (o *CloseMessage) decodeXDR(xr *xdr.Reader) error { + o.Reason = xr.ReadStringMax(1024) + return xr.Error() +} + +/* + +EmptyMessage Structure: + + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + +struct EmptyMessage { +} + +*/ + +func (o EmptyMessage) EncodeXDR(w io.Writer) (int, error) { + var xw = xdr.NewWriter(w) + return o.encodeXDR(xw) +} + +func (o EmptyMessage) MarshalXDR() []byte { + return o.AppendXDR(make([]byte, 0, 128)) +} + +func (o EmptyMessage) AppendXDR(bs []byte) []byte { + var aw = xdr.AppendWriter(bs) + var xw = xdr.NewWriter(&aw) + o.encodeXDR(xw) + return []byte(aw) +} + +func (o EmptyMessage) encodeXDR(xw *xdr.Writer) (int, error) { + return xw.Tot(), xw.Error() +} + +func (o *EmptyMessage) DecodeXDR(r io.Reader) error { + xr := xdr.NewReader(r) + return o.decodeXDR(xr) +} + +func (o *EmptyMessage) UnmarshalXDR(bs []byte) error { + var br = bytes.NewReader(bs) + var xr = xdr.NewReader(br) + return o.decodeXDR(xr) +} + +func (o *EmptyMessage) decodeXDR(xr *xdr.Reader) error { + return xr.Error() +} diff --git a/nativemodel_darwin.go b/nativemodel_darwin.go new file mode 100644 index 000000000..9ac402fe6 --- /dev/null +++ b/nativemodel_darwin.go @@ -0,0 +1,42 @@ +// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// All rights reserved. Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +// +build darwin + +package protocol + +// Darwin uses NFD normalization + +import "code.google.com/p/go.text/unicode/norm" + +type nativeModel struct { + next Model +} + +func (m nativeModel) Index(nodeID NodeID, repo string, files []FileInfo) { + for i := range files { + files[i].Name = norm.NFD.String(files[i].Name) + } + m.next.Index(nodeID, repo, files) +} + +func (m nativeModel) IndexUpdate(nodeID NodeID, repo string, files []FileInfo) { + for i := range files { + files[i].Name = norm.NFD.String(files[i].Name) + } + m.next.IndexUpdate(nodeID, repo, files) +} + +func (m nativeModel) Request(nodeID NodeID, repo string, name string, offset int64, size int) ([]byte, error) { + name = norm.NFD.String(name) + return m.next.Request(nodeID, repo, name, offset, size) +} + +func (m nativeModel) ClusterConfig(nodeID NodeID, config ClusterConfigMessage) { + m.next.ClusterConfig(nodeID, config) +} + +func (m nativeModel) Close(nodeID NodeID, err error) { + m.next.Close(nodeID, err) +} diff --git a/nativemodel_unix.go b/nativemodel_unix.go new file mode 100644 index 000000000..23fbe0b6b --- /dev/null +++ b/nativemodel_unix.go @@ -0,0 +1,33 @@ +// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// All rights reserved. Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +// +build !windows,!darwin + +package protocol + +// Normal Unixes uses NFC and slashes, which is the wire format. + +type nativeModel struct { + next Model +} + +func (m nativeModel) Index(nodeID NodeID, repo string, files []FileInfo) { + m.next.Index(nodeID, repo, files) +} + +func (m nativeModel) IndexUpdate(nodeID NodeID, repo string, files []FileInfo) { + m.next.IndexUpdate(nodeID, repo, files) +} + +func (m nativeModel) Request(nodeID NodeID, repo string, name string, offset int64, size int) ([]byte, error) { + return m.next.Request(nodeID, repo, name, offset, size) +} + +func (m nativeModel) ClusterConfig(nodeID NodeID, config ClusterConfigMessage) { + m.next.ClusterConfig(nodeID, config) +} + +func (m nativeModel) Close(nodeID NodeID, err error) { + m.next.Close(nodeID, err) +} diff --git a/nativemodel_windows.go b/nativemodel_windows.go new file mode 100644 index 000000000..9841d63f7 --- /dev/null +++ b/nativemodel_windows.go @@ -0,0 +1,72 @@ +// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// All rights reserved. Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +// +build windows + +package protocol + +// Windows uses backslashes as file separator and disallows a bunch of +// characters in the filename + +import ( + "path/filepath" + "strings" +) + +var disallowedCharacters = string([]rune{ + '<', '>', ':', '"', '|', '?', '*', + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, + 31, +}) + +type nativeModel struct { + next Model +} + +func (m nativeModel) Index(nodeID NodeID, repo string, files []FileInfo) { + for i, f := range files { + if strings.ContainsAny(f.Name, disallowedCharacters) { + if f.IsDeleted() { + // Don't complain if the file is marked as deleted, since it + // can't possibly exist here anyway. + continue + } + files[i].Flags |= FlagInvalid + l.Warnf("File name %q contains invalid characters; marked as invalid.", f.Name) + } + files[i].Name = filepath.FromSlash(f.Name) + } + m.next.Index(nodeID, repo, files) +} + +func (m nativeModel) IndexUpdate(nodeID NodeID, repo string, files []FileInfo) { + for i, f := range files { + if strings.ContainsAny(f.Name, disallowedCharacters) { + if f.IsDeleted() { + // Don't complain if the file is marked as deleted, since it + // can't possibly exist here anyway. + continue + } + files[i].Flags |= FlagInvalid + l.Warnf("File name %q contains invalid characters; marked as invalid.", f.Name) + } + files[i].Name = filepath.FromSlash(files[i].Name) + } + m.next.IndexUpdate(nodeID, repo, files) +} + +func (m nativeModel) Request(nodeID NodeID, repo string, name string, offset int64, size int) ([]byte, error) { + name = filepath.FromSlash(name) + return m.next.Request(nodeID, repo, name, offset, size) +} + +func (m nativeModel) ClusterConfig(nodeID NodeID, config ClusterConfigMessage) { + m.next.ClusterConfig(nodeID, config) +} + +func (m nativeModel) Close(nodeID NodeID, err error) { + m.next.Close(nodeID, err) +} diff --git a/nodeid.go b/nodeid.go new file mode 100644 index 000000000..9079781b9 --- /dev/null +++ b/nodeid.go @@ -0,0 +1,159 @@ +// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// All rights reserved. Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +package protocol + +import ( + "bytes" + "crypto/sha256" + "encoding/base32" + "errors" + "fmt" + "regexp" + "strings" + + "github.com/syncthing/syncthing/internal/luhn" +) + +type NodeID [32]byte + +var LocalNodeID = NodeID{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff} + +// NewNodeID generates a new node ID from the raw bytes of a certificate +func NewNodeID(rawCert []byte) NodeID { + var n NodeID + hf := sha256.New() + hf.Write(rawCert) + hf.Sum(n[:0]) + return n +} + +func NodeIDFromString(s string) (NodeID, error) { + var n NodeID + err := n.UnmarshalText([]byte(s)) + return n, err +} + +func NodeIDFromBytes(bs []byte) NodeID { + var n NodeID + if len(bs) != len(n) { + panic("incorrect length of byte slice representing node ID") + } + copy(n[:], bs) + return n +} + +// String returns the canonical string representation of the node ID +func (n NodeID) String() string { + id := base32.StdEncoding.EncodeToString(n[:]) + id = strings.Trim(id, "=") + id, err := luhnify(id) + if err != nil { + // Should never happen + panic(err) + } + id = chunkify(id) + return id +} + +func (n NodeID) GoString() string { + return n.String() +} + +func (n NodeID) Compare(other NodeID) int { + return bytes.Compare(n[:], other[:]) +} + +func (n NodeID) Equals(other NodeID) bool { + return bytes.Compare(n[:], other[:]) == 0 +} + +func (n *NodeID) MarshalText() ([]byte, error) { + return []byte(n.String()), nil +} + +func (n *NodeID) UnmarshalText(bs []byte) error { + id := string(bs) + id = strings.Trim(id, "=") + id = strings.ToUpper(id) + id = untypeoify(id) + id = unchunkify(id) + + var err error + switch len(id) { + case 56: + // New style, with check digits + id, err = unluhnify(id) + if err != nil { + return err + } + fallthrough + case 52: + // Old style, no check digits + dec, err := base32.StdEncoding.DecodeString(id + "====") + if err != nil { + return err + } + copy(n[:], dec) + return nil + default: + return errors.New("node ID invalid: incorrect length") + } +} + +func luhnify(s string) (string, error) { + if len(s) != 52 { + panic("unsupported string length") + } + + res := make([]string, 0, 4) + for i := 0; i < 4; i++ { + p := s[i*13 : (i+1)*13] + l, err := luhn.Base32.Generate(p) + if err != nil { + return "", err + } + res = append(res, fmt.Sprintf("%s%c", p, l)) + } + return res[0] + res[1] + res[2] + res[3], nil +} + +func unluhnify(s string) (string, error) { + if len(s) != 56 { + return "", fmt.Errorf("unsupported string length %d", len(s)) + } + + res := make([]string, 0, 4) + for i := 0; i < 4; i++ { + p := s[i*14 : (i+1)*14-1] + l, err := luhn.Base32.Generate(p) + if err != nil { + return "", err + } + if g := fmt.Sprintf("%s%c", p, l); g != s[i*14:(i+1)*14] { + return "", errors.New("check digit incorrect") + } + res = append(res, p) + } + return res[0] + res[1] + res[2] + res[3], nil +} + +func chunkify(s string) string { + s = regexp.MustCompile("(.{7})").ReplaceAllString(s, "$1-") + s = strings.Trim(s, "-") + return s +} + +func unchunkify(s string) string { + s = strings.Replace(s, "-", "", -1) + s = strings.Replace(s, " ", "", -1) + return s +} + +func untypeoify(s string) string { + s = strings.Replace(s, "0", "O", -1) + s = strings.Replace(s, "1", "I", -1) + s = strings.Replace(s, "8", "B", -1) + return s +} diff --git a/nodeid_test.go b/nodeid_test.go new file mode 100644 index 000000000..5b861b6de --- /dev/null +++ b/nodeid_test.go @@ -0,0 +1,78 @@ +// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// All rights reserved. Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +package protocol + +import "testing" + +var formatted = "P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" +var formatCases = []string{ + "P56IOI-7MZJNU-2IQGDR-EYDM2M-GTMGL3-BXNPQ6-W5BTBB-Z4TJXZ-WICQ", + "P56IOI-7MZJNU2Y-IQGDR-EYDM2M-GTI-MGL3-BXNPQ6-W5BM-TBB-Z4TJXZ-WICQ2", + "P56IOI7 MZJNU2I QGDREYD M2MGTMGL 3BXNPQ6W 5BTB BZ4T JXZWICQ", + "P56IOI7 MZJNU2Y IQGDREY DM2MGTI MGL3BXN PQ6W5BM TBBZ4TJ XZWICQ2", + "P56IOI7MZJNU2IQGDREYDM2MGTMGL3BXNPQ6W5BTBBZ4TJXZWICQ", + "p56ioi7mzjnu2iqgdreydm2mgtmgl3bxnpq6w5btbbz4tjxzwicq", + "P56IOI7MZJNU2YIQGDREYDM2MGTIMGL3BXNPQ6W5BMTBBZ4TJXZWICQ2", + "P561017MZJNU2YIQGDREYDM2MGTIMGL3BXNPQ6W5BMT88Z4TJXZWICQ2", + "p56ioi7mzjnu2yiqgdreydm2mgtimgl3bxnpq6w5bmtbbz4tjxzwicq2", + "p561017mzjnu2yiqgdreydm2mgtimgl3bxnpq6w5bmt88z4tjxzwicq2", +} + +func TestFormatNodeID(t *testing.T) { + for i, tc := range formatCases { + var id NodeID + err := id.UnmarshalText([]byte(tc)) + if err != nil { + t.Errorf("#%d UnmarshalText(%q); %v", i, tc, err) + } else if f := id.String(); f != formatted { + t.Errorf("#%d FormatNodeID(%q)\n\t%q !=\n\t%q", i, tc, f, formatted) + } + } +} + +var validateCases = []struct { + s string + ok bool +}{ + {"", false}, + {"P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2", true}, + {"P56IOI7-MZJNU2-IQGDREY-DM2MGT-MGL3BXN-PQ6W5B-TBBZ4TJ-XZWICQ", true}, + {"P56IOI7 MZJNU2I QGDREYD M2MGTMGL 3BXNPQ6W 5BTB BZ4T JXZWICQ", true}, + {"P56IOI7MZJNU2IQGDREYDM2MGTMGL3BXNPQ6W5BTBBZ4TJXZWICQ", true}, + {"P56IOI7MZJNU2IQGDREYDM2MGTMGL3BXNPQ6W5BTBBZ4TJXZWICQCCCC", false}, + {"p56ioi7mzjnu2iqgdreydm2mgtmgl3bxnpq6w5btbbz4tjxzwicq", true}, + {"p56ioi7mzjnu2iqgdreydm2mgtmgl3bxnpq6w5btbbz4tjxzwicqCCCC", false}, +} + +func TestValidateNodeID(t *testing.T) { + for _, tc := range validateCases { + var id NodeID + err := id.UnmarshalText([]byte(tc.s)) + if (err == nil && !tc.ok) || (err != nil && tc.ok) { + t.Errorf("ValidateNodeID(%q); %v != %v", tc.s, err, tc.ok) + } + } +} + +func TestMarshallingNodeID(t *testing.T) { + n0 := NodeID{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 10, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32} + n1 := NodeID{} + n2 := NodeID{} + + bs, _ := n0.MarshalText() + n1.UnmarshalText(bs) + bs, _ = n1.MarshalText() + n2.UnmarshalText(bs) + + if n2.String() != n0.String() { + t.Errorf("String marshalling error; %q != %q", n2.String(), n0.String()) + } + if !n2.Equals(n0) { + t.Error("Equals error") + } + if n2.Compare(n0) != 0 { + t.Error("Compare error") + } +} diff --git a/protocol.go b/protocol.go new file mode 100644 index 000000000..86fd6199f --- /dev/null +++ b/protocol.go @@ -0,0 +1,640 @@ +// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// All rights reserved. Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +package protocol + +import ( + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + "io" + "sync" + "time" + + lz4 "github.com/bkaradzic/go-lz4" +) + +const ( + BlockSize = 128 * 1024 +) + +const ( + messageTypeClusterConfig = 0 + messageTypeIndex = 1 + messageTypeRequest = 2 + messageTypeResponse = 3 + messageTypePing = 4 + messageTypePong = 5 + messageTypeIndexUpdate = 6 + messageTypeClose = 7 +) + +const ( + stateInitial = iota + stateCCRcvd + stateIdxRcvd +) + +const ( + FlagDeleted uint32 = 1 << 12 + FlagInvalid = 1 << 13 + FlagDirectory = 1 << 14 + FlagNoPermBits = 1 << 15 +) + +const ( + FlagShareTrusted uint32 = 1 << 0 + FlagShareReadOnly = 1 << 1 + FlagIntroducer = 1 << 2 + FlagShareBits = 0x000000ff +) + +var ( + ErrClusterHash = fmt.Errorf("configuration error: mismatched cluster hash") + ErrClosed = errors.New("connection closed") +) + +type Model interface { + // An index was received from the peer node + Index(nodeID NodeID, repo string, files []FileInfo) + // An index update was received from the peer node + IndexUpdate(nodeID NodeID, repo string, files []FileInfo) + // A request was made by the peer node + Request(nodeID NodeID, repo string, name string, offset int64, size int) ([]byte, error) + // A cluster configuration message was received + ClusterConfig(nodeID NodeID, config ClusterConfigMessage) + // The peer node closed the connection + Close(nodeID NodeID, err error) +} + +type Connection interface { + ID() NodeID + Name() string + Index(repo string, files []FileInfo) error + IndexUpdate(repo string, files []FileInfo) error + Request(repo string, name string, offset int64, size int) ([]byte, error) + ClusterConfig(config ClusterConfigMessage) + Statistics() Statistics +} + +type rawConnection struct { + id NodeID + name string + receiver Model + state int + + cr *countingReader + cw *countingWriter + + awaiting [4096]chan asyncResult + awaitingMut sync.Mutex + + idxMut sync.Mutex // ensures serialization of Index calls + + nextID chan int + outbox chan hdrMsg + closed chan struct{} + once sync.Once + + compressionThreshold int // compress messages larger than this many bytes + + rdbuf0 []byte // used & reused by readMessage + rdbuf1 []byte // used & reused by readMessage +} + +type asyncResult struct { + val []byte + err error +} + +type hdrMsg struct { + hdr header + msg encodable +} + +type encodable interface { + AppendXDR([]byte) []byte +} + +const ( + pingTimeout = 30 * time.Second + pingIdleTime = 60 * time.Second +) + +func NewConnection(nodeID NodeID, reader io.Reader, writer io.Writer, receiver Model, name string, compress bool) Connection { + cr := &countingReader{Reader: reader} + cw := &countingWriter{Writer: writer} + + compThres := 1<<31 - 1 // compression disabled + if compress { + compThres = 128 // compress messages that are 128 bytes long or larger + } + c := rawConnection{ + id: nodeID, + name: name, + receiver: nativeModel{receiver}, + state: stateInitial, + cr: cr, + cw: cw, + outbox: make(chan hdrMsg), + nextID: make(chan int), + closed: make(chan struct{}), + compressionThreshold: compThres, + } + + go c.readerLoop() + go c.writerLoop() + go c.pingerLoop() + go c.idGenerator() + + return wireFormatConnection{&c} +} + +func (c *rawConnection) ID() NodeID { + return c.id +} + +func (c *rawConnection) Name() string { + return c.name +} + +// Index writes the list of file information to the connected peer node +func (c *rawConnection) Index(repo string, idx []FileInfo) error { + select { + case <-c.closed: + return ErrClosed + default: + } + c.idxMut.Lock() + c.send(-1, messageTypeIndex, IndexMessage{repo, idx}) + c.idxMut.Unlock() + return nil +} + +// IndexUpdate writes the list of file information to the connected peer node as an update +func (c *rawConnection) IndexUpdate(repo string, idx []FileInfo) error { + select { + case <-c.closed: + return ErrClosed + default: + } + c.idxMut.Lock() + c.send(-1, messageTypeIndexUpdate, IndexMessage{repo, idx}) + c.idxMut.Unlock() + return nil +} + +// Request returns the bytes for the specified block after fetching them from the connected peer. +func (c *rawConnection) Request(repo string, name string, offset int64, size int) ([]byte, error) { + var id int + select { + case id = <-c.nextID: + case <-c.closed: + return nil, ErrClosed + } + + c.awaitingMut.Lock() + if ch := c.awaiting[id]; ch != nil { + panic("id taken") + } + rc := make(chan asyncResult, 1) + c.awaiting[id] = rc + c.awaitingMut.Unlock() + + ok := c.send(id, messageTypeRequest, RequestMessage{repo, name, uint64(offset), uint32(size)}) + if !ok { + return nil, ErrClosed + } + + res, ok := <-rc + if !ok { + return nil, ErrClosed + } + return res.val, res.err +} + +// ClusterConfig send the cluster configuration message to the peer and returns any error +func (c *rawConnection) ClusterConfig(config ClusterConfigMessage) { + c.send(-1, messageTypeClusterConfig, config) +} + +func (c *rawConnection) ping() bool { + var id int + select { + case id = <-c.nextID: + case <-c.closed: + return false + } + + rc := make(chan asyncResult, 1) + c.awaitingMut.Lock() + c.awaiting[id] = rc + c.awaitingMut.Unlock() + + ok := c.send(id, messageTypePing, nil) + if !ok { + return false + } + + res, ok := <-rc + return ok && res.err == nil +} + +func (c *rawConnection) readerLoop() (err error) { + defer func() { + c.close(err) + }() + + for { + select { + case <-c.closed: + return ErrClosed + default: + } + + hdr, msg, err := c.readMessage() + if err != nil { + return err + } + + switch hdr.msgType { + case messageTypeIndex: + if c.state < stateCCRcvd { + return fmt.Errorf("protocol error: index message in state %d", c.state) + } + c.handleIndex(msg.(IndexMessage)) + c.state = stateIdxRcvd + + case messageTypeIndexUpdate: + if c.state < stateIdxRcvd { + return fmt.Errorf("protocol error: index update message in state %d", c.state) + } + c.handleIndexUpdate(msg.(IndexMessage)) + + case messageTypeRequest: + if c.state < stateIdxRcvd { + return fmt.Errorf("protocol error: request message in state %d", c.state) + } + // Requests are handled asynchronously + go c.handleRequest(hdr.msgID, msg.(RequestMessage)) + + case messageTypeResponse: + if c.state < stateIdxRcvd { + return fmt.Errorf("protocol error: response message in state %d", c.state) + } + c.handleResponse(hdr.msgID, msg.(ResponseMessage)) + + case messageTypePing: + c.send(hdr.msgID, messageTypePong, EmptyMessage{}) + + case messageTypePong: + c.handlePong(hdr.msgID) + + case messageTypeClusterConfig: + if c.state != stateInitial { + return fmt.Errorf("protocol error: cluster config message in state %d", c.state) + } + go c.receiver.ClusterConfig(c.id, msg.(ClusterConfigMessage)) + c.state = stateCCRcvd + + case messageTypeClose: + return errors.New(msg.(CloseMessage).Reason) + + default: + return fmt.Errorf("protocol error: %s: unknown message type %#x", c.id, hdr.msgType) + } + } +} + +func (c *rawConnection) readMessage() (hdr header, msg encodable, err error) { + if cap(c.rdbuf0) < 8 { + c.rdbuf0 = make([]byte, 8) + } else { + c.rdbuf0 = c.rdbuf0[:8] + } + _, err = io.ReadFull(c.cr, c.rdbuf0) + if err != nil { + return + } + + hdr = decodeHeader(binary.BigEndian.Uint32(c.rdbuf0[0:4])) + msglen := int(binary.BigEndian.Uint32(c.rdbuf0[4:8])) + + if debug { + l.Debugf("read header %v (msglen=%d)", hdr, msglen) + } + + if cap(c.rdbuf0) < msglen { + c.rdbuf0 = make([]byte, msglen) + } else { + c.rdbuf0 = c.rdbuf0[:msglen] + } + _, err = io.ReadFull(c.cr, c.rdbuf0) + if err != nil { + return + } + + if debug { + l.Debugf("read %d bytes", len(c.rdbuf0)) + } + + msgBuf := c.rdbuf0 + if hdr.compression { + c.rdbuf1 = c.rdbuf1[:cap(c.rdbuf1)] + c.rdbuf1, err = lz4.Decode(c.rdbuf1, c.rdbuf0) + if err != nil { + return + } + msgBuf = c.rdbuf1 + if debug { + l.Debugf("decompressed to %d bytes", len(msgBuf)) + } + } + + if debug { + if len(msgBuf) > 1024 { + l.Debugf("message data:\n%s", hex.Dump(msgBuf[:1024])) + } else { + l.Debugf("message data:\n%s", hex.Dump(msgBuf)) + } + } + + switch hdr.msgType { + case messageTypeIndex, messageTypeIndexUpdate: + var idx IndexMessage + err = idx.UnmarshalXDR(msgBuf) + msg = idx + + case messageTypeRequest: + var req RequestMessage + err = req.UnmarshalXDR(msgBuf) + msg = req + + case messageTypeResponse: + var resp ResponseMessage + err = resp.UnmarshalXDR(msgBuf) + msg = resp + + case messageTypePing, messageTypePong: + msg = EmptyMessage{} + + case messageTypeClusterConfig: + var cc ClusterConfigMessage + err = cc.UnmarshalXDR(msgBuf) + msg = cc + + case messageTypeClose: + var cm CloseMessage + err = cm.UnmarshalXDR(msgBuf) + msg = cm + + default: + err = fmt.Errorf("protocol error: %s: unknown message type %#x", c.id, hdr.msgType) + } + + return +} + +func (c *rawConnection) handleIndex(im IndexMessage) { + if debug { + l.Debugf("Index(%v, %v, %d files)", c.id, im.Repository, len(im.Files)) + } + c.receiver.Index(c.id, im.Repository, im.Files) +} + +func (c *rawConnection) handleIndexUpdate(im IndexMessage) { + if debug { + l.Debugf("queueing IndexUpdate(%v, %v, %d files)", c.id, im.Repository, len(im.Files)) + } + c.receiver.IndexUpdate(c.id, im.Repository, im.Files) +} + +func (c *rawConnection) handleRequest(msgID int, req RequestMessage) { + data, _ := c.receiver.Request(c.id, req.Repository, req.Name, int64(req.Offset), int(req.Size)) + + c.send(msgID, messageTypeResponse, ResponseMessage{data}) +} + +func (c *rawConnection) handleResponse(msgID int, resp ResponseMessage) { + c.awaitingMut.Lock() + if rc := c.awaiting[msgID]; rc != nil { + c.awaiting[msgID] = nil + rc <- asyncResult{resp.Data, nil} + close(rc) + } + c.awaitingMut.Unlock() +} + +func (c *rawConnection) handlePong(msgID int) { + c.awaitingMut.Lock() + if rc := c.awaiting[msgID]; rc != nil { + c.awaiting[msgID] = nil + rc <- asyncResult{} + close(rc) + } + c.awaitingMut.Unlock() +} + +func (c *rawConnection) send(msgID int, msgType int, msg encodable) bool { + if msgID < 0 { + select { + case id := <-c.nextID: + msgID = id + case <-c.closed: + return false + } + } + + hdr := header{ + version: 0, + msgID: msgID, + msgType: msgType, + } + + select { + case c.outbox <- hdrMsg{hdr, msg}: + return true + case <-c.closed: + return false + } +} + +func (c *rawConnection) writerLoop() { + var msgBuf = make([]byte, 8) // buffer for wire format message, kept and reused + var uncBuf []byte // buffer for uncompressed message, kept and reused + for { + var tempBuf []byte + var err error + + select { + case hm := <-c.outbox: + if hm.msg != nil { + // Uncompressed message in uncBuf + uncBuf = hm.msg.AppendXDR(uncBuf[:0]) + + if len(uncBuf) >= c.compressionThreshold { + // Use compression for large messages + hm.hdr.compression = true + + // Make sure we have enough space for the compressed message plus header in msgBug + msgBuf = msgBuf[:cap(msgBuf)] + if maxLen := lz4.CompressBound(len(uncBuf)) + 8; maxLen > len(msgBuf) { + msgBuf = make([]byte, maxLen) + } + + // Compressed is written to msgBuf, we keep tb for the length only + tempBuf, err = lz4.Encode(msgBuf[8:], uncBuf) + binary.BigEndian.PutUint32(msgBuf[4:8], uint32(len(tempBuf))) + msgBuf = msgBuf[0 : len(tempBuf)+8] + + if debug { + l.Debugf("write compressed message; %v (len=%d)", hm.hdr, len(tempBuf)) + } + } else { + // No point in compressing very short messages + hm.hdr.compression = false + + msgBuf = msgBuf[:cap(msgBuf)] + if l := len(uncBuf) + 8; l > len(msgBuf) { + msgBuf = make([]byte, l) + } + + binary.BigEndian.PutUint32(msgBuf[4:8], uint32(len(uncBuf))) + msgBuf = msgBuf[0 : len(uncBuf)+8] + copy(msgBuf[8:], uncBuf) + + if debug { + l.Debugf("write uncompressed message; %v (len=%d)", hm.hdr, len(uncBuf)) + } + } + } else { + if debug { + l.Debugf("write empty message; %v", hm.hdr) + } + binary.BigEndian.PutUint32(msgBuf[4:8], 0) + msgBuf = msgBuf[:8] + } + + binary.BigEndian.PutUint32(msgBuf[0:4], encodeHeader(hm.hdr)) + + if err == nil { + var n int + n, err = c.cw.Write(msgBuf) + if debug { + l.Debugf("wrote %d bytes on the wire", n) + } + } + if err != nil { + c.close(err) + return + } + case <-c.closed: + return + } + } +} + +func (c *rawConnection) close(err error) { + c.once.Do(func() { + close(c.closed) + + c.awaitingMut.Lock() + for i, ch := range c.awaiting { + if ch != nil { + close(ch) + c.awaiting[i] = nil + } + } + c.awaitingMut.Unlock() + + go c.receiver.Close(c.id, err) + }) +} + +func (c *rawConnection) idGenerator() { + nextID := 0 + for { + nextID = (nextID + 1) & 0xfff + select { + case c.nextID <- nextID: + case <-c.closed: + return + } + } +} + +func (c *rawConnection) pingerLoop() { + var rc = make(chan bool, 1) + ticker := time.Tick(pingIdleTime / 2) + for { + select { + case <-ticker: + if d := time.Since(c.cr.Last()); d < pingIdleTime { + if debug { + l.Debugln(c.id, "ping skipped after rd", d) + } + continue + } + if d := time.Since(c.cw.Last()); d < pingIdleTime { + if debug { + l.Debugln(c.id, "ping skipped after wr", d) + } + continue + } + go func() { + if debug { + l.Debugln(c.id, "ping ->") + } + rc <- c.ping() + }() + select { + case ok := <-rc: + if debug { + l.Debugln(c.id, "<- pong") + } + if !ok { + c.close(fmt.Errorf("ping failure")) + } + case <-time.After(pingTimeout): + c.close(fmt.Errorf("ping timeout")) + case <-c.closed: + return + } + + case <-c.closed: + return + } + } +} + +type Statistics struct { + At time.Time + InBytesTotal uint64 + OutBytesTotal uint64 +} + +func (c *rawConnection) Statistics() Statistics { + return Statistics{ + At: time.Now(), + InBytesTotal: c.cr.Tot(), + OutBytesTotal: c.cw.Tot(), + } +} + +func IsDeleted(bits uint32) bool { + return bits&FlagDeleted != 0 +} + +func IsInvalid(bits uint32) bool { + return bits&FlagInvalid != 0 +} + +func IsDirectory(bits uint32) bool { + return bits&FlagDirectory != 0 +} + +func HasPermissionBits(bits uint32) bool { + return bits&FlagNoPermBits == 0 +} diff --git a/protocol_test.go b/protocol_test.go new file mode 100644 index 000000000..56a46f2ec --- /dev/null +++ b/protocol_test.go @@ -0,0 +1,383 @@ +// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// All rights reserved. Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +package protocol + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "os" + "reflect" + "testing" + "testing/quick" + + "github.com/calmh/xdr" +) + +var ( + c0ID = NewNodeID([]byte{1}) + c1ID = NewNodeID([]byte{2}) +) + +func TestHeaderFunctions(t *testing.T) { + f := func(ver, id, typ int) bool { + ver = int(uint(ver) % 16) + id = int(uint(id) % 4096) + typ = int(uint(typ) % 256) + h0 := header{version: ver, msgID: id, msgType: typ} + h1 := decodeHeader(encodeHeader(h0)) + return h0 == h1 + } + if err := quick.Check(f, nil); err != nil { + t.Error(err) + } +} + +func TestHeaderLayout(t *testing.T) { + var e, a uint32 + + // Version are the first four bits + e = 0xf0000000 + a = encodeHeader(header{version: 0xf}) + if a != e { + t.Errorf("Header layout incorrect; %08x != %08x", a, e) + } + + // Message ID are the following 12 bits + e = 0x0fff0000 + a = encodeHeader(header{msgID: 0xfff}) + if a != e { + t.Errorf("Header layout incorrect; %08x != %08x", a, e) + } + + // Type are the last 8 bits before reserved + e = 0x0000ff00 + a = encodeHeader(header{msgType: 0xff}) + if a != e { + t.Errorf("Header layout incorrect; %08x != %08x", a, e) + } +} + +func TestPing(t *testing.T) { + ar, aw := io.Pipe() + br, bw := io.Pipe() + + c0 := NewConnection(c0ID, ar, bw, nil, "name", true).(wireFormatConnection).next.(*rawConnection) + c1 := NewConnection(c1ID, br, aw, nil, "name", true).(wireFormatConnection).next.(*rawConnection) + + if ok := c0.ping(); !ok { + t.Error("c0 ping failed") + } + if ok := c1.ping(); !ok { + t.Error("c1 ping failed") + } +} + +func TestPingErr(t *testing.T) { + e := errors.New("something broke") + + for i := 0; i < 16; i++ { + for j := 0; j < 16; j++ { + m0 := newTestModel() + m1 := newTestModel() + + ar, aw := io.Pipe() + br, bw := io.Pipe() + eaw := &ErrPipe{PipeWriter: *aw, max: i, err: e} + ebw := &ErrPipe{PipeWriter: *bw, max: j, err: e} + + c0 := NewConnection(c0ID, ar, ebw, m0, "name", true).(wireFormatConnection).next.(*rawConnection) + NewConnection(c1ID, br, eaw, m1, "name", true) + + res := c0.ping() + if (i < 8 || j < 8) && res { + t.Errorf("Unexpected ping success; i=%d, j=%d", i, j) + } else if (i >= 12 && j >= 12) && !res { + t.Errorf("Unexpected ping fail; i=%d, j=%d", i, j) + } + } + } +} + +// func TestRequestResponseErr(t *testing.T) { +// e := errors.New("something broke") + +// var pass bool +// for i := 0; i < 48; i++ { +// for j := 0; j < 38; j++ { +// m0 := newTestModel() +// m0.data = []byte("response data") +// m1 := newTestModel() + +// ar, aw := io.Pipe() +// br, bw := io.Pipe() +// eaw := &ErrPipe{PipeWriter: *aw, max: i, err: e} +// ebw := &ErrPipe{PipeWriter: *bw, max: j, err: e} + +// NewConnection(c0ID, ar, ebw, m0, nil) +// c1 := NewConnection(c1ID, br, eaw, m1, nil).(wireFormatConnection).next.(*rawConnection) + +// d, err := c1.Request("default", "tn", 1234, 5678) +// if err == e || err == ErrClosed { +// t.Logf("Error at %d+%d bytes", i, j) +// if !m1.isClosed() { +// t.Fatal("c1 not closed") +// } +// if !m0.isClosed() { +// t.Fatal("c0 not closed") +// } +// continue +// } +// if err != nil { +// t.Fatal(err) +// } +// if string(d) != "response data" { +// t.Fatalf("Incorrect response data %q", string(d)) +// } +// if m0.repo != "default" { +// t.Fatalf("Incorrect repo %q", m0.repo) +// } +// if m0.name != "tn" { +// t.Fatalf("Incorrect name %q", m0.name) +// } +// if m0.offset != 1234 { +// t.Fatalf("Incorrect offset %d", m0.offset) +// } +// if m0.size != 5678 { +// t.Fatalf("Incorrect size %d", m0.size) +// } +// t.Logf("Pass at %d+%d bytes", i, j) +// pass = true +// } +// } +// if !pass { +// t.Fatal("Never passed") +// } +// } + +func TestVersionErr(t *testing.T) { + m0 := newTestModel() + m1 := newTestModel() + + ar, aw := io.Pipe() + br, bw := io.Pipe() + + c0 := NewConnection(c0ID, ar, bw, m0, "name", true).(wireFormatConnection).next.(*rawConnection) + NewConnection(c1ID, br, aw, m1, "name", true) + + w := xdr.NewWriter(c0.cw) + w.WriteUint32(encodeHeader(header{ + version: 2, + msgID: 0, + msgType: 0, + })) + w.WriteUint32(0) + + if !m1.isClosed() { + t.Error("Connection should close due to unknown version") + } +} + +func TestTypeErr(t *testing.T) { + m0 := newTestModel() + m1 := newTestModel() + + ar, aw := io.Pipe() + br, bw := io.Pipe() + + c0 := NewConnection(c0ID, ar, bw, m0, "name", true).(wireFormatConnection).next.(*rawConnection) + NewConnection(c1ID, br, aw, m1, "name", true) + + w := xdr.NewWriter(c0.cw) + w.WriteUint32(encodeHeader(header{ + version: 0, + msgID: 0, + msgType: 42, + })) + w.WriteUint32(0) + + if !m1.isClosed() { + t.Error("Connection should close due to unknown message type") + } +} + +func TestClose(t *testing.T) { + m0 := newTestModel() + m1 := newTestModel() + + ar, aw := io.Pipe() + br, bw := io.Pipe() + + c0 := NewConnection(c0ID, ar, bw, m0, "name", true).(wireFormatConnection).next.(*rawConnection) + NewConnection(c1ID, br, aw, m1, "name", true) + + c0.close(nil) + + <-c0.closed + if !m0.isClosed() { + t.Fatal("Connection should be closed") + } + + // None of these should panic, some should return an error + + if c0.ping() { + t.Error("Ping should not return true") + } + + c0.Index("default", nil) + c0.Index("default", nil) + + if _, err := c0.Request("default", "foo", 0, 0); err == nil { + t.Error("Request should return an error") + } +} + +func TestElementSizeExceededNested(t *testing.T) { + m := ClusterConfigMessage{ + Repositories: []Repository{ + {ID: "longstringlongstringlongstringinglongstringlongstringlonlongstringlongstringlon"}, + }, + } + _, err := m.EncodeXDR(ioutil.Discard) + if err == nil { + t.Errorf("ID length %d > max 64, but no error", len(m.Repositories[0].ID)) + } +} + +func TestMarshalIndexMessage(t *testing.T) { + var quickCfg = &quick.Config{MaxCountScale: 10} + if testing.Short() { + quickCfg = nil + } + + f := func(m1 IndexMessage) bool { + for _, f := range m1.Files { + for i := range f.Blocks { + f.Blocks[i].Offset = 0 + if len(f.Blocks[i].Hash) == 0 { + f.Blocks[i].Hash = nil + } + } + } + + return testMarshal(t, "index", &m1, &IndexMessage{}) + } + + if err := quick.Check(f, quickCfg); err != nil { + t.Error(err) + } +} + +func TestMarshalRequestMessage(t *testing.T) { + var quickCfg = &quick.Config{MaxCountScale: 10} + if testing.Short() { + quickCfg = nil + } + + f := func(m1 RequestMessage) bool { + return testMarshal(t, "request", &m1, &RequestMessage{}) + } + + if err := quick.Check(f, quickCfg); err != nil { + t.Error(err) + } +} + +func TestMarshalResponseMessage(t *testing.T) { + var quickCfg = &quick.Config{MaxCountScale: 10} + if testing.Short() { + quickCfg = nil + } + + f := func(m1 ResponseMessage) bool { + if len(m1.Data) == 0 { + m1.Data = nil + } + return testMarshal(t, "response", &m1, &ResponseMessage{}) + } + + if err := quick.Check(f, quickCfg); err != nil { + t.Error(err) + } +} + +func TestMarshalClusterConfigMessage(t *testing.T) { + var quickCfg = &quick.Config{MaxCountScale: 10} + if testing.Short() { + quickCfg = nil + } + + f := func(m1 ClusterConfigMessage) bool { + return testMarshal(t, "clusterconfig", &m1, &ClusterConfigMessage{}) + } + + if err := quick.Check(f, quickCfg); err != nil { + t.Error(err) + } +} + +func TestMarshalCloseMessage(t *testing.T) { + var quickCfg = &quick.Config{MaxCountScale: 10} + if testing.Short() { + quickCfg = nil + } + + f := func(m1 CloseMessage) bool { + return testMarshal(t, "close", &m1, &CloseMessage{}) + } + + if err := quick.Check(f, quickCfg); err != nil { + t.Error(err) + } +} + +type message interface { + EncodeXDR(io.Writer) (int, error) + DecodeXDR(io.Reader) error +} + +func testMarshal(t *testing.T, prefix string, m1, m2 message) bool { + var buf bytes.Buffer + + failed := func(bc []byte) { + bs, _ := json.MarshalIndent(m1, "", " ") + ioutil.WriteFile(prefix+"-1.txt", bs, 0644) + bs, _ = json.MarshalIndent(m2, "", " ") + ioutil.WriteFile(prefix+"-2.txt", bs, 0644) + if len(bc) > 0 { + f, _ := os.Create(prefix + "-data.txt") + fmt.Fprint(f, hex.Dump(bc)) + f.Close() + } + } + + _, err := m1.EncodeXDR(&buf) + if err == xdr.ErrElementSizeExceeded { + return true + } + if err != nil { + failed(nil) + t.Fatal(err) + } + + bc := make([]byte, len(buf.Bytes())) + copy(bc, buf.Bytes()) + + err = m2.DecodeXDR(&buf) + if err != nil { + failed(bc) + t.Fatal(err) + } + + ok := reflect.DeepEqual(m1, m2) + if !ok { + failed(bc) + } + return ok +} diff --git a/wireformat.go b/wireformat.go new file mode 100644 index 000000000..987c03eff --- /dev/null +++ b/wireformat.go @@ -0,0 +1,58 @@ +// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// All rights reserved. Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +package protocol + +import ( + "path/filepath" + + "code.google.com/p/go.text/unicode/norm" +) + +type wireFormatConnection struct { + next Connection +} + +func (c wireFormatConnection) ID() NodeID { + return c.next.ID() +} + +func (c wireFormatConnection) Name() string { + return c.next.Name() +} + +func (c wireFormatConnection) Index(repo string, fs []FileInfo) error { + var myFs = make([]FileInfo, len(fs)) + copy(myFs, fs) + + for i := range fs { + myFs[i].Name = norm.NFC.String(filepath.ToSlash(myFs[i].Name)) + } + + return c.next.Index(repo, myFs) +} + +func (c wireFormatConnection) IndexUpdate(repo string, fs []FileInfo) error { + var myFs = make([]FileInfo, len(fs)) + copy(myFs, fs) + + for i := range fs { + myFs[i].Name = norm.NFC.String(filepath.ToSlash(myFs[i].Name)) + } + + return c.next.IndexUpdate(repo, myFs) +} + +func (c wireFormatConnection) Request(repo, name string, offset int64, size int) ([]byte, error) { + name = norm.NFC.String(filepath.ToSlash(name)) + return c.next.Request(repo, name, offset, size) +} + +func (c wireFormatConnection) ClusterConfig(config ClusterConfigMessage) { + c.next.ClusterConfig(config) +} + +func (c wireFormatConnection) Statistics() Statistics { + return c.next.Statistics() +} From 4b488a2d28ea2bdff814ac9a4e6be3b26475dab1 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Sun, 28 Sep 2014 12:00:38 +0100 Subject: [PATCH 02/94] Rename Repository -> Folder, Node -> Device (fixes #739) --- common_test.go | 14 ++-- nodeid.go => deviceid.go | 36 +++++----- nodeid_test.go => deviceid_test.go | 20 +++--- message.go | 12 ++-- message_xdr.go | 108 ++++++++++++++--------------- nativemodel_darwin.go | 20 +++--- nativemodel_unix.go | 20 +++--- nativemodel_windows.go | 20 +++--- protocol.go | 60 ++++++++-------- protocol_test.go | 12 ++-- wireformat.go | 14 ++-- 11 files changed, 168 insertions(+), 168 deletions(-) rename nodeid.go => deviceid.go (74%) rename nodeid_test.go => deviceid_test.go (82%) diff --git a/common_test.go b/common_test.go index 9d387825c..8d62c8ee1 100644 --- a/common_test.go +++ b/common_test.go @@ -11,7 +11,7 @@ import ( type TestModel struct { data []byte - repo string + folder string name string offset int64 size int @@ -24,25 +24,25 @@ func newTestModel() *TestModel { } } -func (t *TestModel) Index(nodeID NodeID, repo string, files []FileInfo) { +func (t *TestModel) Index(deviceID DeviceID, folder string, files []FileInfo) { } -func (t *TestModel) IndexUpdate(nodeID NodeID, repo string, files []FileInfo) { +func (t *TestModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) { } -func (t *TestModel) Request(nodeID NodeID, repo, name string, offset int64, size int) ([]byte, error) { - t.repo = repo +func (t *TestModel) Request(deviceID DeviceID, folder, name string, offset int64, size int) ([]byte, error) { + t.folder = folder t.name = name t.offset = offset t.size = size return t.data, nil } -func (t *TestModel) Close(nodeID NodeID, err error) { +func (t *TestModel) Close(deviceID DeviceID, err error) { close(t.closedCh) } -func (t *TestModel) ClusterConfig(nodeID NodeID, config ClusterConfigMessage) { +func (t *TestModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) { } func (t *TestModel) isClosed() bool { diff --git a/nodeid.go b/deviceid.go similarity index 74% rename from nodeid.go rename to deviceid.go index 9079781b9..f4427dc6f 100644 --- a/nodeid.go +++ b/deviceid.go @@ -16,36 +16,36 @@ import ( "github.com/syncthing/syncthing/internal/luhn" ) -type NodeID [32]byte +type DeviceID [32]byte -var LocalNodeID = NodeID{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff} +var LocalDeviceID = DeviceID{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff} -// NewNodeID generates a new node ID from the raw bytes of a certificate -func NewNodeID(rawCert []byte) NodeID { - var n NodeID +// NewDeviceID generates a new device ID from the raw bytes of a certificate +func NewDeviceID(rawCert []byte) DeviceID { + var n DeviceID hf := sha256.New() hf.Write(rawCert) hf.Sum(n[:0]) return n } -func NodeIDFromString(s string) (NodeID, error) { - var n NodeID +func DeviceIDFromString(s string) (DeviceID, error) { + var n DeviceID err := n.UnmarshalText([]byte(s)) return n, err } -func NodeIDFromBytes(bs []byte) NodeID { - var n NodeID +func DeviceIDFromBytes(bs []byte) DeviceID { + var n DeviceID if len(bs) != len(n) { - panic("incorrect length of byte slice representing node ID") + panic("incorrect length of byte slice representing device ID") } copy(n[:], bs) return n } -// String returns the canonical string representation of the node ID -func (n NodeID) String() string { +// String returns the canonical string representation of the device ID +func (n DeviceID) String() string { id := base32.StdEncoding.EncodeToString(n[:]) id = strings.Trim(id, "=") id, err := luhnify(id) @@ -57,23 +57,23 @@ func (n NodeID) String() string { return id } -func (n NodeID) GoString() string { +func (n DeviceID) GoString() string { return n.String() } -func (n NodeID) Compare(other NodeID) int { +func (n DeviceID) Compare(other DeviceID) int { return bytes.Compare(n[:], other[:]) } -func (n NodeID) Equals(other NodeID) bool { +func (n DeviceID) Equals(other DeviceID) bool { return bytes.Compare(n[:], other[:]) == 0 } -func (n *NodeID) MarshalText() ([]byte, error) { +func (n *DeviceID) MarshalText() ([]byte, error) { return []byte(n.String()), nil } -func (n *NodeID) UnmarshalText(bs []byte) error { +func (n *DeviceID) UnmarshalText(bs []byte) error { id := string(bs) id = strings.Trim(id, "=") id = strings.ToUpper(id) @@ -98,7 +98,7 @@ func (n *NodeID) UnmarshalText(bs []byte) error { copy(n[:], dec) return nil default: - return errors.New("node ID invalid: incorrect length") + return errors.New("device ID invalid: incorrect length") } } diff --git a/nodeid_test.go b/deviceid_test.go similarity index 82% rename from nodeid_test.go rename to deviceid_test.go index 5b861b6de..069e4b988 100644 --- a/nodeid_test.go +++ b/deviceid_test.go @@ -20,14 +20,14 @@ var formatCases = []string{ "p561017mzjnu2yiqgdreydm2mgtimgl3bxnpq6w5bmt88z4tjxzwicq2", } -func TestFormatNodeID(t *testing.T) { +func TestFormatDeviceID(t *testing.T) { for i, tc := range formatCases { - var id NodeID + var id DeviceID err := id.UnmarshalText([]byte(tc)) if err != nil { t.Errorf("#%d UnmarshalText(%q); %v", i, tc, err) } else if f := id.String(); f != formatted { - t.Errorf("#%d FormatNodeID(%q)\n\t%q !=\n\t%q", i, tc, f, formatted) + t.Errorf("#%d FormatDeviceID(%q)\n\t%q !=\n\t%q", i, tc, f, formatted) } } } @@ -46,20 +46,20 @@ var validateCases = []struct { {"p56ioi7mzjnu2iqgdreydm2mgtmgl3bxnpq6w5btbbz4tjxzwicqCCCC", false}, } -func TestValidateNodeID(t *testing.T) { +func TestValidateDeviceID(t *testing.T) { for _, tc := range validateCases { - var id NodeID + var id DeviceID err := id.UnmarshalText([]byte(tc.s)) if (err == nil && !tc.ok) || (err != nil && tc.ok) { - t.Errorf("ValidateNodeID(%q); %v != %v", tc.s, err, tc.ok) + t.Errorf("ValidateDeviceID(%q); %v != %v", tc.s, err, tc.ok) } } } -func TestMarshallingNodeID(t *testing.T) { - n0 := NodeID{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 10, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32} - n1 := NodeID{} - n2 := NodeID{} +func TestMarshallingDeviceID(t *testing.T) { + n0 := DeviceID{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 10, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32} + n1 := DeviceID{} + n2 := DeviceID{} bs, _ := n0.MarshalText() n1.UnmarshalText(bs) diff --git a/message.go b/message.go index 779817a7b..09cb323c8 100644 --- a/message.go +++ b/message.go @@ -7,7 +7,7 @@ package protocol import "fmt" type IndexMessage struct { - Repository string // max:64 + Folder string // max:64 Files []FileInfo } @@ -90,7 +90,7 @@ func (b BlockInfo) String() string { } type RequestMessage struct { - Repository string // max:64 + Folder string // max:64 Name string // max:8192 Offset uint64 Size uint32 @@ -103,7 +103,7 @@ type ResponseMessage struct { type ClusterConfigMessage struct { ClientName string // max:64 ClientVersion string // max:64 - Repositories []Repository // max:64 + Folders []Folder // max:64 Options []Option // max:64 } @@ -116,12 +116,12 @@ func (o *ClusterConfigMessage) GetOption(key string) string { return "" } -type Repository struct { +type Folder struct { ID string // max:64 - Nodes []Node // max:64 + Devices []Device // max:64 } -type Node struct { +type Device struct { ID []byte // max:32 Flags uint32 MaxLocalVersion uint64 diff --git a/message_xdr.go b/message_xdr.go index 11a11da34..c7f16a173 100644 --- a/message_xdr.go +++ b/message_xdr.go @@ -18,10 +18,10 @@ IndexMessage Structure: 0 1 2 3 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Length of Repository | +| Length of Folder | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ / / -\ Repository (variable length) \ +\ Folder (variable length) \ / / +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ | Number of Files | @@ -33,7 +33,7 @@ IndexMessage Structure: struct IndexMessage { - string Repository<64>; + string Folder<64>; FileInfo Files<>; } @@ -56,10 +56,10 @@ func (o IndexMessage) AppendXDR(bs []byte) []byte { } func (o IndexMessage) encodeXDR(xw *xdr.Writer) (int, error) { - if len(o.Repository) > 64 { + if len(o.Folder) > 64 { return xw.Tot(), xdr.ErrElementSizeExceeded } - xw.WriteString(o.Repository) + xw.WriteString(o.Folder) xw.WriteUint32(uint32(len(o.Files))) for i := range o.Files { _, err := o.Files[i].encodeXDR(xw) @@ -82,7 +82,7 @@ func (o *IndexMessage) UnmarshalXDR(bs []byte) error { } func (o *IndexMessage) decodeXDR(xr *xdr.Reader) error { - o.Repository = xr.ReadStringMax(64) + o.Folder = xr.ReadStringMax(64) _FilesSize := int(xr.ReadUint32()) o.Files = make([]FileInfo, _FilesSize) for i := range o.Files { @@ -362,10 +362,10 @@ RequestMessage Structure: 0 1 2 3 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Length of Repository | +| Length of Folder | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ / / -\ Repository (variable length) \ +\ Folder (variable length) \ / / +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ | Length of Name | @@ -383,7 +383,7 @@ RequestMessage Structure: struct RequestMessage { - string Repository<64>; + string Folder<64>; string Name<8192>; unsigned hyper Offset; unsigned int Size; @@ -408,10 +408,10 @@ func (o RequestMessage) AppendXDR(bs []byte) []byte { } func (o RequestMessage) encodeXDR(xw *xdr.Writer) (int, error) { - if len(o.Repository) > 64 { + if len(o.Folder) > 64 { return xw.Tot(), xdr.ErrElementSizeExceeded } - xw.WriteString(o.Repository) + xw.WriteString(o.Folder) if len(o.Name) > 8192 { return xw.Tot(), xdr.ErrElementSizeExceeded } @@ -433,7 +433,7 @@ func (o *RequestMessage) UnmarshalXDR(bs []byte) error { } func (o *RequestMessage) decodeXDR(xr *xdr.Reader) error { - o.Repository = xr.ReadStringMax(64) + o.Folder = xr.ReadStringMax(64) o.Name = xr.ReadStringMax(8192) o.Offset = xr.ReadUint64() o.Size = xr.ReadUint32() @@ -517,10 +517,10 @@ ClusterConfigMessage Structure: \ Client Version (variable length) \ / / +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Number of Repositories | +| Number of Folders | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ / / -\ Zero or more Repository Structures \ +\ Zero or more Folder Structures \ / / +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ | Number of Options | @@ -534,7 +534,7 @@ ClusterConfigMessage Structure: struct ClusterConfigMessage { string ClientName<64>; string ClientVersion<64>; - Repository Repositories<64>; + Folder Folders<64>; Option Options<64>; } @@ -565,12 +565,12 @@ func (o ClusterConfigMessage) encodeXDR(xw *xdr.Writer) (int, error) { return xw.Tot(), xdr.ErrElementSizeExceeded } xw.WriteString(o.ClientVersion) - if len(o.Repositories) > 64 { + if len(o.Folders) > 64 { return xw.Tot(), xdr.ErrElementSizeExceeded } - xw.WriteUint32(uint32(len(o.Repositories))) - for i := range o.Repositories { - _, err := o.Repositories[i].encodeXDR(xw) + xw.WriteUint32(uint32(len(o.Folders))) + for i := range o.Folders { + _, err := o.Folders[i].encodeXDR(xw) if err != nil { return xw.Tot(), err } @@ -602,13 +602,13 @@ func (o *ClusterConfigMessage) UnmarshalXDR(bs []byte) error { func (o *ClusterConfigMessage) decodeXDR(xr *xdr.Reader) error { o.ClientName = xr.ReadStringMax(64) o.ClientVersion = xr.ReadStringMax(64) - _RepositoriesSize := int(xr.ReadUint32()) - if _RepositoriesSize > 64 { + _FoldersSize := int(xr.ReadUint32()) + if _FoldersSize > 64 { return xdr.ErrElementSizeExceeded } - o.Repositories = make([]Repository, _RepositoriesSize) - for i := range o.Repositories { - (&o.Repositories[i]).decodeXDR(xr) + o.Folders = make([]Folder, _FoldersSize) + for i := range o.Folders { + (&o.Folders[i]).decodeXDR(xr) } _OptionsSize := int(xr.ReadUint32()) if _OptionsSize > 64 { @@ -623,7 +623,7 @@ func (o *ClusterConfigMessage) decodeXDR(xr *xdr.Reader) error { /* -Repository Structure: +Folder Structure: 0 1 2 3 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 @@ -634,48 +634,48 @@ Repository Structure: \ ID (variable length) \ / / +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Number of Nodes | +| Number of Devices | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ / / -\ Zero or more Node Structures \ +\ Zero or more Device Structures \ / / +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -struct Repository { +struct Folder { string ID<64>; - Node Nodes<64>; + Device Devices<64>; } */ -func (o Repository) EncodeXDR(w io.Writer) (int, error) { +func (o Folder) EncodeXDR(w io.Writer) (int, error) { var xw = xdr.NewWriter(w) return o.encodeXDR(xw) } -func (o Repository) MarshalXDR() []byte { +func (o Folder) MarshalXDR() []byte { return o.AppendXDR(make([]byte, 0, 128)) } -func (o Repository) AppendXDR(bs []byte) []byte { +func (o Folder) AppendXDR(bs []byte) []byte { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) o.encodeXDR(xw) return []byte(aw) } -func (o Repository) encodeXDR(xw *xdr.Writer) (int, error) { +func (o Folder) encodeXDR(xw *xdr.Writer) (int, error) { if len(o.ID) > 64 { return xw.Tot(), xdr.ErrElementSizeExceeded } xw.WriteString(o.ID) - if len(o.Nodes) > 64 { + if len(o.Devices) > 64 { return xw.Tot(), xdr.ErrElementSizeExceeded } - xw.WriteUint32(uint32(len(o.Nodes))) - for i := range o.Nodes { - _, err := o.Nodes[i].encodeXDR(xw) + xw.WriteUint32(uint32(len(o.Devices))) + for i := range o.Devices { + _, err := o.Devices[i].encodeXDR(xw) if err != nil { return xw.Tot(), err } @@ -683,33 +683,33 @@ func (o Repository) encodeXDR(xw *xdr.Writer) (int, error) { return xw.Tot(), xw.Error() } -func (o *Repository) DecodeXDR(r io.Reader) error { +func (o *Folder) DecodeXDR(r io.Reader) error { xr := xdr.NewReader(r) return o.decodeXDR(xr) } -func (o *Repository) UnmarshalXDR(bs []byte) error { +func (o *Folder) UnmarshalXDR(bs []byte) error { var br = bytes.NewReader(bs) var xr = xdr.NewReader(br) return o.decodeXDR(xr) } -func (o *Repository) decodeXDR(xr *xdr.Reader) error { +func (o *Folder) decodeXDR(xr *xdr.Reader) error { o.ID = xr.ReadStringMax(64) - _NodesSize := int(xr.ReadUint32()) - if _NodesSize > 64 { + _DevicesSize := int(xr.ReadUint32()) + if _DevicesSize > 64 { return xdr.ErrElementSizeExceeded } - o.Nodes = make([]Node, _NodesSize) - for i := range o.Nodes { - (&o.Nodes[i]).decodeXDR(xr) + o.Devices = make([]Device, _DevicesSize) + for i := range o.Devices { + (&o.Devices[i]).decodeXDR(xr) } return xr.Error() } /* -Node Structure: +Device Structure: 0 1 2 3 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 @@ -728,7 +728,7 @@ Node Structure: +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -struct Node { +struct Device { opaque ID<32>; unsigned int Flags; unsigned hyper MaxLocalVersion; @@ -736,23 +736,23 @@ struct Node { */ -func (o Node) EncodeXDR(w io.Writer) (int, error) { +func (o Device) EncodeXDR(w io.Writer) (int, error) { var xw = xdr.NewWriter(w) return o.encodeXDR(xw) } -func (o Node) MarshalXDR() []byte { +func (o Device) MarshalXDR() []byte { return o.AppendXDR(make([]byte, 0, 128)) } -func (o Node) AppendXDR(bs []byte) []byte { +func (o Device) AppendXDR(bs []byte) []byte { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) o.encodeXDR(xw) return []byte(aw) } -func (o Node) encodeXDR(xw *xdr.Writer) (int, error) { +func (o Device) encodeXDR(xw *xdr.Writer) (int, error) { if len(o.ID) > 32 { return xw.Tot(), xdr.ErrElementSizeExceeded } @@ -762,18 +762,18 @@ func (o Node) encodeXDR(xw *xdr.Writer) (int, error) { return xw.Tot(), xw.Error() } -func (o *Node) DecodeXDR(r io.Reader) error { +func (o *Device) DecodeXDR(r io.Reader) error { xr := xdr.NewReader(r) return o.decodeXDR(xr) } -func (o *Node) UnmarshalXDR(bs []byte) error { +func (o *Device) UnmarshalXDR(bs []byte) error { var br = bytes.NewReader(bs) var xr = xdr.NewReader(br) return o.decodeXDR(xr) } -func (o *Node) decodeXDR(xr *xdr.Reader) error { +func (o *Device) decodeXDR(xr *xdr.Reader) error { o.ID = xr.ReadBytesMax(32) o.Flags = xr.ReadUint32() o.MaxLocalVersion = xr.ReadUint64() diff --git a/nativemodel_darwin.go b/nativemodel_darwin.go index 9ac402fe6..8e8c7a421 100644 --- a/nativemodel_darwin.go +++ b/nativemodel_darwin.go @@ -14,29 +14,29 @@ type nativeModel struct { next Model } -func (m nativeModel) Index(nodeID NodeID, repo string, files []FileInfo) { +func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo) { for i := range files { files[i].Name = norm.NFD.String(files[i].Name) } - m.next.Index(nodeID, repo, files) + m.next.Index(deviceID, folder, files) } -func (m nativeModel) IndexUpdate(nodeID NodeID, repo string, files []FileInfo) { +func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) { for i := range files { files[i].Name = norm.NFD.String(files[i].Name) } - m.next.IndexUpdate(nodeID, repo, files) + m.next.IndexUpdate(deviceID, folder, files) } -func (m nativeModel) Request(nodeID NodeID, repo string, name string, offset int64, size int) ([]byte, error) { +func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int) ([]byte, error) { name = norm.NFD.String(name) - return m.next.Request(nodeID, repo, name, offset, size) + return m.next.Request(deviceID, folder, name, offset, size) } -func (m nativeModel) ClusterConfig(nodeID NodeID, config ClusterConfigMessage) { - m.next.ClusterConfig(nodeID, config) +func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) { + m.next.ClusterConfig(deviceID, config) } -func (m nativeModel) Close(nodeID NodeID, err error) { - m.next.Close(nodeID, err) +func (m nativeModel) Close(deviceID DeviceID, err error) { + m.next.Close(deviceID, err) } diff --git a/nativemodel_unix.go b/nativemodel_unix.go index 23fbe0b6b..77d335fae 100644 --- a/nativemodel_unix.go +++ b/nativemodel_unix.go @@ -12,22 +12,22 @@ type nativeModel struct { next Model } -func (m nativeModel) Index(nodeID NodeID, repo string, files []FileInfo) { - m.next.Index(nodeID, repo, files) +func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo) { + m.next.Index(deviceID, folder, files) } -func (m nativeModel) IndexUpdate(nodeID NodeID, repo string, files []FileInfo) { - m.next.IndexUpdate(nodeID, repo, files) +func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) { + m.next.IndexUpdate(deviceID, folder, files) } -func (m nativeModel) Request(nodeID NodeID, repo string, name string, offset int64, size int) ([]byte, error) { - return m.next.Request(nodeID, repo, name, offset, size) +func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int) ([]byte, error) { + return m.next.Request(deviceID, folder, name, offset, size) } -func (m nativeModel) ClusterConfig(nodeID NodeID, config ClusterConfigMessage) { - m.next.ClusterConfig(nodeID, config) +func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) { + m.next.ClusterConfig(deviceID, config) } -func (m nativeModel) Close(nodeID NodeID, err error) { - m.next.Close(nodeID, err) +func (m nativeModel) Close(deviceID DeviceID, err error) { + m.next.Close(deviceID, err) } diff --git a/nativemodel_windows.go b/nativemodel_windows.go index 9841d63f7..252860ea5 100644 --- a/nativemodel_windows.go +++ b/nativemodel_windows.go @@ -26,7 +26,7 @@ type nativeModel struct { next Model } -func (m nativeModel) Index(nodeID NodeID, repo string, files []FileInfo) { +func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo) { for i, f := range files { if strings.ContainsAny(f.Name, disallowedCharacters) { if f.IsDeleted() { @@ -39,10 +39,10 @@ func (m nativeModel) Index(nodeID NodeID, repo string, files []FileInfo) { } files[i].Name = filepath.FromSlash(f.Name) } - m.next.Index(nodeID, repo, files) + m.next.Index(deviceID, folder, files) } -func (m nativeModel) IndexUpdate(nodeID NodeID, repo string, files []FileInfo) { +func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) { for i, f := range files { if strings.ContainsAny(f.Name, disallowedCharacters) { if f.IsDeleted() { @@ -55,18 +55,18 @@ func (m nativeModel) IndexUpdate(nodeID NodeID, repo string, files []FileInfo) { } files[i].Name = filepath.FromSlash(files[i].Name) } - m.next.IndexUpdate(nodeID, repo, files) + m.next.IndexUpdate(deviceID, folder, files) } -func (m nativeModel) Request(nodeID NodeID, repo string, name string, offset int64, size int) ([]byte, error) { +func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int) ([]byte, error) { name = filepath.FromSlash(name) - return m.next.Request(nodeID, repo, name, offset, size) + return m.next.Request(deviceID, folder, name, offset, size) } -func (m nativeModel) ClusterConfig(nodeID NodeID, config ClusterConfigMessage) { - m.next.ClusterConfig(nodeID, config) +func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) { + m.next.ClusterConfig(deviceID, config) } -func (m nativeModel) Close(nodeID NodeID, err error) { - m.next.Close(nodeID, err) +func (m nativeModel) Close(deviceID DeviceID, err error) { + m.next.Close(deviceID, err) } diff --git a/protocol.go b/protocol.go index 86fd6199f..26d75de4f 100644 --- a/protocol.go +++ b/protocol.go @@ -57,30 +57,30 @@ var ( ) type Model interface { - // An index was received from the peer node - Index(nodeID NodeID, repo string, files []FileInfo) - // An index update was received from the peer node - IndexUpdate(nodeID NodeID, repo string, files []FileInfo) - // A request was made by the peer node - Request(nodeID NodeID, repo string, name string, offset int64, size int) ([]byte, error) + // An index was received from the peer device + Index(deviceID DeviceID, folder string, files []FileInfo) + // An index update was received from the peer device + IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) + // A request was made by the peer device + Request(deviceID DeviceID, folder string, name string, offset int64, size int) ([]byte, error) // A cluster configuration message was received - ClusterConfig(nodeID NodeID, config ClusterConfigMessage) - // The peer node closed the connection - Close(nodeID NodeID, err error) + ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) + // The peer device closed the connection + Close(deviceID DeviceID, err error) } type Connection interface { - ID() NodeID + ID() DeviceID Name() string - Index(repo string, files []FileInfo) error - IndexUpdate(repo string, files []FileInfo) error - Request(repo string, name string, offset int64, size int) ([]byte, error) + Index(folder string, files []FileInfo) error + IndexUpdate(folder string, files []FileInfo) error + Request(folder string, name string, offset int64, size int) ([]byte, error) ClusterConfig(config ClusterConfigMessage) Statistics() Statistics } type rawConnection struct { - id NodeID + id DeviceID name string receiver Model state int @@ -123,7 +123,7 @@ const ( pingIdleTime = 60 * time.Second ) -func NewConnection(nodeID NodeID, reader io.Reader, writer io.Writer, receiver Model, name string, compress bool) Connection { +func NewConnection(deviceID DeviceID, reader io.Reader, writer io.Writer, receiver Model, name string, compress bool) Connection { cr := &countingReader{Reader: reader} cw := &countingWriter{Writer: writer} @@ -132,7 +132,7 @@ func NewConnection(nodeID NodeID, reader io.Reader, writer io.Writer, receiver M compThres = 128 // compress messages that are 128 bytes long or larger } c := rawConnection{ - id: nodeID, + id: deviceID, name: name, receiver: nativeModel{receiver}, state: stateInitial, @@ -152,7 +152,7 @@ func NewConnection(nodeID NodeID, reader io.Reader, writer io.Writer, receiver M return wireFormatConnection{&c} } -func (c *rawConnection) ID() NodeID { +func (c *rawConnection) ID() DeviceID { return c.id } @@ -160,34 +160,34 @@ func (c *rawConnection) Name() string { return c.name } -// Index writes the list of file information to the connected peer node -func (c *rawConnection) Index(repo string, idx []FileInfo) error { +// Index writes the list of file information to the connected peer device +func (c *rawConnection) Index(folder string, idx []FileInfo) error { select { case <-c.closed: return ErrClosed default: } c.idxMut.Lock() - c.send(-1, messageTypeIndex, IndexMessage{repo, idx}) + c.send(-1, messageTypeIndex, IndexMessage{folder, idx}) c.idxMut.Unlock() return nil } -// IndexUpdate writes the list of file information to the connected peer node as an update -func (c *rawConnection) IndexUpdate(repo string, idx []FileInfo) error { +// IndexUpdate writes the list of file information to the connected peer device as an update +func (c *rawConnection) IndexUpdate(folder string, idx []FileInfo) error { select { case <-c.closed: return ErrClosed default: } c.idxMut.Lock() - c.send(-1, messageTypeIndexUpdate, IndexMessage{repo, idx}) + c.send(-1, messageTypeIndexUpdate, IndexMessage{folder, idx}) c.idxMut.Unlock() return nil } // Request returns the bytes for the specified block after fetching them from the connected peer. -func (c *rawConnection) Request(repo string, name string, offset int64, size int) ([]byte, error) { +func (c *rawConnection) Request(folder string, name string, offset int64, size int) ([]byte, error) { var id int select { case id = <-c.nextID: @@ -203,7 +203,7 @@ func (c *rawConnection) Request(repo string, name string, offset int64, size int c.awaiting[id] = rc c.awaitingMut.Unlock() - ok := c.send(id, messageTypeRequest, RequestMessage{repo, name, uint64(offset), uint32(size)}) + ok := c.send(id, messageTypeRequest, RequestMessage{folder, name, uint64(offset), uint32(size)}) if !ok { return nil, ErrClosed } @@ -399,20 +399,20 @@ func (c *rawConnection) readMessage() (hdr header, msg encodable, err error) { func (c *rawConnection) handleIndex(im IndexMessage) { if debug { - l.Debugf("Index(%v, %v, %d files)", c.id, im.Repository, len(im.Files)) + l.Debugf("Index(%v, %v, %d files)", c.id, im.Folder, len(im.Files)) } - c.receiver.Index(c.id, im.Repository, im.Files) + c.receiver.Index(c.id, im.Folder, im.Files) } func (c *rawConnection) handleIndexUpdate(im IndexMessage) { if debug { - l.Debugf("queueing IndexUpdate(%v, %v, %d files)", c.id, im.Repository, len(im.Files)) + l.Debugf("queueing IndexUpdate(%v, %v, %d files)", c.id, im.Folder, len(im.Files)) } - c.receiver.IndexUpdate(c.id, im.Repository, im.Files) + c.receiver.IndexUpdate(c.id, im.Folder, im.Files) } func (c *rawConnection) handleRequest(msgID int, req RequestMessage) { - data, _ := c.receiver.Request(c.id, req.Repository, req.Name, int64(req.Offset), int(req.Size)) + data, _ := c.receiver.Request(c.id, req.Folder, req.Name, int64(req.Offset), int(req.Size)) c.send(msgID, messageTypeResponse, ResponseMessage{data}) } diff --git a/protocol_test.go b/protocol_test.go index 56a46f2ec..ee8587586 100644 --- a/protocol_test.go +++ b/protocol_test.go @@ -21,8 +21,8 @@ import ( ) var ( - c0ID = NewNodeID([]byte{1}) - c1ID = NewNodeID([]byte{2}) + c0ID = NewDeviceID([]byte{1}) + c1ID = NewDeviceID([]byte{2}) ) func TestHeaderFunctions(t *testing.T) { @@ -140,8 +140,8 @@ func TestPingErr(t *testing.T) { // if string(d) != "response data" { // t.Fatalf("Incorrect response data %q", string(d)) // } -// if m0.repo != "default" { -// t.Fatalf("Incorrect repo %q", m0.repo) +// if m0.folder != "default" { +// t.Fatalf("Incorrect folder %q", m0.folder) // } // if m0.name != "tn" { // t.Fatalf("Incorrect name %q", m0.name) @@ -240,13 +240,13 @@ func TestClose(t *testing.T) { func TestElementSizeExceededNested(t *testing.T) { m := ClusterConfigMessage{ - Repositories: []Repository{ + Folders: []Folder{ {ID: "longstringlongstringlongstringinglongstringlongstringlonlongstringlongstringlon"}, }, } _, err := m.EncodeXDR(ioutil.Discard) if err == nil { - t.Errorf("ID length %d > max 64, but no error", len(m.Repositories[0].ID)) + t.Errorf("ID length %d > max 64, but no error", len(m.Folders[0].ID)) } } diff --git a/wireformat.go b/wireformat.go index 987c03eff..ebd714cd2 100644 --- a/wireformat.go +++ b/wireformat.go @@ -14,7 +14,7 @@ type wireFormatConnection struct { next Connection } -func (c wireFormatConnection) ID() NodeID { +func (c wireFormatConnection) ID() DeviceID { return c.next.ID() } @@ -22,7 +22,7 @@ func (c wireFormatConnection) Name() string { return c.next.Name() } -func (c wireFormatConnection) Index(repo string, fs []FileInfo) error { +func (c wireFormatConnection) Index(folder string, fs []FileInfo) error { var myFs = make([]FileInfo, len(fs)) copy(myFs, fs) @@ -30,10 +30,10 @@ func (c wireFormatConnection) Index(repo string, fs []FileInfo) error { myFs[i].Name = norm.NFC.String(filepath.ToSlash(myFs[i].Name)) } - return c.next.Index(repo, myFs) + return c.next.Index(folder, myFs) } -func (c wireFormatConnection) IndexUpdate(repo string, fs []FileInfo) error { +func (c wireFormatConnection) IndexUpdate(folder string, fs []FileInfo) error { var myFs = make([]FileInfo, len(fs)) copy(myFs, fs) @@ -41,12 +41,12 @@ func (c wireFormatConnection) IndexUpdate(repo string, fs []FileInfo) error { myFs[i].Name = norm.NFC.String(filepath.ToSlash(myFs[i].Name)) } - return c.next.IndexUpdate(repo, myFs) + return c.next.IndexUpdate(folder, myFs) } -func (c wireFormatConnection) Request(repo, name string, offset int64, size int) ([]byte, error) { +func (c wireFormatConnection) Request(folder, name string, offset int64, size int) ([]byte, error) { name = norm.NFC.String(filepath.ToSlash(name)) - return c.next.Request(repo, name, offset, size) + return c.next.Request(folder, name, offset, size) } func (c wireFormatConnection) ClusterConfig(config ClusterConfigMessage) { From 1bc5632771b9daec3c521ab3353256099da02772 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Sun, 28 Sep 2014 12:05:25 +0100 Subject: [PATCH 03/94] Run go fmt -w --- common_test.go | 2 +- message.go | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/common_test.go b/common_test.go index 8d62c8ee1..e337c7e5b 100644 --- a/common_test.go +++ b/common_test.go @@ -11,7 +11,7 @@ import ( type TestModel struct { data []byte - folder string + folder string name string offset int64 size int diff --git a/message.go b/message.go index 09cb323c8..a99f2d514 100644 --- a/message.go +++ b/message.go @@ -8,7 +8,7 @@ import "fmt" type IndexMessage struct { Folder string // max:64 - Files []FileInfo + Files []FileInfo } type FileInfo struct { @@ -91,9 +91,9 @@ func (b BlockInfo) String() string { type RequestMessage struct { Folder string // max:64 - Name string // max:8192 - Offset uint64 - Size uint32 + Name string // max:8192 + Offset uint64 + Size uint32 } type ResponseMessage struct { @@ -101,10 +101,10 @@ type ResponseMessage struct { } type ClusterConfigMessage struct { - ClientName string // max:64 - ClientVersion string // max:64 - Folders []Folder // max:64 - Options []Option // max:64 + ClientName string // max:64 + ClientVersion string // max:64 + Folders []Folder // max:64 + Options []Option // max:64 } func (o *ClusterConfigMessage) GetOption(key string) string { @@ -117,7 +117,7 @@ func (o *ClusterConfigMessage) GetOption(key string) string { } type Folder struct { - ID string // max:64 + ID string // max:64 Devices []Device // max:64 } From 43289103cbb14f918f64f301099d730980bc641c Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Mon, 29 Sep 2014 21:43:32 +0200 Subject: [PATCH 04/94] Relicense to GPL --- common_test.go | 15 +++++++++++++-- counting.go | 15 +++++++++++++-- debug.go | 15 +++++++++++++-- deviceid.go | 15 +++++++++++++-- deviceid_test.go | 15 +++++++++++++-- doc.go | 15 +++++++++++++-- header.go | 15 +++++++++++++-- message.go | 15 +++++++++++++-- nativemodel_darwin.go | 15 +++++++++++++-- nativemodel_unix.go | 15 +++++++++++++-- nativemodel_windows.go | 15 +++++++++++++-- protocol.go | 15 +++++++++++++-- protocol_test.go | 15 +++++++++++++-- wireformat.go | 15 +++++++++++++-- 14 files changed, 182 insertions(+), 28 deletions(-) diff --git a/common_test.go b/common_test.go index e337c7e5b..29b910bab 100644 --- a/common_test.go +++ b/common_test.go @@ -1,6 +1,17 @@ // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). -// All rights reserved. Use of this source code is governed by an MIT-style -// license that can be found in the LICENSE file. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along +// with this program. If not, see . package protocol diff --git a/counting.go b/counting.go index 512774fba..4f95c9f9a 100644 --- a/counting.go +++ b/counting.go @@ -1,6 +1,17 @@ // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). -// All rights reserved. Use of this source code is governed by an MIT-style -// license that can be found in the LICENSE file. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along +// with this program. If not, see . package protocol diff --git a/debug.go b/debug.go index 6c586b90e..930aaf344 100644 --- a/debug.go +++ b/debug.go @@ -1,6 +1,17 @@ // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). -// All rights reserved. Use of this source code is governed by an MIT-style -// license that can be found in the LICENSE file. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along +// with this program. If not, see . package protocol diff --git a/deviceid.go b/deviceid.go index f4427dc6f..4b034f3f1 100644 --- a/deviceid.go +++ b/deviceid.go @@ -1,6 +1,17 @@ // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). -// All rights reserved. Use of this source code is governed by an MIT-style -// license that can be found in the LICENSE file. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along +// with this program. If not, see . package protocol diff --git a/deviceid_test.go b/deviceid_test.go index 069e4b988..14e2dfa34 100644 --- a/deviceid_test.go +++ b/deviceid_test.go @@ -1,6 +1,17 @@ // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). -// All rights reserved. Use of this source code is governed by an MIT-style -// license that can be found in the LICENSE file. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along +// with this program. If not, see . package protocol diff --git a/doc.go b/doc.go index 8c6b524e6..95b5e102f 100644 --- a/doc.go +++ b/doc.go @@ -1,6 +1,17 @@ // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). -// All rights reserved. Use of this source code is governed by an MIT-style -// license that can be found in the LICENSE file. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along +// with this program. If not, see . // Package protocol implements the Block Exchange Protocol. package protocol diff --git a/header.go b/header.go index 6fd2ebb8f..96ccff66f 100644 --- a/header.go +++ b/header.go @@ -1,6 +1,17 @@ // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). -// All rights reserved. Use of this source code is governed by an MIT-style -// license that can be found in the LICENSE file. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along +// with this program. If not, see . package protocol diff --git a/message.go b/message.go index a99f2d514..feeba69df 100644 --- a/message.go +++ b/message.go @@ -1,6 +1,17 @@ // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). -// All rights reserved. Use of this source code is governed by an MIT-style -// license that can be found in the LICENSE file. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along +// with this program. If not, see . package protocol diff --git a/nativemodel_darwin.go b/nativemodel_darwin.go index 8e8c7a421..7fb5c964e 100644 --- a/nativemodel_darwin.go +++ b/nativemodel_darwin.go @@ -1,6 +1,17 @@ // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). -// All rights reserved. Use of this source code is governed by an MIT-style -// license that can be found in the LICENSE file. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along +// with this program. If not, see . // +build darwin diff --git a/nativemodel_unix.go b/nativemodel_unix.go index 77d335fae..61ad9c438 100644 --- a/nativemodel_unix.go +++ b/nativemodel_unix.go @@ -1,6 +1,17 @@ // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). -// All rights reserved. Use of this source code is governed by an MIT-style -// license that can be found in the LICENSE file. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along +// with this program. If not, see . // +build !windows,!darwin diff --git a/nativemodel_windows.go b/nativemodel_windows.go index 252860ea5..57d3f3446 100644 --- a/nativemodel_windows.go +++ b/nativemodel_windows.go @@ -1,6 +1,17 @@ // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). -// All rights reserved. Use of this source code is governed by an MIT-style -// license that can be found in the LICENSE file. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along +// with this program. If not, see . // +build windows diff --git a/protocol.go b/protocol.go index 26d75de4f..19cdfbb16 100644 --- a/protocol.go +++ b/protocol.go @@ -1,6 +1,17 @@ // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). -// All rights reserved. Use of this source code is governed by an MIT-style -// license that can be found in the LICENSE file. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along +// with this program. If not, see . package protocol diff --git a/protocol_test.go b/protocol_test.go index ee8587586..cc6f9472a 100644 --- a/protocol_test.go +++ b/protocol_test.go @@ -1,6 +1,17 @@ // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). -// All rights reserved. Use of this source code is governed by an MIT-style -// license that can be found in the LICENSE file. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along +// with this program. If not, see . package protocol diff --git a/wireformat.go b/wireformat.go index ebd714cd2..f06b9454e 100644 --- a/wireformat.go +++ b/wireformat.go @@ -1,6 +1,17 @@ // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). -// All rights reserved. Use of this source code is governed by an MIT-style -// license that can be found in the LICENSE file. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation, either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along +// with this program. If not, see . package protocol From 1ef8378a3001dc64f0d174d39b1c745d6c897b1c Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Thu, 16 Oct 2014 09:26:20 +0200 Subject: [PATCH 05/94] FileInfoTruncated.String() for stindex' benefit --- message.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/message.go b/message.go index feeba69df..800787b91 100644 --- a/message.go +++ b/message.go @@ -64,6 +64,11 @@ type FileInfoTruncated struct { NumBlocks uint32 } +func (f FileInfoTruncated) String() string { + return fmt.Sprintf("File{Name:%q, Flags:0%o, Modified:%d, Version:%d, Size:%d, NumBlocks:%d}", + f.Name, f.Flags, f.Modified, f.Version, f.Size(), f.NumBlocks) +} + // Returns a statistical guess on the size, not the exact figure func (f FileInfoTruncated) Size() int64 { if IsDeleted(f.Flags) || IsDirectory(f.Flags) { From c618eba9a9c6466db70e2ce42247723b2ea6c778 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Mon, 6 Oct 2014 21:57:33 +0100 Subject: [PATCH 06/94] Implement BlockMap --- message.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/message.go b/message.go index 800787b91..b47145055 100644 --- a/message.go +++ b/message.go @@ -54,6 +54,10 @@ func (f FileInfo) IsInvalid() bool { return IsInvalid(f.Flags) } +func (f FileInfo) IsDirectory() bool { + return IsDirectory(f.Flags) +} + // Used for unmarshalling a FileInfo structure but skipping the actual block list type FileInfoTruncated struct { Name string // max:8192 From ec9d68960fe196cd53abd83243abe24bf57cb5b3 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Mon, 20 Oct 2014 21:45:26 +0100 Subject: [PATCH 07/94] Remove 64 device limit --- message.go | 4 ++-- message_xdr.go | 8 +------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/message.go b/message.go index b47145055..445ac7a75 100644 --- a/message.go +++ b/message.go @@ -137,8 +137,8 @@ func (o *ClusterConfigMessage) GetOption(key string) string { } type Folder struct { - ID string // max:64 - Devices []Device // max:64 + ID string // max:64 + Devices []Device } type Device struct { diff --git a/message_xdr.go b/message_xdr.go index c7f16a173..324125ea4 100644 --- a/message_xdr.go +++ b/message_xdr.go @@ -644,7 +644,7 @@ Folder Structure: struct Folder { string ID<64>; - Device Devices<64>; + Device Devices<>; } */ @@ -670,9 +670,6 @@ func (o Folder) encodeXDR(xw *xdr.Writer) (int, error) { return xw.Tot(), xdr.ErrElementSizeExceeded } xw.WriteString(o.ID) - if len(o.Devices) > 64 { - return xw.Tot(), xdr.ErrElementSizeExceeded - } xw.WriteUint32(uint32(len(o.Devices))) for i := range o.Devices { _, err := o.Devices[i].encodeXDR(xw) @@ -697,9 +694,6 @@ func (o *Folder) UnmarshalXDR(bs []byte) error { func (o *Folder) decodeXDR(xr *xdr.Reader) error { o.ID = xr.ReadStringMax(64) _DevicesSize := int(xr.ReadUint32()) - if _DevicesSize > 64 { - return xdr.ErrElementSizeExceeded - } o.Devices = make([]Device, _DevicesSize) for i := range o.Devices { (&o.Devices[i]).decodeXDR(xr) From 65eb528e2d459de38d71afae5b3dc5d49ce4015b Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Tue, 21 Oct 2014 08:40:05 +0200 Subject: [PATCH 08/94] Update xdr; handle marshalling errors --- message_xdr.go | 256 ++++++++++++++++++++++++++++++++--------------- protocol.go | 8 +- protocol_test.go | 3 +- 3 files changed, 184 insertions(+), 83 deletions(-) diff --git a/message_xdr.go b/message_xdr.go index 324125ea4..948e63c32 100644 --- a/message_xdr.go +++ b/message_xdr.go @@ -44,20 +44,28 @@ func (o IndexMessage) EncodeXDR(w io.Writer) (int, error) { return o.encodeXDR(xw) } -func (o IndexMessage) MarshalXDR() []byte { +func (o IndexMessage) MarshalXDR() ([]byte, error) { return o.AppendXDR(make([]byte, 0, 128)) } -func (o IndexMessage) AppendXDR(bs []byte) []byte { +func (o IndexMessage) MustMarshalXDR() []byte { + bs, err := o.MarshalXDR() + if err != nil { + panic(err) + } + return bs +} + +func (o IndexMessage) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - o.encodeXDR(xw) - return []byte(aw) + _, err := o.encodeXDR(xw) + return []byte(aw), err } func (o IndexMessage) encodeXDR(xw *xdr.Writer) (int, error) { - if len(o.Folder) > 64 { - return xw.Tot(), xdr.ErrElementSizeExceeded + if l := len(o.Folder); l > 64 { + return xw.Tot(), xdr.ElementSizeExceeded("Folder", l, 64) } xw.WriteString(o.Folder) xw.WriteUint32(uint32(len(o.Files))) @@ -142,20 +150,28 @@ func (o FileInfo) EncodeXDR(w io.Writer) (int, error) { return o.encodeXDR(xw) } -func (o FileInfo) MarshalXDR() []byte { +func (o FileInfo) MarshalXDR() ([]byte, error) { return o.AppendXDR(make([]byte, 0, 128)) } -func (o FileInfo) AppendXDR(bs []byte) []byte { +func (o FileInfo) MustMarshalXDR() []byte { + bs, err := o.MarshalXDR() + if err != nil { + panic(err) + } + return bs +} + +func (o FileInfo) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - o.encodeXDR(xw) - return []byte(aw) + _, err := o.encodeXDR(xw) + return []byte(aw), err } func (o FileInfo) encodeXDR(xw *xdr.Writer) (int, error) { - if len(o.Name) > 8192 { - return xw.Tot(), xdr.ErrElementSizeExceeded + if l := len(o.Name); l > 8192 { + return xw.Tot(), xdr.ElementSizeExceeded("Name", l, 8192) } xw.WriteString(o.Name) xw.WriteUint32(o.Flags) @@ -244,20 +260,28 @@ func (o FileInfoTruncated) EncodeXDR(w io.Writer) (int, error) { return o.encodeXDR(xw) } -func (o FileInfoTruncated) MarshalXDR() []byte { +func (o FileInfoTruncated) MarshalXDR() ([]byte, error) { return o.AppendXDR(make([]byte, 0, 128)) } -func (o FileInfoTruncated) AppendXDR(bs []byte) []byte { +func (o FileInfoTruncated) MustMarshalXDR() []byte { + bs, err := o.MarshalXDR() + if err != nil { + panic(err) + } + return bs +} + +func (o FileInfoTruncated) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - o.encodeXDR(xw) - return []byte(aw) + _, err := o.encodeXDR(xw) + return []byte(aw), err } func (o FileInfoTruncated) encodeXDR(xw *xdr.Writer) (int, error) { - if len(o.Name) > 8192 { - return xw.Tot(), xdr.ErrElementSizeExceeded + if l := len(o.Name); l > 8192 { + return xw.Tot(), xdr.ElementSizeExceeded("Name", l, 8192) } xw.WriteString(o.Name) xw.WriteUint32(o.Flags) @@ -318,21 +342,29 @@ func (o BlockInfo) EncodeXDR(w io.Writer) (int, error) { return o.encodeXDR(xw) } -func (o BlockInfo) MarshalXDR() []byte { +func (o BlockInfo) MarshalXDR() ([]byte, error) { return o.AppendXDR(make([]byte, 0, 128)) } -func (o BlockInfo) AppendXDR(bs []byte) []byte { +func (o BlockInfo) MustMarshalXDR() []byte { + bs, err := o.MarshalXDR() + if err != nil { + panic(err) + } + return bs +} + +func (o BlockInfo) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - o.encodeXDR(xw) - return []byte(aw) + _, err := o.encodeXDR(xw) + return []byte(aw), err } func (o BlockInfo) encodeXDR(xw *xdr.Writer) (int, error) { xw.WriteUint32(o.Size) - if len(o.Hash) > 64 { - return xw.Tot(), xdr.ErrElementSizeExceeded + if l := len(o.Hash); l > 64 { + return xw.Tot(), xdr.ElementSizeExceeded("Hash", l, 64) } xw.WriteBytes(o.Hash) return xw.Tot(), xw.Error() @@ -396,24 +428,32 @@ func (o RequestMessage) EncodeXDR(w io.Writer) (int, error) { return o.encodeXDR(xw) } -func (o RequestMessage) MarshalXDR() []byte { +func (o RequestMessage) MarshalXDR() ([]byte, error) { return o.AppendXDR(make([]byte, 0, 128)) } -func (o RequestMessage) AppendXDR(bs []byte) []byte { +func (o RequestMessage) MustMarshalXDR() []byte { + bs, err := o.MarshalXDR() + if err != nil { + panic(err) + } + return bs +} + +func (o RequestMessage) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - o.encodeXDR(xw) - return []byte(aw) + _, err := o.encodeXDR(xw) + return []byte(aw), err } func (o RequestMessage) encodeXDR(xw *xdr.Writer) (int, error) { - if len(o.Folder) > 64 { - return xw.Tot(), xdr.ErrElementSizeExceeded + if l := len(o.Folder); l > 64 { + return xw.Tot(), xdr.ElementSizeExceeded("Folder", l, 64) } xw.WriteString(o.Folder) - if len(o.Name) > 8192 { - return xw.Tot(), xdr.ErrElementSizeExceeded + if l := len(o.Name); l > 8192 { + return xw.Tot(), xdr.ElementSizeExceeded("Name", l, 8192) } xw.WriteString(o.Name) xw.WriteUint64(o.Offset) @@ -466,15 +506,23 @@ func (o ResponseMessage) EncodeXDR(w io.Writer) (int, error) { return o.encodeXDR(xw) } -func (o ResponseMessage) MarshalXDR() []byte { +func (o ResponseMessage) MarshalXDR() ([]byte, error) { return o.AppendXDR(make([]byte, 0, 128)) } -func (o ResponseMessage) AppendXDR(bs []byte) []byte { +func (o ResponseMessage) MustMarshalXDR() []byte { + bs, err := o.MarshalXDR() + if err != nil { + panic(err) + } + return bs +} + +func (o ResponseMessage) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - o.encodeXDR(xw) - return []byte(aw) + _, err := o.encodeXDR(xw) + return []byte(aw), err } func (o ResponseMessage) encodeXDR(xw *xdr.Writer) (int, error) { @@ -545,28 +593,36 @@ func (o ClusterConfigMessage) EncodeXDR(w io.Writer) (int, error) { return o.encodeXDR(xw) } -func (o ClusterConfigMessage) MarshalXDR() []byte { +func (o ClusterConfigMessage) MarshalXDR() ([]byte, error) { return o.AppendXDR(make([]byte, 0, 128)) } -func (o ClusterConfigMessage) AppendXDR(bs []byte) []byte { +func (o ClusterConfigMessage) MustMarshalXDR() []byte { + bs, err := o.MarshalXDR() + if err != nil { + panic(err) + } + return bs +} + +func (o ClusterConfigMessage) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - o.encodeXDR(xw) - return []byte(aw) + _, err := o.encodeXDR(xw) + return []byte(aw), err } func (o ClusterConfigMessage) encodeXDR(xw *xdr.Writer) (int, error) { - if len(o.ClientName) > 64 { - return xw.Tot(), xdr.ErrElementSizeExceeded + if l := len(o.ClientName); l > 64 { + return xw.Tot(), xdr.ElementSizeExceeded("ClientName", l, 64) } xw.WriteString(o.ClientName) - if len(o.ClientVersion) > 64 { - return xw.Tot(), xdr.ErrElementSizeExceeded + if l := len(o.ClientVersion); l > 64 { + return xw.Tot(), xdr.ElementSizeExceeded("ClientVersion", l, 64) } xw.WriteString(o.ClientVersion) - if len(o.Folders) > 64 { - return xw.Tot(), xdr.ErrElementSizeExceeded + if l := len(o.Folders); l > 64 { + return xw.Tot(), xdr.ElementSizeExceeded("Folders", l, 64) } xw.WriteUint32(uint32(len(o.Folders))) for i := range o.Folders { @@ -575,8 +631,8 @@ func (o ClusterConfigMessage) encodeXDR(xw *xdr.Writer) (int, error) { return xw.Tot(), err } } - if len(o.Options) > 64 { - return xw.Tot(), xdr.ErrElementSizeExceeded + if l := len(o.Options); l > 64 { + return xw.Tot(), xdr.ElementSizeExceeded("Options", l, 64) } xw.WriteUint32(uint32(len(o.Options))) for i := range o.Options { @@ -604,7 +660,7 @@ func (o *ClusterConfigMessage) decodeXDR(xr *xdr.Reader) error { o.ClientVersion = xr.ReadStringMax(64) _FoldersSize := int(xr.ReadUint32()) if _FoldersSize > 64 { - return xdr.ErrElementSizeExceeded + return xdr.ElementSizeExceeded("Folders", _FoldersSize, 64) } o.Folders = make([]Folder, _FoldersSize) for i := range o.Folders { @@ -612,7 +668,7 @@ func (o *ClusterConfigMessage) decodeXDR(xr *xdr.Reader) error { } _OptionsSize := int(xr.ReadUint32()) if _OptionsSize > 64 { - return xdr.ErrElementSizeExceeded + return xdr.ElementSizeExceeded("Options", _OptionsSize, 64) } o.Options = make([]Option, _OptionsSize) for i := range o.Options { @@ -654,20 +710,28 @@ func (o Folder) EncodeXDR(w io.Writer) (int, error) { return o.encodeXDR(xw) } -func (o Folder) MarshalXDR() []byte { +func (o Folder) MarshalXDR() ([]byte, error) { return o.AppendXDR(make([]byte, 0, 128)) } -func (o Folder) AppendXDR(bs []byte) []byte { +func (o Folder) MustMarshalXDR() []byte { + bs, err := o.MarshalXDR() + if err != nil { + panic(err) + } + return bs +} + +func (o Folder) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - o.encodeXDR(xw) - return []byte(aw) + _, err := o.encodeXDR(xw) + return []byte(aw), err } func (o Folder) encodeXDR(xw *xdr.Writer) (int, error) { - if len(o.ID) > 64 { - return xw.Tot(), xdr.ErrElementSizeExceeded + if l := len(o.ID); l > 64 { + return xw.Tot(), xdr.ElementSizeExceeded("ID", l, 64) } xw.WriteString(o.ID) xw.WriteUint32(uint32(len(o.Devices))) @@ -735,20 +799,28 @@ func (o Device) EncodeXDR(w io.Writer) (int, error) { return o.encodeXDR(xw) } -func (o Device) MarshalXDR() []byte { +func (o Device) MarshalXDR() ([]byte, error) { return o.AppendXDR(make([]byte, 0, 128)) } -func (o Device) AppendXDR(bs []byte) []byte { +func (o Device) MustMarshalXDR() []byte { + bs, err := o.MarshalXDR() + if err != nil { + panic(err) + } + return bs +} + +func (o Device) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - o.encodeXDR(xw) - return []byte(aw) + _, err := o.encodeXDR(xw) + return []byte(aw), err } func (o Device) encodeXDR(xw *xdr.Writer) (int, error) { - if len(o.ID) > 32 { - return xw.Tot(), xdr.ErrElementSizeExceeded + if l := len(o.ID); l > 32 { + return xw.Tot(), xdr.ElementSizeExceeded("ID", l, 32) } xw.WriteBytes(o.ID) xw.WriteUint32(o.Flags) @@ -807,24 +879,32 @@ func (o Option) EncodeXDR(w io.Writer) (int, error) { return o.encodeXDR(xw) } -func (o Option) MarshalXDR() []byte { +func (o Option) MarshalXDR() ([]byte, error) { return o.AppendXDR(make([]byte, 0, 128)) } -func (o Option) AppendXDR(bs []byte) []byte { +func (o Option) MustMarshalXDR() []byte { + bs, err := o.MarshalXDR() + if err != nil { + panic(err) + } + return bs +} + +func (o Option) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - o.encodeXDR(xw) - return []byte(aw) + _, err := o.encodeXDR(xw) + return []byte(aw), err } func (o Option) encodeXDR(xw *xdr.Writer) (int, error) { - if len(o.Key) > 64 { - return xw.Tot(), xdr.ErrElementSizeExceeded + if l := len(o.Key); l > 64 { + return xw.Tot(), xdr.ElementSizeExceeded("Key", l, 64) } xw.WriteString(o.Key) - if len(o.Value) > 1024 { - return xw.Tot(), xdr.ErrElementSizeExceeded + if l := len(o.Value); l > 1024 { + return xw.Tot(), xdr.ElementSizeExceeded("Value", l, 1024) } xw.WriteString(o.Value) return xw.Tot(), xw.Error() @@ -873,20 +953,28 @@ func (o CloseMessage) EncodeXDR(w io.Writer) (int, error) { return o.encodeXDR(xw) } -func (o CloseMessage) MarshalXDR() []byte { +func (o CloseMessage) MarshalXDR() ([]byte, error) { return o.AppendXDR(make([]byte, 0, 128)) } -func (o CloseMessage) AppendXDR(bs []byte) []byte { +func (o CloseMessage) MustMarshalXDR() []byte { + bs, err := o.MarshalXDR() + if err != nil { + panic(err) + } + return bs +} + +func (o CloseMessage) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - o.encodeXDR(xw) - return []byte(aw) + _, err := o.encodeXDR(xw) + return []byte(aw), err } func (o CloseMessage) encodeXDR(xw *xdr.Writer) (int, error) { - if len(o.Reason) > 1024 { - return xw.Tot(), xdr.ErrElementSizeExceeded + if l := len(o.Reason); l > 1024 { + return xw.Tot(), xdr.ElementSizeExceeded("Reason", l, 1024) } xw.WriteString(o.Reason) return xw.Tot(), xw.Error() @@ -927,15 +1015,23 @@ func (o EmptyMessage) EncodeXDR(w io.Writer) (int, error) { return o.encodeXDR(xw) } -func (o EmptyMessage) MarshalXDR() []byte { +func (o EmptyMessage) MarshalXDR() ([]byte, error) { return o.AppendXDR(make([]byte, 0, 128)) } -func (o EmptyMessage) AppendXDR(bs []byte) []byte { +func (o EmptyMessage) MustMarshalXDR() []byte { + bs, err := o.MarshalXDR() + if err != nil { + panic(err) + } + return bs +} + +func (o EmptyMessage) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - o.encodeXDR(xw) - return []byte(aw) + _, err := o.encodeXDR(xw) + return []byte(aw), err } func (o EmptyMessage) encodeXDR(xw *xdr.Writer) (int, error) { diff --git a/protocol.go b/protocol.go index 19cdfbb16..08ef226f0 100644 --- a/protocol.go +++ b/protocol.go @@ -126,7 +126,7 @@ type hdrMsg struct { } type encodable interface { - AppendXDR([]byte) []byte + AppendXDR([]byte) ([]byte, error) } const ( @@ -483,7 +483,11 @@ func (c *rawConnection) writerLoop() { case hm := <-c.outbox: if hm.msg != nil { // Uncompressed message in uncBuf - uncBuf = hm.msg.AppendXDR(uncBuf[:0]) + uncBuf, err = hm.msg.AppendXDR(uncBuf[:0]) + if err != nil { + c.close(err) + return + } if len(uncBuf) >= c.compressionThreshold { // Use compression for large messages diff --git a/protocol_test.go b/protocol_test.go index cc6f9472a..a7bb1416c 100644 --- a/protocol_test.go +++ b/protocol_test.go @@ -25,6 +25,7 @@ import ( "io/ioutil" "os" "reflect" + "strings" "testing" "testing/quick" @@ -369,7 +370,7 @@ func testMarshal(t *testing.T, prefix string, m1, m2 message) bool { } _, err := m1.EncodeXDR(&buf) - if err == xdr.ErrElementSizeExceeded { + if err != nil && strings.Contains(err.Error(), "exceeds size") { return true } if err != nil { From 28610a9a426fac3ae88b812897b1c1a540112071 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Sun, 26 Oct 2014 13:15:14 +0100 Subject: [PATCH 09/94] Break out logger as a reusable component --- debug.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug.go b/debug.go index 930aaf344..c46f4a984 100644 --- a/debug.go +++ b/debug.go @@ -19,7 +19,7 @@ import ( "os" "strings" - "github.com/syncthing/syncthing/internal/logger" + "github.com/calmh/logger" ) var ( From ad29093ac12cf822f1b55f8834b820dcd015a9e3 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Sun, 16 Nov 2014 21:13:20 +0100 Subject: [PATCH 10/94] Use more inclusive copyright header --- common_test.go | 2 +- counting.go | 2 +- debug.go | 2 +- deviceid.go | 2 +- deviceid_test.go | 2 +- doc.go | 2 +- header.go | 2 +- message.go | 2 +- nativemodel_darwin.go | 2 +- nativemodel_unix.go | 2 +- nativemodel_windows.go | 2 +- protocol.go | 2 +- protocol_test.go | 2 +- wireformat.go | 2 +- 14 files changed, 14 insertions(+), 14 deletions(-) diff --git a/common_test.go b/common_test.go index 29b910bab..e38da00a8 100644 --- a/common_test.go +++ b/common_test.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// Copyright (C) 2014 The Syncthing Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU General Public License as published by the Free diff --git a/counting.go b/counting.go index 4f95c9f9a..ac4aeaa9a 100644 --- a/counting.go +++ b/counting.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// Copyright (C) 2014 The Syncthing Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU General Public License as published by the Free diff --git a/debug.go b/debug.go index c46f4a984..4664b0e23 100644 --- a/debug.go +++ b/debug.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// Copyright (C) 2014 The Syncthing Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU General Public License as published by the Free diff --git a/deviceid.go b/deviceid.go index 4b034f3f1..93753d7fa 100644 --- a/deviceid.go +++ b/deviceid.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// Copyright (C) 2014 The Syncthing Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU General Public License as published by the Free diff --git a/deviceid_test.go b/deviceid_test.go index 14e2dfa34..da398c6fb 100644 --- a/deviceid_test.go +++ b/deviceid_test.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// Copyright (C) 2014 The Syncthing Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU General Public License as published by the Free diff --git a/doc.go b/doc.go index 95b5e102f..4f73e6c86 100644 --- a/doc.go +++ b/doc.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// Copyright (C) 2014 The Syncthing Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU General Public License as published by the Free diff --git a/header.go b/header.go index 96ccff66f..8da092e5c 100644 --- a/header.go +++ b/header.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// Copyright (C) 2014 The Syncthing Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU General Public License as published by the Free diff --git a/message.go b/message.go index 445ac7a75..eff837fe1 100644 --- a/message.go +++ b/message.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// Copyright (C) 2014 The Syncthing Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU General Public License as published by the Free diff --git a/nativemodel_darwin.go b/nativemodel_darwin.go index 7fb5c964e..8b2b842a4 100644 --- a/nativemodel_darwin.go +++ b/nativemodel_darwin.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// Copyright (C) 2014 The Syncthing Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU General Public License as published by the Free diff --git a/nativemodel_unix.go b/nativemodel_unix.go index 61ad9c438..bf6499e63 100644 --- a/nativemodel_unix.go +++ b/nativemodel_unix.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// Copyright (C) 2014 The Syncthing Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU General Public License as published by the Free diff --git a/nativemodel_windows.go b/nativemodel_windows.go index 57d3f3446..d2c079bdc 100644 --- a/nativemodel_windows.go +++ b/nativemodel_windows.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// Copyright (C) 2014 The Syncthing Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU General Public License as published by the Free diff --git a/protocol.go b/protocol.go index 08ef226f0..ae7f480b3 100644 --- a/protocol.go +++ b/protocol.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// Copyright (C) 2014 The Syncthing Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU General Public License as published by the Free diff --git a/protocol_test.go b/protocol_test.go index a7bb1416c..9da9422c1 100644 --- a/protocol_test.go +++ b/protocol_test.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// Copyright (C) 2014 The Syncthing Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU General Public License as published by the Free diff --git a/wireformat.go b/wireformat.go index f06b9454e..b7f4e367d 100644 --- a/wireformat.go +++ b/wireformat.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file). +// Copyright (C) 2014 The Syncthing Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU General Public License as published by the Free From e0da2764c930ecde44dcf2d66283c523830da0ae Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Tue, 4 Nov 2014 23:22:15 +0000 Subject: [PATCH 11/94] Code smell --- message.go | 28 +++++++++++++++++++++------- protocol.go | 16 ---------------- 2 files changed, 21 insertions(+), 23 deletions(-) diff --git a/message.go b/message.go index eff837fe1..5d35f17cb 100644 --- a/message.go +++ b/message.go @@ -37,7 +37,7 @@ func (f FileInfo) String() string { } func (f FileInfo) Size() (bytes int64) { - if IsDeleted(f.Flags) || IsDirectory(f.Flags) { + if f.IsDeleted() || f.IsDirectory() { return 128 } for _, b := range f.Blocks { @@ -47,15 +47,19 @@ func (f FileInfo) Size() (bytes int64) { } func (f FileInfo) IsDeleted() bool { - return IsDeleted(f.Flags) + return f.Flags&FlagDeleted != 0 } func (f FileInfo) IsInvalid() bool { - return IsInvalid(f.Flags) + return f.Flags&FlagInvalid != 0 } func (f FileInfo) IsDirectory() bool { - return IsDirectory(f.Flags) + return f.Flags&FlagDirectory != 0 +} + +func (f FileInfo) HasPermissionBits() bool { + return f.Flags&FlagNoPermBits == 0 } // Used for unmarshalling a FileInfo structure but skipping the actual block list @@ -75,7 +79,7 @@ func (f FileInfoTruncated) String() string { // Returns a statistical guess on the size, not the exact figure func (f FileInfoTruncated) Size() int64 { - if IsDeleted(f.Flags) || IsDirectory(f.Flags) { + if f.IsDeleted() || f.IsDirectory() { return 128 } if f.NumBlocks < 2 { @@ -86,17 +90,27 @@ func (f FileInfoTruncated) Size() int64 { } func (f FileInfoTruncated) IsDeleted() bool { - return IsDeleted(f.Flags) + return f.Flags&FlagDeleted != 0 } func (f FileInfoTruncated) IsInvalid() bool { - return IsInvalid(f.Flags) + return f.Flags&FlagInvalid != 0 +} + +func (f FileInfoTruncated) IsDirectory() bool { + return f.Flags&FlagDirectory != 0 +} + +func (f FileInfoTruncated) HasPermissionBits() bool { + return f.Flags&FlagNoPermBits == 0 } type FileIntf interface { Size() int64 IsDeleted() bool IsInvalid() bool + IsDirectory() bool + HasPermissionBits() bool } type BlockInfo struct { diff --git a/protocol.go b/protocol.go index ae7f480b3..7de53eab9 100644 --- a/protocol.go +++ b/protocol.go @@ -637,19 +637,3 @@ func (c *rawConnection) Statistics() Statistics { OutBytesTotal: c.cw.Tot(), } } - -func IsDeleted(bits uint32) bool { - return bits&FlagDeleted != 0 -} - -func IsInvalid(bits uint32) bool { - return bits&FlagInvalid != 0 -} - -func IsDirectory(bits uint32) bool { - return bits&FlagDirectory != 0 -} - -func HasPermissionBits(bits uint32) bool { - return bits&FlagNoPermBits == 0 -} From ddc56c8a0ddf0afa5cd9b8798f39cf1bacd6b821 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Fri, 7 Nov 2014 23:06:04 +0000 Subject: [PATCH 12/94] Add symlink support at the protocol level --- message.go | 9 +++++++++ protocol.go | 12 ++++++++---- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/message.go b/message.go index 5d35f17cb..355050cab 100644 --- a/message.go +++ b/message.go @@ -58,6 +58,10 @@ func (f FileInfo) IsDirectory() bool { return f.Flags&FlagDirectory != 0 } +func (f FileInfo) IsSymlink() bool { + return f.Flags&FlagSymlink != 0 +} + func (f FileInfo) HasPermissionBits() bool { return f.Flags&FlagNoPermBits == 0 } @@ -101,6 +105,10 @@ func (f FileInfoTruncated) IsDirectory() bool { return f.Flags&FlagDirectory != 0 } +func (f FileInfoTruncated) IsSymlink() bool { + return f.Flags&FlagSymlink != 0 +} + func (f FileInfoTruncated) HasPermissionBits() bool { return f.Flags&FlagNoPermBits == 0 } @@ -110,6 +118,7 @@ type FileIntf interface { IsDeleted() bool IsInvalid() bool IsDirectory() bool + IsSymlink() bool HasPermissionBits() bool } diff --git a/protocol.go b/protocol.go index 7de53eab9..fb51a1c01 100644 --- a/protocol.go +++ b/protocol.go @@ -49,10 +49,14 @@ const ( ) const ( - FlagDeleted uint32 = 1 << 12 - FlagInvalid = 1 << 13 - FlagDirectory = 1 << 14 - FlagNoPermBits = 1 << 15 + FlagDeleted uint32 = 1 << 12 + FlagInvalid = 1 << 13 + FlagDirectory = 1 << 14 + FlagNoPermBits = 1 << 15 + FlagSymlink = 1 << 16 + FlagSymlinkMissingTarget = 1 << 17 + + SymlinkTypeMask = FlagDirectory | FlagSymlinkMissingTarget ) const ( From 3af96e50bd09a36167202e0f5e9fe8d310c70a3b Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Sun, 23 Nov 2014 00:52:48 +0000 Subject: [PATCH 13/94] Use custom structure for /need calls (fixes #1001) Also, remove trimming by number of blocks as this no longer affects the size of the response. --- message.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/message.go b/message.go index 355050cab..9a96f80c5 100644 --- a/message.go +++ b/message.go @@ -81,16 +81,19 @@ func (f FileInfoTruncated) String() string { f.Name, f.Flags, f.Modified, f.Version, f.Size(), f.NumBlocks) } +func BlocksToSize(num uint32) int64 { + if num < 2 { + return BlockSize / 2 + } + return int64(num-1)*BlockSize + BlockSize/2 +} + // Returns a statistical guess on the size, not the exact figure func (f FileInfoTruncated) Size() int64 { if f.IsDeleted() || f.IsDirectory() { return 128 } - if f.NumBlocks < 2 { - return BlockSize / 2 - } else { - return int64(f.NumBlocks-1)*BlockSize + BlockSize/2 - } + return BlocksToSize(f.NumBlocks) } func (f FileInfoTruncated) IsDeleted() bool { From dc71ec734d7f4ddbc83043920134075bf8b2d3f0 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Sun, 30 Nov 2014 00:17:00 +0100 Subject: [PATCH 14/94] Dependency update, new golang.org/x package names --- nativemodel_darwin.go | 2 +- wireformat.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nativemodel_darwin.go b/nativemodel_darwin.go index 8b2b842a4..ba30d1ae1 100644 --- a/nativemodel_darwin.go +++ b/nativemodel_darwin.go @@ -19,7 +19,7 @@ package protocol // Darwin uses NFD normalization -import "code.google.com/p/go.text/unicode/norm" +import "golang.org/x/text/unicode/norm" type nativeModel struct { next Model diff --git a/wireformat.go b/wireformat.go index b7f4e367d..84da71423 100644 --- a/wireformat.go +++ b/wireformat.go @@ -18,7 +18,7 @@ package protocol import ( "path/filepath" - "code.google.com/p/go.text/unicode/norm" + "golang.org/x/text/unicode/norm" ) type wireFormatConnection struct { From 190c61ba2f07b4b698f8281c3bef9118ca785d52 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Sat, 6 Dec 2014 14:23:10 +0100 Subject: [PATCH 15/94] Use Go 1.4 'generate' to create XDR codec --- message.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/message.go b/message.go index 9a96f80c5..8cc191d8c 100644 --- a/message.go +++ b/message.go @@ -13,6 +13,9 @@ // You should have received a copy of the GNU General Public License along // with this program. If not, see . +//go:generate -command genxdr go run ../../Godeps/_workspace/src/github.com/calmh/xdr/cmd/genxdr/main.go +//go:generate genxdr -o message_xdr.go message.go + package protocol import "fmt" From 09b534b8a32392776c7eed5c41f3f0cc1356b56e Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Mon, 1 Dec 2014 19:23:06 +0000 Subject: [PATCH 16/94] Add job queue (fixes #629) Request to terminate currently ongoing downloads and jump to the bumped file incoming in 3, 2, 1. Also, has a slightly strange effect where we pop a job off the queue, but the copyChannel is still busy and blocks, though it gets moved to the progress slice in the jobqueue, and looks like it's in progress which it isn't as it's waiting to be picked up from the copyChan. As a result, the progress emitter doesn't register on the task, and hence the file doesn't have a progress bar, but cannot be replaced by a bump. I guess I can fix progress bar issue by moving the progressEmiter.Register just before passing the file to the copyChan, but then we are back to the initial problem of a file with a progress bar, but no progress happening as it's stuck on write to copyChan I checked if there is a way to check for channel writeability (before popping) but got struck by lightning just for bringing the idea up in #go-nuts. My ideal scenario would be to check if copyChan is writeable, pop job from the queue and shove it down handleFile. This way jobs would stay in the queue while they cannot be handled, meaning that the `Bump` could bring your file up higher. --- message.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/message.go b/message.go index 8cc191d8c..ae04a9da8 100644 --- a/message.go +++ b/message.go @@ -69,6 +69,17 @@ func (f FileInfo) HasPermissionBits() bool { return f.Flags&FlagNoPermBits == 0 } +func (f FileInfo) ToTruncated() FileInfoTruncated { + return FileInfoTruncated{ + Name: f.Name, + Flags: f.Flags, + Modified: f.Modified, + Version: f.Version, + LocalVersion: f.LocalVersion, + NumBlocks: uint32(len(f.Blocks)), + } +} + // Used for unmarshalling a FileInfo structure but skipping the actual block list type FileInfoTruncated struct { Name string // max:8192 From a3ea9427d1b9167490d910add0af6ed106c882a7 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Thu, 8 Jan 2015 14:21:58 +0100 Subject: [PATCH 17/94] Ensure backwards compatibility before modifying protocol This change makes sure that things work smoothly when "we" are a newer version than our peer and have more fields in our messages than they do. Missing fields will be left at zero/nil. (The other side will ignore our extra fields, for the same effect.) --- protocol.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/protocol.go b/protocol.go index fb51a1c01..a6a4b8b4f 100644 --- a/protocol.go +++ b/protocol.go @@ -133,6 +133,10 @@ type encodable interface { AppendXDR([]byte) ([]byte, error) } +type isEofer interface { + IsEOF() bool +} + const ( pingTimeout = 30 * time.Second pingIdleTime = 60 * time.Second @@ -376,20 +380,36 @@ func (c *rawConnection) readMessage() (hdr header, msg encodable, err error) { } } + // We check each returned error for the XDRError.IsEOF() method. + // IsEOF()==true here means that the message contained fewer fields than + // expected. It does not signify an EOF on the socket, because we've + // successfully read a size value and that many bytes already. New fields + // we expected but the other peer didn't send should be interpreted as + // zero/nil, and if that's not valid we'll verify it somewhere else. + switch hdr.msgType { case messageTypeIndex, messageTypeIndexUpdate: var idx IndexMessage err = idx.UnmarshalXDR(msgBuf) + if xdrErr, ok := err.(isEofer); ok && xdrErr.IsEOF() { + err = nil + } msg = idx case messageTypeRequest: var req RequestMessage err = req.UnmarshalXDR(msgBuf) + if xdrErr, ok := err.(isEofer); ok && xdrErr.IsEOF() { + err = nil + } msg = req case messageTypeResponse: var resp ResponseMessage err = resp.UnmarshalXDR(msgBuf) + if xdrErr, ok := err.(isEofer); ok && xdrErr.IsEOF() { + err = nil + } msg = resp case messageTypePing, messageTypePong: @@ -398,11 +418,17 @@ func (c *rawConnection) readMessage() (hdr header, msg encodable, err error) { case messageTypeClusterConfig: var cc ClusterConfigMessage err = cc.UnmarshalXDR(msgBuf) + if xdrErr, ok := err.(isEofer); ok && xdrErr.IsEOF() { + err = nil + } msg = cc case messageTypeClose: var cm CloseMessage err = cm.UnmarshalXDR(msgBuf) + if xdrErr, ok := err.(isEofer); ok && xdrErr.IsEOF() { + err = nil + } msg = cm default: From c111ed4b2046d307c2767cff61dab77aba6fca0e Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Thu, 8 Jan 2015 10:28:39 +0100 Subject: [PATCH 18/94] Add fields for future extensibility This adds a number of fields to the end of existing messages. This is a backwards compatible change. --- message.go | 21 ++++++++----- message_xdr.go | 82 ++++++++++++++++++++++++++++++++++++++++++++++++++ protocol.go | 21 ++++++++++--- 3 files changed, 113 insertions(+), 11 deletions(-) diff --git a/message.go b/message.go index ae04a9da8..e0c66b9dc 100644 --- a/message.go +++ b/message.go @@ -21,8 +21,10 @@ package protocol import "fmt" type IndexMessage struct { - Folder string // max:64 - Files []FileInfo + Folder string // max:64 + Files []FileInfo + Flags uint32 + Options []Option // max:64 } type FileInfo struct { @@ -150,14 +152,18 @@ func (b BlockInfo) String() string { } type RequestMessage struct { - Folder string // max:64 - Name string // max:8192 - Offset uint64 - Size uint32 + Folder string // max:64 + Name string // max:8192 + Offset uint64 + Size uint32 + Hash []byte // max:64 + Flags uint32 + Options []Option // max:64 } type ResponseMessage struct { - Data []byte + Data []byte + Error uint32 } type ClusterConfigMessage struct { @@ -194,6 +200,7 @@ type Option struct { type CloseMessage struct { Reason string // max:1024 + Code uint32 } type EmptyMessage struct{} diff --git a/message_xdr.go b/message_xdr.go index 948e63c32..da13111c2 100644 --- a/message_xdr.go +++ b/message_xdr.go @@ -30,11 +30,21 @@ IndexMessage Structure: \ Zero or more FileInfo Structures \ / / +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Flags | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Number of Options | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Zero or more Option Structures \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ struct IndexMessage { string Folder<64>; FileInfo Files<>; + unsigned int Flags; + Option Options<64>; } */ @@ -75,6 +85,17 @@ func (o IndexMessage) encodeXDR(xw *xdr.Writer) (int, error) { return xw.Tot(), err } } + xw.WriteUint32(o.Flags) + if l := len(o.Options); l > 64 { + return xw.Tot(), xdr.ElementSizeExceeded("Options", l, 64) + } + xw.WriteUint32(uint32(len(o.Options))) + for i := range o.Options { + _, err := o.Options[i].encodeXDR(xw) + if err != nil { + return xw.Tot(), err + } + } return xw.Tot(), xw.Error() } @@ -96,6 +117,15 @@ func (o *IndexMessage) decodeXDR(xr *xdr.Reader) error { for i := range o.Files { (&o.Files[i]).decodeXDR(xr) } + o.Flags = xr.ReadUint32() + _OptionsSize := int(xr.ReadUint32()) + if _OptionsSize > 64 { + return xdr.ElementSizeExceeded("Options", _OptionsSize, 64) + } + o.Options = make([]Option, _OptionsSize) + for i := range o.Options { + (&o.Options[i]).decodeXDR(xr) + } return xr.Error() } @@ -412,6 +442,20 @@ RequestMessage Structure: +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ | Size | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Length of Hash | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Hash (variable length) \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Flags | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Number of Options | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Zero or more Option Structures \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ struct RequestMessage { @@ -419,6 +463,9 @@ struct RequestMessage { string Name<8192>; unsigned hyper Offset; unsigned int Size; + opaque Hash<64>; + unsigned int Flags; + Option Options<64>; } */ @@ -458,6 +505,21 @@ func (o RequestMessage) encodeXDR(xw *xdr.Writer) (int, error) { xw.WriteString(o.Name) xw.WriteUint64(o.Offset) xw.WriteUint32(o.Size) + if l := len(o.Hash); l > 64 { + return xw.Tot(), xdr.ElementSizeExceeded("Hash", l, 64) + } + xw.WriteBytes(o.Hash) + xw.WriteUint32(o.Flags) + if l := len(o.Options); l > 64 { + return xw.Tot(), xdr.ElementSizeExceeded("Options", l, 64) + } + xw.WriteUint32(uint32(len(o.Options))) + for i := range o.Options { + _, err := o.Options[i].encodeXDR(xw) + if err != nil { + return xw.Tot(), err + } + } return xw.Tot(), xw.Error() } @@ -477,6 +539,16 @@ func (o *RequestMessage) decodeXDR(xr *xdr.Reader) error { o.Name = xr.ReadStringMax(8192) o.Offset = xr.ReadUint64() o.Size = xr.ReadUint32() + o.Hash = xr.ReadBytesMax(64) + o.Flags = xr.ReadUint32() + _OptionsSize := int(xr.ReadUint32()) + if _OptionsSize > 64 { + return xdr.ElementSizeExceeded("Options", _OptionsSize, 64) + } + o.Options = make([]Option, _OptionsSize) + for i := range o.Options { + (&o.Options[i]).decodeXDR(xr) + } return xr.Error() } @@ -493,10 +565,13 @@ ResponseMessage Structure: \ Data (variable length) \ / / +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Error | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ struct ResponseMessage { opaque Data<>; + unsigned int Error; } */ @@ -527,6 +602,7 @@ func (o ResponseMessage) AppendXDR(bs []byte) ([]byte, error) { func (o ResponseMessage) encodeXDR(xw *xdr.Writer) (int, error) { xw.WriteBytes(o.Data) + xw.WriteUint32(o.Error) return xw.Tot(), xw.Error() } @@ -543,6 +619,7 @@ func (o *ResponseMessage) UnmarshalXDR(bs []byte) error { func (o *ResponseMessage) decodeXDR(xr *xdr.Reader) error { o.Data = xr.ReadBytes() + o.Error = xr.ReadUint32() return xr.Error() } @@ -940,10 +1017,13 @@ CloseMessage Structure: \ Reason (variable length) \ / / +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Code | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ struct CloseMessage { string Reason<1024>; + unsigned int Code; } */ @@ -977,6 +1057,7 @@ func (o CloseMessage) encodeXDR(xw *xdr.Writer) (int, error) { return xw.Tot(), xdr.ElementSizeExceeded("Reason", l, 1024) } xw.WriteString(o.Reason) + xw.WriteUint32(o.Code) return xw.Tot(), xw.Error() } @@ -993,6 +1074,7 @@ func (o *CloseMessage) UnmarshalXDR(bs []byte) error { func (o *CloseMessage) decodeXDR(xr *xdr.Reader) error { o.Reason = xr.ReadStringMax(1024) + o.Code = xr.ReadUint32() return xr.Error() } diff --git a/protocol.go b/protocol.go index a6a4b8b4f..e65d5e72e 100644 --- a/protocol.go +++ b/protocol.go @@ -187,7 +187,10 @@ func (c *rawConnection) Index(folder string, idx []FileInfo) error { default: } c.idxMut.Lock() - c.send(-1, messageTypeIndex, IndexMessage{folder, idx}) + c.send(-1, messageTypeIndex, IndexMessage{ + Folder: folder, + Files: idx, + }) c.idxMut.Unlock() return nil } @@ -200,7 +203,10 @@ func (c *rawConnection) IndexUpdate(folder string, idx []FileInfo) error { default: } c.idxMut.Lock() - c.send(-1, messageTypeIndexUpdate, IndexMessage{folder, idx}) + c.send(-1, messageTypeIndexUpdate, IndexMessage{ + Folder: folder, + Files: idx, + }) c.idxMut.Unlock() return nil } @@ -222,7 +228,12 @@ func (c *rawConnection) Request(folder string, name string, offset int64, size i c.awaiting[id] = rc c.awaitingMut.Unlock() - ok := c.send(id, messageTypeRequest, RequestMessage{folder, name, uint64(offset), uint32(size)}) + ok := c.send(id, messageTypeRequest, RequestMessage{ + Folder: folder, + Name: name, + Offset: uint64(offset), + Size: uint32(size), + }) if !ok { return nil, ErrClosed } @@ -455,7 +466,9 @@ func (c *rawConnection) handleIndexUpdate(im IndexMessage) { func (c *rawConnection) handleRequest(msgID int, req RequestMessage) { data, _ := c.receiver.Request(c.id, req.Folder, req.Name, int64(req.Offset), int(req.Size)) - c.send(msgID, messageTypeResponse, ResponseMessage{data}) + c.send(msgID, messageTypeResponse, ResponseMessage{ + Data: data, + }) } func (c *rawConnection) handleResponse(msgID int, resp ResponseMessage) { From 8c32955da13ca35484069ac2587d9ddee56f3b90 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Thu, 8 Jan 2015 22:11:10 +0100 Subject: [PATCH 19/94] Actually close connection based on unknown protocol version --- protocol.go | 5 +++++ protocol_test.go | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/protocol.go b/protocol.go index e65d5e72e..a55256799 100644 --- a/protocol.go +++ b/protocol.go @@ -356,6 +356,11 @@ func (c *rawConnection) readMessage() (hdr header, msg encodable, err error) { l.Debugf("read header %v (msglen=%d)", hdr, msglen) } + if hdr.version != 0 { + err = fmt.Errorf("unknown protocol version 0x%x", hdr.version) + return + } + if cap(c.rdbuf0) < msglen { c.rdbuf0 = make([]byte, msglen) } else { diff --git a/protocol_test.go b/protocol_test.go index 9da9422c1..75e0086ee 100644 --- a/protocol_test.go +++ b/protocol_test.go @@ -189,7 +189,7 @@ func TestVersionErr(t *testing.T) { msgID: 0, msgType: 0, })) - w.WriteUint32(0) + w.WriteUint32(0) // Avoids reader closing due to EOF if !m1.isClosed() { t.Error("Connection should close due to unknown version") @@ -212,7 +212,7 @@ func TestTypeErr(t *testing.T) { msgID: 0, msgType: 42, })) - w.WriteUint32(0) + w.WriteUint32(0) // Avoids reader closing due to EOF if !m1.isClosed() { t.Error("Connection should close due to unknown message type") From 36708a5067f340cdb7383c6ea0c80c4defcdbafd Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Fri, 9 Jan 2015 08:18:42 +0100 Subject: [PATCH 20/94] Move FileIntf to files package, expose Iterator type This is where FileIntf is used, so it should be defined here (it's not a protocol thing, really). --- message.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/message.go b/message.go index e0c66b9dc..3129f3de0 100644 --- a/message.go +++ b/message.go @@ -132,15 +132,6 @@ func (f FileInfoTruncated) HasPermissionBits() bool { return f.Flags&FlagNoPermBits == 0 } -type FileIntf interface { - Size() int64 - IsDeleted() bool - IsInvalid() bool - IsDirectory() bool - IsSymlink() bool - HasPermissionBits() bool -} - type BlockInfo struct { Offset int64 // noencode (cache only) Size uint32 From d9ed8e125e3b9678830047738c868eeb77e56488 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Fri, 9 Jan 2015 08:19:32 +0100 Subject: [PATCH 21/94] Move FileInfoTruncated to files package This is where it's used, and it clarifies that it's never used over the wire. --- message.go | 61 ------------------------------ message_xdr.go | 100 ------------------------------------------------- 2 files changed, 161 deletions(-) diff --git a/message.go b/message.go index 3129f3de0..0d8de8070 100644 --- a/message.go +++ b/message.go @@ -71,67 +71,6 @@ func (f FileInfo) HasPermissionBits() bool { return f.Flags&FlagNoPermBits == 0 } -func (f FileInfo) ToTruncated() FileInfoTruncated { - return FileInfoTruncated{ - Name: f.Name, - Flags: f.Flags, - Modified: f.Modified, - Version: f.Version, - LocalVersion: f.LocalVersion, - NumBlocks: uint32(len(f.Blocks)), - } -} - -// Used for unmarshalling a FileInfo structure but skipping the actual block list -type FileInfoTruncated struct { - Name string // max:8192 - Flags uint32 - Modified int64 - Version uint64 - LocalVersion uint64 - NumBlocks uint32 -} - -func (f FileInfoTruncated) String() string { - return fmt.Sprintf("File{Name:%q, Flags:0%o, Modified:%d, Version:%d, Size:%d, NumBlocks:%d}", - f.Name, f.Flags, f.Modified, f.Version, f.Size(), f.NumBlocks) -} - -func BlocksToSize(num uint32) int64 { - if num < 2 { - return BlockSize / 2 - } - return int64(num-1)*BlockSize + BlockSize/2 -} - -// Returns a statistical guess on the size, not the exact figure -func (f FileInfoTruncated) Size() int64 { - if f.IsDeleted() || f.IsDirectory() { - return 128 - } - return BlocksToSize(f.NumBlocks) -} - -func (f FileInfoTruncated) IsDeleted() bool { - return f.Flags&FlagDeleted != 0 -} - -func (f FileInfoTruncated) IsInvalid() bool { - return f.Flags&FlagInvalid != 0 -} - -func (f FileInfoTruncated) IsDirectory() bool { - return f.Flags&FlagDirectory != 0 -} - -func (f FileInfoTruncated) IsSymlink() bool { - return f.Flags&FlagSymlink != 0 -} - -func (f FileInfoTruncated) HasPermissionBits() bool { - return f.Flags&FlagNoPermBits == 0 -} - type BlockInfo struct { Offset int64 // noencode (cache only) Size uint32 diff --git a/message_xdr.go b/message_xdr.go index da13111c2..fa7b5802b 100644 --- a/message_xdr.go +++ b/message_xdr.go @@ -245,106 +245,6 @@ func (o *FileInfo) decodeXDR(xr *xdr.Reader) error { /* -FileInfoTruncated Structure: - - 0 1 2 3 - 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Length of Name | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -/ / -\ Name (variable length) \ -/ / -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Flags | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| | -+ Modified (64 bits) + -| | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| | -+ Version (64 bits) + -| | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| | -+ Local Version (64 bits) + -| | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Num Blocks | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - - -struct FileInfoTruncated { - string Name<8192>; - unsigned int Flags; - hyper Modified; - unsigned hyper Version; - unsigned hyper LocalVersion; - unsigned int NumBlocks; -} - -*/ - -func (o FileInfoTruncated) EncodeXDR(w io.Writer) (int, error) { - var xw = xdr.NewWriter(w) - return o.encodeXDR(xw) -} - -func (o FileInfoTruncated) MarshalXDR() ([]byte, error) { - return o.AppendXDR(make([]byte, 0, 128)) -} - -func (o FileInfoTruncated) MustMarshalXDR() []byte { - bs, err := o.MarshalXDR() - if err != nil { - panic(err) - } - return bs -} - -func (o FileInfoTruncated) AppendXDR(bs []byte) ([]byte, error) { - var aw = xdr.AppendWriter(bs) - var xw = xdr.NewWriter(&aw) - _, err := o.encodeXDR(xw) - return []byte(aw), err -} - -func (o FileInfoTruncated) encodeXDR(xw *xdr.Writer) (int, error) { - if l := len(o.Name); l > 8192 { - return xw.Tot(), xdr.ElementSizeExceeded("Name", l, 8192) - } - xw.WriteString(o.Name) - xw.WriteUint32(o.Flags) - xw.WriteUint64(uint64(o.Modified)) - xw.WriteUint64(o.Version) - xw.WriteUint64(o.LocalVersion) - xw.WriteUint32(o.NumBlocks) - return xw.Tot(), xw.Error() -} - -func (o *FileInfoTruncated) DecodeXDR(r io.Reader) error { - xr := xdr.NewReader(r) - return o.decodeXDR(xr) -} - -func (o *FileInfoTruncated) UnmarshalXDR(bs []byte) error { - var br = bytes.NewReader(bs) - var xr = xdr.NewReader(br) - return o.decodeXDR(xr) -} - -func (o *FileInfoTruncated) decodeXDR(xr *xdr.Reader) error { - o.Name = xr.ReadStringMax(8192) - o.Flags = xr.ReadUint32() - o.Modified = int64(xr.ReadUint64()) - o.Version = xr.ReadUint64() - o.LocalVersion = xr.ReadUint64() - o.NumBlocks = xr.ReadUint32() - return xr.Error() -} - -/* - BlockInfo Structure: 0 1 2 3 From 7a0a702ec0bcd4adfd6798695df5e8913586edf5 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Sun, 11 Jan 2015 13:24:56 +0100 Subject: [PATCH 22/94] Refactor readerLoop to switch on message type directly --- protocol.go | 58 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 24 deletions(-) diff --git a/protocol.go b/protocol.go index a55256799..1febf8b71 100644 --- a/protocol.go +++ b/protocol.go @@ -71,6 +71,10 @@ var ( ErrClosed = errors.New("connection closed") ) +// Specific variants of empty messages... +type pingMessage struct{ EmptyMessage } +type pongMessage struct{ EmptyMessage } + type Model interface { // An index was received from the peer device Index(deviceID DeviceID, folder string, files []FileInfo) @@ -289,48 +293,51 @@ func (c *rawConnection) readerLoop() (err error) { return err } - switch hdr.msgType { - case messageTypeIndex: - if c.state < stateCCRcvd { - return fmt.Errorf("protocol error: index message in state %d", c.state) - } - c.handleIndex(msg.(IndexMessage)) - c.state = stateIdxRcvd + switch msg := msg.(type) { + case IndexMessage: + switch hdr.msgType { + case messageTypeIndex: + if c.state < stateCCRcvd { + return fmt.Errorf("protocol error: index message in state %d", c.state) + } + c.handleIndex(msg) + c.state = stateIdxRcvd - case messageTypeIndexUpdate: - if c.state < stateIdxRcvd { - return fmt.Errorf("protocol error: index update message in state %d", c.state) + case messageTypeIndexUpdate: + if c.state < stateIdxRcvd { + return fmt.Errorf("protocol error: index update message in state %d", c.state) + } + c.handleIndexUpdate(msg) } - c.handleIndexUpdate(msg.(IndexMessage)) - case messageTypeRequest: + case RequestMessage: if c.state < stateIdxRcvd { return fmt.Errorf("protocol error: request message in state %d", c.state) } // Requests are handled asynchronously - go c.handleRequest(hdr.msgID, msg.(RequestMessage)) + go c.handleRequest(hdr.msgID, msg) - case messageTypeResponse: + case ResponseMessage: if c.state < stateIdxRcvd { return fmt.Errorf("protocol error: response message in state %d", c.state) } - c.handleResponse(hdr.msgID, msg.(ResponseMessage)) + c.handleResponse(hdr.msgID, msg) - case messageTypePing: - c.send(hdr.msgID, messageTypePong, EmptyMessage{}) + case pingMessage: + c.send(hdr.msgID, messageTypePong, pongMessage{}) - case messageTypePong: + case pongMessage: c.handlePong(hdr.msgID) - case messageTypeClusterConfig: + case ClusterConfigMessage: if c.state != stateInitial { return fmt.Errorf("protocol error: cluster config message in state %d", c.state) } - go c.receiver.ClusterConfig(c.id, msg.(ClusterConfigMessage)) + go c.receiver.ClusterConfig(c.id, msg) c.state = stateCCRcvd - case messageTypeClose: - return errors.New(msg.(CloseMessage).Reason) + case CloseMessage: + return errors.New(msg.Reason) default: return fmt.Errorf("protocol error: %s: unknown message type %#x", c.id, hdr.msgType) @@ -428,8 +435,11 @@ func (c *rawConnection) readMessage() (hdr header, msg encodable, err error) { } msg = resp - case messageTypePing, messageTypePong: - msg = EmptyMessage{} + case messageTypePing: + msg = pingMessage{} + + case messageTypePong: + msg = pongMessage{} case messageTypeClusterConfig: var cc ClusterConfigMessage From cd34eea017f9a955c415744db145be240f5774ae Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Sun, 11 Jan 2015 13:25:22 +0100 Subject: [PATCH 23/94] Reject Index and Request messages with unexpected flags --- protocol.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/protocol.go b/protocol.go index 1febf8b71..0c59ef56e 100644 --- a/protocol.go +++ b/protocol.go @@ -295,6 +295,11 @@ func (c *rawConnection) readerLoop() (err error) { switch msg := msg.(type) { case IndexMessage: + if msg.Flags != 0 { + // We don't currently support or expect any flags. + return fmt.Errorf("protocol error: unknown flags 0x%x in Index(Update) message", msg.Flags) + } + switch hdr.msgType { case messageTypeIndex: if c.state < stateCCRcvd { @@ -311,6 +316,10 @@ func (c *rawConnection) readerLoop() (err error) { } case RequestMessage: + if msg.Flags != 0 { + // We don't currently support or expect any flags. + return fmt.Errorf("protocol error: unknown flags 0x%x in Request message", msg.Flags) + } if c.state < stateIdxRcvd { return fmt.Errorf("protocol error: request message in state %d", c.state) } From 6213c4f2cdb09f12a51090ab1e389f9e2718902d Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Wed, 7 Jan 2015 15:44:36 +0100 Subject: [PATCH 24/94] Remove nil filenames from database and indexes (fixes #1243) --- protocol.go | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/protocol.go b/protocol.go index 0c59ef56e..5fe7cc344 100644 --- a/protocol.go +++ b/protocol.go @@ -477,14 +477,37 @@ func (c *rawConnection) handleIndex(im IndexMessage) { if debug { l.Debugf("Index(%v, %v, %d files)", c.id, im.Folder, len(im.Files)) } - c.receiver.Index(c.id, im.Folder, im.Files) + c.receiver.Index(c.id, im.Folder, filterIndexMessageFiles(im.Files)) } func (c *rawConnection) handleIndexUpdate(im IndexMessage) { if debug { l.Debugf("queueing IndexUpdate(%v, %v, %d files)", c.id, im.Folder, len(im.Files)) } - c.receiver.IndexUpdate(c.id, im.Folder, im.Files) + c.receiver.IndexUpdate(c.id, im.Folder, filterIndexMessageFiles(im.Files)) +} + +func filterIndexMessageFiles(fs []FileInfo) []FileInfo { + var out []FileInfo + for i, f := range fs { + if f.Name == "" { + l.Infoln("Dropping nil filename from incoming index") + if out == nil { + // Most incoming updates won't contain anything invalid, so we + // delay the allocation and copy to output slice until we + // really need to do it, then copy all the so var valid files + // to it. + out = make([]FileInfo, i, len(fs)-1) + copy(out, fs) + } + } else if out != nil { + out = append(out, f) + } + } + if out != nil { + return out + } + return fs } func (c *rawConnection) handleRequest(msgID int, req RequestMessage) { From d02158c0effa009fa4bb61421a868882aad0e470 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Tue, 13 Jan 2015 12:28:35 +0100 Subject: [PATCH 25/94] Also filter out some other obviously invalid filenames (ref #1243) --- protocol.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/protocol.go b/protocol.go index 5fe7cc344..414704286 100644 --- a/protocol.go +++ b/protocol.go @@ -490,8 +490,9 @@ func (c *rawConnection) handleIndexUpdate(im IndexMessage) { func filterIndexMessageFiles(fs []FileInfo) []FileInfo { var out []FileInfo for i, f := range fs { - if f.Name == "" { - l.Infoln("Dropping nil filename from incoming index") + switch f.Name { + case "", ".", "..", "/": // A few obviously invalid filenames + l.Infof("Dropping invalid filename %q from incoming index", f.Name) if out == nil { // Most incoming updates won't contain anything invalid, so we // delay the allocation and copy to output slice until we @@ -500,8 +501,10 @@ func filterIndexMessageFiles(fs []FileInfo) []FileInfo { out = make([]FileInfo, i, len(fs)-1) copy(out, fs) } - } else if out != nil { - out = append(out, f) + default: + if out != nil { + out = append(out, f) + } } } if out != nil { From 2ceaca88288923524ae402537bbdd9062c0aba39 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Tue, 13 Jan 2015 13:09:59 +0100 Subject: [PATCH 26/94] Add documentation copied from Syncthing --- AUTHORS | 4 + CONTRIBUTING.md | 76 ++++++ LICENSE | 674 ++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 13 + 4 files changed, 767 insertions(+) create mode 100644 AUTHORS create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE create mode 100644 README.md diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 000000000..f10d40d01 --- /dev/null +++ b/AUTHORS @@ -0,0 +1,4 @@ +# This is the official list of authors for copyright purposes. + +Audrius Butkevicius +Jakob Borg diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..856dde7d8 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,76 @@ +## Reporting Bugs + +Please file bugs in the [Github Issue +Tracker](https://github.com/syncthing/protocol/issues). + +## Contributing Code + +Every contribution is welcome. Following the points below will make this +a smoother process. + +Individuals making significant and valuable contributions are given +commit-access to the project. If you make a significant contribution and +are not considered for commit-access, please contact any of the +Syncthing core team members. + +All nontrivial contributions should go through the pull request +mechanism for internal review. Determining what is "nontrivial" is left +at the discretion of the contributor. + +### Authorship + +All code authors are listed in the AUTHORS file. Commits must be made +with the same name and email as listed in the AUTHORS file. To +accomplish this, ensure that your git configuration is set correctly +prior to making your first commit; + + $ git config --global user.name "Jane Doe" + $ git config --global user.email janedoe@example.com + +You must be reachable on the given email address. If you do not wish to +use your real name for whatever reason, using a nickname or pseudonym is +perfectly acceptable. + +## Coding Style + +- Follow the conventions laid out in [Effective Go](https://golang.org/doc/effective_go.html) + as much as makes sense. + +- All text files use Unix line endings. + +- Each commit should be `go fmt` clean. + +- The commit message subject should be a single short sentence + describing the change, starting with a capital letter. + +- Commits that resolve an existing issue must include the issue number + as `(fixes #123)` at the end of the commit message subject. + +- Imports are grouped per `goimports` standard; that is, standard + library first, then third party libraries after a blank line. + +- A contribution solving a single issue or introducing a single new + feature should probably be a single commit based on the current + `master` branch. You may be asked to "rebase" or "squash" your pull + request to make sure this is the case, especially if there have been + amendments during review. + +## Licensing + +All contributions are made under the same GPL license as the rest of the +project, except documentation, user interface text and translation +strings which are licensed under the Creative Commons Attribution 4.0 +International License. You retain the copyright to code you have +written. + +When accepting your first contribution, the maintainer of the project +will ensure that you are added to the AUTHORS file. You are welcome to +add yourself as a separate commit in your first pull request. + +## Tests + +Yes please! + +## License + +GPLv3 diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..94a9ed024 --- /dev/null +++ b/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/README.md b/README.md new file mode 100644 index 000000000..f7dcf8dfb --- /dev/null +++ b/README.md @@ -0,0 +1,13 @@ +The BEPv1 Protocol +================== + +[![Latest Build](http://img.shields.io/jenkins/s/http/build.syncthing.net/protocol.svg?style=flat-square)](http://build.syncthing.net/job/protocol/lastBuild/) +[![API Documentation](http://img.shields.io/badge/api-Godoc-blue.svg?style=flat-square)](http://godoc.org/github.com/syncthing/protocol) +[![GPL License](http://img.shields.io/badge/license-GPL-blue.svg?style=flat-square)](http://opensource.org/licenses/GPL-3.0) + +This is the protocol implementation used by Syncthing. + +License +======= + +GPLv3 From 4833b6085c5b152914620d5fa70e548fdd019ca7 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Tue, 13 Jan 2015 13:20:29 +0100 Subject: [PATCH 27/94] The luhn package moved --- deviceid.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deviceid.go b/deviceid.go index 93753d7fa..21b323255 100644 --- a/deviceid.go +++ b/deviceid.go @@ -24,7 +24,7 @@ import ( "regexp" "strings" - "github.com/syncthing/syncthing/internal/luhn" + "github.com/calmh/luhn" ) type DeviceID [32]byte From d84a8e64043f8d6c41cc8d6b7d5ab31c0a25b4c2 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Tue, 13 Jan 2015 13:31:14 +0100 Subject: [PATCH 28/94] Relicense as MIT --- AUTHORS | 2 +- CONTRIBUTING.md | 4 +- LICENSE | 693 ++--------------------------------------- README.md | 4 +- common_test.go | 15 +- counting.go | 15 +- debug.go | 15 +- deviceid.go | 15 +- deviceid_test.go | 15 +- doc.go | 15 +- header.go | 15 +- message.go | 15 +- nativemodel_darwin.go | 15 +- nativemodel_unix.go | 15 +- nativemodel_windows.go | 15 +- protocol.go | 15 +- protocol_test.go | 15 +- wireformat.go | 15 +- 18 files changed, 38 insertions(+), 875 deletions(-) diff --git a/AUTHORS b/AUTHORS index f10d40d01..d84404ee2 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,4 +1,4 @@ -# This is the official list of authors for copyright purposes. +# This is the official list of Protocol Authors for copyright purposes. Audrius Butkevicius Jakob Borg diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 856dde7d8..67e6a9c70 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -57,7 +57,7 @@ perfectly acceptable. ## Licensing -All contributions are made under the same GPL license as the rest of the +All contributions are made under the same MIT license as the rest of the project, except documentation, user interface text and translation strings which are licensed under the Creative Commons Attribution 4.0 International License. You retain the copyright to code you have @@ -73,4 +73,4 @@ Yes please! ## License -GPLv3 +MIT diff --git a/LICENSE b/LICENSE index 94a9ed024..6f6960a75 100644 --- a/LICENSE +++ b/LICENSE @@ -1,674 +1,19 @@ - GNU GENERAL PUBLIC LICENSE - Version 3, 29 June 2007 - - Copyright (C) 2007 Free Software Foundation, Inc. - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The GNU General Public License is a free, copyleft license for -software and other kinds of works. - - The licenses for most software and other practical works are designed -to take away your freedom to share and change the works. By contrast, -the GNU General Public License is intended to guarantee your freedom to -share and change all versions of a program--to make sure it remains free -software for all its users. We, the Free Software Foundation, use the -GNU General Public License for most of our software; it applies also to -any other work released this way by its authors. You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -them if you wish), that you receive source code or can get it if you -want it, that you can change the software or use pieces of it in new -free programs, and that you know you can do these things. - - To protect your rights, we need to prevent others from denying you -these rights or asking you to surrender the rights. Therefore, you have -certain responsibilities if you distribute copies of the software, or if -you modify it: responsibilities to respect the freedom of others. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must pass on to the recipients the same -freedoms that you received. You must make sure that they, too, receive -or can get the source code. And you must show them these terms so they -know their rights. - - Developers that use the GNU GPL protect your rights with two steps: -(1) assert copyright on the software, and (2) offer you this License -giving you legal permission to copy, distribute and/or modify it. - - For the developers' and authors' protection, the GPL clearly explains -that there is no warranty for this free software. For both users' and -authors' sake, the GPL requires that modified versions be marked as -changed, so that their problems will not be attributed erroneously to -authors of previous versions. - - Some devices are designed to deny users access to install or run -modified versions of the software inside them, although the manufacturer -can do so. This is fundamentally incompatible with the aim of -protecting users' freedom to change the software. The systematic -pattern of such abuse occurs in the area of products for individuals to -use, which is precisely where it is most unacceptable. Therefore, we -have designed this version of the GPL to prohibit the practice for those -products. If such problems arise substantially in other domains, we -stand ready to extend this provision to those domains in future versions -of the GPL, as needed to protect the freedom of users. - - Finally, every program is threatened constantly by software patents. -States should not allow patents to restrict development and use of -software on general-purpose computers, but in those that do, we wish to -avoid the special danger that patents applied to a free program could -make it effectively proprietary. To prevent this, the GPL assures that -patents cannot be used to render the program non-free. - - The precise terms and conditions for copying, distribution and -modification follow. - - TERMS AND CONDITIONS - - 0. Definitions. - - "This License" refers to version 3 of the GNU General Public License. - - "Copyright" also means copyright-like laws that apply to other kinds of -works, such as semiconductor masks. - - "The Program" refers to any copyrightable work licensed under this -License. Each licensee is addressed as "you". "Licensees" and -"recipients" may be individuals or organizations. - - To "modify" a work means to copy from or adapt all or part of the work -in a fashion requiring copyright permission, other than the making of an -exact copy. The resulting work is called a "modified version" of the -earlier work or a work "based on" the earlier work. - - A "covered work" means either the unmodified Program or a work based -on the Program. - - To "propagate" a work means to do anything with it that, without -permission, would make you directly or secondarily liable for -infringement under applicable copyright law, except executing it on a -computer or modifying a private copy. Propagation includes copying, -distribution (with or without modification), making available to the -public, and in some countries other activities as well. - - To "convey" a work means any kind of propagation that enables other -parties to make or receive copies. Mere interaction with a user through -a computer network, with no transfer of a copy, is not conveying. - - An interactive user interface displays "Appropriate Legal Notices" -to the extent that it includes a convenient and prominently visible -feature that (1) displays an appropriate copyright notice, and (2) -tells the user that there is no warranty for the work (except to the -extent that warranties are provided), that licensees may convey the -work under this License, and how to view a copy of this License. If -the interface presents a list of user commands or options, such as a -menu, a prominent item in the list meets this criterion. - - 1. Source Code. - - The "source code" for a work means the preferred form of the work -for making modifications to it. "Object code" means any non-source -form of a work. - - A "Standard Interface" means an interface that either is an official -standard defined by a recognized standards body, or, in the case of -interfaces specified for a particular programming language, one that -is widely used among developers working in that language. - - The "System Libraries" of an executable work include anything, other -than the work as a whole, that (a) is included in the normal form of -packaging a Major Component, but which is not part of that Major -Component, and (b) serves only to enable use of the work with that -Major Component, or to implement a Standard Interface for which an -implementation is available to the public in source code form. A -"Major Component", in this context, means a major essential component -(kernel, window system, and so on) of the specific operating system -(if any) on which the executable work runs, or a compiler used to -produce the work, or an object code interpreter used to run it. - - The "Corresponding Source" for a work in object code form means all -the source code needed to generate, install, and (for an executable -work) run the object code and to modify the work, including scripts to -control those activities. However, it does not include the work's -System Libraries, or general-purpose tools or generally available free -programs which are used unmodified in performing those activities but -which are not part of the work. For example, Corresponding Source -includes interface definition files associated with source files for -the work, and the source code for shared libraries and dynamically -linked subprograms that the work is specifically designed to require, -such as by intimate data communication or control flow between those -subprograms and other parts of the work. - - The Corresponding Source need not include anything that users -can regenerate automatically from other parts of the Corresponding -Source. - - The Corresponding Source for a work in source code form is that -same work. - - 2. Basic Permissions. - - All rights granted under this License are granted for the term of -copyright on the Program, and are irrevocable provided the stated -conditions are met. This License explicitly affirms your unlimited -permission to run the unmodified Program. The output from running a -covered work is covered by this License only if the output, given its -content, constitutes a covered work. This License acknowledges your -rights of fair use or other equivalent, as provided by copyright law. - - You may make, run and propagate covered works that you do not -convey, without conditions so long as your license otherwise remains -in force. You may convey covered works to others for the sole purpose -of having them make modifications exclusively for you, or provide you -with facilities for running those works, provided that you comply with -the terms of this License in conveying all material for which you do -not control copyright. Those thus making or running the covered works -for you must do so exclusively on your behalf, under your direction -and control, on terms that prohibit them from making any copies of -your copyrighted material outside their relationship with you. - - Conveying under any other circumstances is permitted solely under -the conditions stated below. Sublicensing is not allowed; section 10 -makes it unnecessary. - - 3. Protecting Users' Legal Rights From Anti-Circumvention Law. - - No covered work shall be deemed part of an effective technological -measure under any applicable law fulfilling obligations under article -11 of the WIPO copyright treaty adopted on 20 December 1996, or -similar laws prohibiting or restricting circumvention of such -measures. - - When you convey a covered work, you waive any legal power to forbid -circumvention of technological measures to the extent such circumvention -is effected by exercising rights under this License with respect to -the covered work, and you disclaim any intention to limit operation or -modification of the work as a means of enforcing, against the work's -users, your or third parties' legal rights to forbid circumvention of -technological measures. - - 4. Conveying Verbatim Copies. - - You may convey verbatim copies of the Program's source code as you -receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice; -keep intact all notices stating that this License and any -non-permissive terms added in accord with section 7 apply to the code; -keep intact all notices of the absence of any warranty; and give all -recipients a copy of this License along with the Program. - - You may charge any price or no price for each copy that you convey, -and you may offer support or warranty protection for a fee. - - 5. Conveying Modified Source Versions. - - You may convey a work based on the Program, or the modifications to -produce it from the Program, in the form of source code under the -terms of section 4, provided that you also meet all of these conditions: - - a) The work must carry prominent notices stating that you modified - it, and giving a relevant date. - - b) The work must carry prominent notices stating that it is - released under this License and any conditions added under section - 7. This requirement modifies the requirement in section 4 to - "keep intact all notices". - - c) You must license the entire work, as a whole, under this - License to anyone who comes into possession of a copy. This - License will therefore apply, along with any applicable section 7 - additional terms, to the whole of the work, and all its parts, - regardless of how they are packaged. This License gives no - permission to license the work in any other way, but it does not - invalidate such permission if you have separately received it. - - d) If the work has interactive user interfaces, each must display - Appropriate Legal Notices; however, if the Program has interactive - interfaces that do not display Appropriate Legal Notices, your - work need not make them do so. - - A compilation of a covered work with other separate and independent -works, which are not by their nature extensions of the covered work, -and which are not combined with it such as to form a larger program, -in or on a volume of a storage or distribution medium, is called an -"aggregate" if the compilation and its resulting copyright are not -used to limit the access or legal rights of the compilation's users -beyond what the individual works permit. Inclusion of a covered work -in an aggregate does not cause this License to apply to the other -parts of the aggregate. - - 6. Conveying Non-Source Forms. - - You may convey a covered work in object code form under the terms -of sections 4 and 5, provided that you also convey the -machine-readable Corresponding Source under the terms of this License, -in one of these ways: - - a) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by the - Corresponding Source fixed on a durable physical medium - customarily used for software interchange. - - b) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by a - written offer, valid for at least three years and valid for as - long as you offer spare parts or customer support for that product - model, to give anyone who possesses the object code either (1) a - copy of the Corresponding Source for all the software in the - product that is covered by this License, on a durable physical - medium customarily used for software interchange, for a price no - more than your reasonable cost of physically performing this - conveying of source, or (2) access to copy the - Corresponding Source from a network server at no charge. - - c) Convey individual copies of the object code with a copy of the - written offer to provide the Corresponding Source. This - alternative is allowed only occasionally and noncommercially, and - only if you received the object code with such an offer, in accord - with subsection 6b. - - d) Convey the object code by offering access from a designated - place (gratis or for a charge), and offer equivalent access to the - Corresponding Source in the same way through the same place at no - further charge. You need not require recipients to copy the - Corresponding Source along with the object code. If the place to - copy the object code is a network server, the Corresponding Source - may be on a different server (operated by you or a third party) - that supports equivalent copying facilities, provided you maintain - clear directions next to the object code saying where to find the - Corresponding Source. Regardless of what server hosts the - Corresponding Source, you remain obligated to ensure that it is - available for as long as needed to satisfy these requirements. - - e) Convey the object code using peer-to-peer transmission, provided - you inform other peers where the object code and Corresponding - Source of the work are being offered to the general public at no - charge under subsection 6d. - - A separable portion of the object code, whose source code is excluded -from the Corresponding Source as a System Library, need not be -included in conveying the object code work. - - A "User Product" is either (1) a "consumer product", which means any -tangible personal property which is normally used for personal, family, -or household purposes, or (2) anything designed or sold for incorporation -into a dwelling. In determining whether a product is a consumer product, -doubtful cases shall be resolved in favor of coverage. For a particular -product received by a particular user, "normally used" refers to a -typical or common use of that class of product, regardless of the status -of the particular user or of the way in which the particular user -actually uses, or expects or is expected to use, the product. A product -is a consumer product regardless of whether the product has substantial -commercial, industrial or non-consumer uses, unless such uses represent -the only significant mode of use of the product. - - "Installation Information" for a User Product means any methods, -procedures, authorization keys, or other information required to install -and execute modified versions of a covered work in that User Product from -a modified version of its Corresponding Source. The information must -suffice to ensure that the continued functioning of the modified object -code is in no case prevented or interfered with solely because -modification has been made. - - If you convey an object code work under this section in, or with, or -specifically for use in, a User Product, and the conveying occurs as -part of a transaction in which the right of possession and use of the -User Product is transferred to the recipient in perpetuity or for a -fixed term (regardless of how the transaction is characterized), the -Corresponding Source conveyed under this section must be accompanied -by the Installation Information. But this requirement does not apply -if neither you nor any third party retains the ability to install -modified object code on the User Product (for example, the work has -been installed in ROM). - - The requirement to provide Installation Information does not include a -requirement to continue to provide support service, warranty, or updates -for a work that has been modified or installed by the recipient, or for -the User Product in which it has been modified or installed. Access to a -network may be denied when the modification itself materially and -adversely affects the operation of the network or violates the rules and -protocols for communication across the network. - - Corresponding Source conveyed, and Installation Information provided, -in accord with this section must be in a format that is publicly -documented (and with an implementation available to the public in -source code form), and must require no special password or key for -unpacking, reading or copying. - - 7. Additional Terms. - - "Additional permissions" are terms that supplement the terms of this -License by making exceptions from one or more of its conditions. -Additional permissions that are applicable to the entire Program shall -be treated as though they were included in this License, to the extent -that they are valid under applicable law. If additional permissions -apply only to part of the Program, that part may be used separately -under those permissions, but the entire Program remains governed by -this License without regard to the additional permissions. - - When you convey a copy of a covered work, you may at your option -remove any additional permissions from that copy, or from any part of -it. (Additional permissions may be written to require their own -removal in certain cases when you modify the work.) You may place -additional permissions on material, added by you to a covered work, -for which you have or can give appropriate copyright permission. - - Notwithstanding any other provision of this License, for material you -add to a covered work, you may (if authorized by the copyright holders of -that material) supplement the terms of this License with terms: - - a) Disclaiming warranty or limiting liability differently from the - terms of sections 15 and 16 of this License; or - - b) Requiring preservation of specified reasonable legal notices or - author attributions in that material or in the Appropriate Legal - Notices displayed by works containing it; or - - c) Prohibiting misrepresentation of the origin of that material, or - requiring that modified versions of such material be marked in - reasonable ways as different from the original version; or - - d) Limiting the use for publicity purposes of names of licensors or - authors of the material; or - - e) Declining to grant rights under trademark law for use of some - trade names, trademarks, or service marks; or - - f) Requiring indemnification of licensors and authors of that - material by anyone who conveys the material (or modified versions of - it) with contractual assumptions of liability to the recipient, for - any liability that these contractual assumptions directly impose on - those licensors and authors. - - All other non-permissive additional terms are considered "further -restrictions" within the meaning of section 10. If the Program as you -received it, or any part of it, contains a notice stating that it is -governed by this License along with a term that is a further -restriction, you may remove that term. If a license document contains -a further restriction but permits relicensing or conveying under this -License, you may add to a covered work material governed by the terms -of that license document, provided that the further restriction does -not survive such relicensing or conveying. - - If you add terms to a covered work in accord with this section, you -must place, in the relevant source files, a statement of the -additional terms that apply to those files, or a notice indicating -where to find the applicable terms. - - Additional terms, permissive or non-permissive, may be stated in the -form of a separately written license, or stated as exceptions; -the above requirements apply either way. - - 8. Termination. - - You may not propagate or modify a covered work except as expressly -provided under this License. Any attempt otherwise to propagate or -modify it is void, and will automatically terminate your rights under -this License (including any patent licenses granted under the third -paragraph of section 11). - - However, if you cease all violation of this License, then your -license from a particular copyright holder is reinstated (a) -provisionally, unless and until the copyright holder explicitly and -finally terminates your license, and (b) permanently, if the copyright -holder fails to notify you of the violation by some reasonable means -prior to 60 days after the cessation. - - Moreover, your license from a particular copyright holder is -reinstated permanently if the copyright holder notifies you of the -violation by some reasonable means, this is the first time you have -received notice of violation of this License (for any work) from that -copyright holder, and you cure the violation prior to 30 days after -your receipt of the notice. - - Termination of your rights under this section does not terminate the -licenses of parties who have received copies or rights from you under -this License. If your rights have been terminated and not permanently -reinstated, you do not qualify to receive new licenses for the same -material under section 10. - - 9. Acceptance Not Required for Having Copies. - - You are not required to accept this License in order to receive or -run a copy of the Program. Ancillary propagation of a covered work -occurring solely as a consequence of using peer-to-peer transmission -to receive a copy likewise does not require acceptance. However, -nothing other than this License grants you permission to propagate or -modify any covered work. These actions infringe copyright if you do -not accept this License. Therefore, by modifying or propagating a -covered work, you indicate your acceptance of this License to do so. - - 10. Automatic Licensing of Downstream Recipients. - - Each time you convey a covered work, the recipient automatically -receives a license from the original licensors, to run, modify and -propagate that work, subject to this License. You are not responsible -for enforcing compliance by third parties with this License. - - An "entity transaction" is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an -organization, or merging organizations. If propagation of a covered -work results from an entity transaction, each party to that -transaction who receives a copy of the work also receives whatever -licenses to the work the party's predecessor in interest had or could -give under the previous paragraph, plus a right to possession of the -Corresponding Source of the work from the predecessor in interest, if -the predecessor has it or can get it with reasonable efforts. - - You may not impose any further restrictions on the exercise of the -rights granted or affirmed under this License. For example, you may -not impose a license fee, royalty, or other charge for exercise of -rights granted under this License, and you may not initiate litigation -(including a cross-claim or counterclaim in a lawsuit) alleging that -any patent claim is infringed by making, using, selling, offering for -sale, or importing the Program or any portion of it. - - 11. Patents. - - A "contributor" is a copyright holder who authorizes use under this -License of the Program or a work on which the Program is based. The -work thus licensed is called the contributor's "contributor version". - - A contributor's "essential patent claims" are all patent claims -owned or controlled by the contributor, whether already acquired or -hereafter acquired, that would be infringed by some manner, permitted -by this License, of making, using, or selling its contributor version, -but do not include claims that would be infringed only as a -consequence of further modification of the contributor version. For -purposes of this definition, "control" includes the right to grant -patent sublicenses in a manner consistent with the requirements of -this License. - - Each contributor grants you a non-exclusive, worldwide, royalty-free -patent license under the contributor's essential patent claims, to -make, use, sell, offer for sale, import and otherwise run, modify and -propagate the contents of its contributor version. - - In the following three paragraphs, a "patent license" is any express -agreement or commitment, however denominated, not to enforce a patent -(such as an express permission to practice a patent or covenant not to -sue for patent infringement). To "grant" such a patent license to a -party means to make such an agreement or commitment not to enforce a -patent against the party. - - If you convey a covered work, knowingly relying on a patent license, -and the Corresponding Source of the work is not available for anyone -to copy, free of charge and under the terms of this License, through a -publicly available network server or other readily accessible means, -then you must either (1) cause the Corresponding Source to be so -available, or (2) arrange to deprive yourself of the benefit of the -patent license for this particular work, or (3) arrange, in a manner -consistent with the requirements of this License, to extend the patent -license to downstream recipients. "Knowingly relying" means you have -actual knowledge that, but for the patent license, your conveying the -covered work in a country, or your recipient's use of the covered work -in a country, would infringe one or more identifiable patents in that -country that you have reason to believe are valid. - - If, pursuant to or in connection with a single transaction or -arrangement, you convey, or propagate by procuring conveyance of, a -covered work, and grant a patent license to some of the parties -receiving the covered work authorizing them to use, propagate, modify -or convey a specific copy of the covered work, then the patent license -you grant is automatically extended to all recipients of the covered -work and works based on it. - - A patent license is "discriminatory" if it does not include within -the scope of its coverage, prohibits the exercise of, or is -conditioned on the non-exercise of one or more of the rights that are -specifically granted under this License. You may not convey a covered -work if you are a party to an arrangement with a third party that is -in the business of distributing software, under which you make payment -to the third party based on the extent of your activity of conveying -the work, and under which the third party grants, to any of the -parties who would receive the covered work from you, a discriminatory -patent license (a) in connection with copies of the covered work -conveyed by you (or copies made from those copies), or (b) primarily -for and in connection with specific products or compilations that -contain the covered work, unless you entered into that arrangement, -or that patent license was granted, prior to 28 March 2007. - - Nothing in this License shall be construed as excluding or limiting -any implied license or other defenses to infringement that may -otherwise be available to you under applicable patent law. - - 12. No Surrender of Others' Freedom. - - If conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot convey a -covered work so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you may -not convey it at all. For example, if you agree to terms that obligate you -to collect a royalty for further conveying from those to whom you convey -the Program, the only way you could satisfy both those terms and this -License would be to refrain entirely from conveying the Program. - - 13. Use with the GNU Affero General Public License. - - Notwithstanding any other provision of this License, you have -permission to link or combine any covered work with a work licensed -under version 3 of the GNU Affero General Public License into a single -combined work, and to convey the resulting work. The terms of this -License will continue to apply to the part which is the covered work, -but the special requirements of the GNU Affero General Public License, -section 13, concerning interaction through a network will apply to the -combination as such. - - 14. Revised Versions of this License. - - The Free Software Foundation may publish revised and/or new versions of -the GNU General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - - Each version is given a distinguishing version number. If the -Program specifies that a certain numbered version of the GNU General -Public License "or any later version" applies to it, you have the -option of following the terms and conditions either of that numbered -version or of any later version published by the Free Software -Foundation. If the Program does not specify a version number of the -GNU General Public License, you may choose any version ever published -by the Free Software Foundation. - - If the Program specifies that a proxy can decide which future -versions of the GNU General Public License can be used, that proxy's -public statement of acceptance of a version permanently authorizes you -to choose that version for the Program. - - Later license versions may give you additional or different -permissions. However, no additional obligations are imposed on any -author or copyright holder as a result of your choosing to follow a -later version. - - 15. Disclaimer of Warranty. - - THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY -APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT -HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY -OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM -IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF -ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. Limitation of Liability. - - IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS -THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE -USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF -DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD -PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), -EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF -SUCH DAMAGES. - - 17. Interpretation of Sections 15 and 16. - - If the disclaimer of warranty and limitation of liability provided -above cannot be given local legal effect according to their terms, -reviewing courts shall apply local law that most closely approximates -an absolute waiver of all civil liability in connection with the -Program, unless a warranty or assumption of liability accompanies a -copy of the Program in return for a fee. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -state the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - -Also add information on how to contact you by electronic and paper mail. - - If the program does terminal interaction, make it output a short -notice like this when it starts in an interactive mode: - - Copyright (C) - This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, your program's commands -might be different; for a GUI interface, you would use an "about box". - - You should also get your employer (if you work as a programmer) or school, -if any, to sign a "copyright disclaimer" for the program, if necessary. -For more information on this, and how to apply and follow the GNU GPL, see -. - - The GNU General Public License does not permit incorporating your program -into proprietary programs. If your program is a subroutine library, you -may consider it more useful to permit linking proprietary applications with -the library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. But first, please read -. +Copyright (C) 2014-2015 The Protocol Authors + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +- The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index f7dcf8dfb..bcba44b42 100644 --- a/README.md +++ b/README.md @@ -3,11 +3,11 @@ The BEPv1 Protocol [![Latest Build](http://img.shields.io/jenkins/s/http/build.syncthing.net/protocol.svg?style=flat-square)](http://build.syncthing.net/job/protocol/lastBuild/) [![API Documentation](http://img.shields.io/badge/api-Godoc-blue.svg?style=flat-square)](http://godoc.org/github.com/syncthing/protocol) -[![GPL License](http://img.shields.io/badge/license-GPL-blue.svg?style=flat-square)](http://opensource.org/licenses/GPL-3.0) +[![MIT License](http://img.shields.io/badge/license-MIT-blue.svg?style=flat-square)](http://opensource.org/licenses/MIT) This is the protocol implementation used by Syncthing. License ======= -GPLv3 +MIT diff --git a/common_test.go b/common_test.go index e38da00a8..f67fb4812 100644 --- a/common_test.go +++ b/common_test.go @@ -1,17 +1,4 @@ -// Copyright (C) 2014 The Syncthing Authors. -// -// This program is free software: you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along -// with this program. If not, see . +// Copyright (C) 2014 The Protocol Authors. package protocol diff --git a/counting.go b/counting.go index ac4aeaa9a..490b77fd1 100644 --- a/counting.go +++ b/counting.go @@ -1,17 +1,4 @@ -// Copyright (C) 2014 The Syncthing Authors. -// -// This program is free software: you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along -// with this program. If not, see . +// Copyright (C) 2014 The Protocol Authors. package protocol diff --git a/debug.go b/debug.go index 4664b0e23..435d7f5d2 100644 --- a/debug.go +++ b/debug.go @@ -1,17 +1,4 @@ -// Copyright (C) 2014 The Syncthing Authors. -// -// This program is free software: you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along -// with this program. If not, see . +// Copyright (C) 2014 The Protocol Authors. package protocol diff --git a/deviceid.go b/deviceid.go index 21b323255..f3b3c5a31 100644 --- a/deviceid.go +++ b/deviceid.go @@ -1,17 +1,4 @@ -// Copyright (C) 2014 The Syncthing Authors. -// -// This program is free software: you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along -// with this program. If not, see . +// Copyright (C) 2014 The Protocol Authors. package protocol diff --git a/deviceid_test.go b/deviceid_test.go index da398c6fb..613557d32 100644 --- a/deviceid_test.go +++ b/deviceid_test.go @@ -1,17 +1,4 @@ -// Copyright (C) 2014 The Syncthing Authors. -// -// This program is free software: you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along -// with this program. If not, see . +// Copyright (C) 2014 The Protocol Authors. package protocol diff --git a/doc.go b/doc.go index 4f73e6c86..2c6ea8ef2 100644 --- a/doc.go +++ b/doc.go @@ -1,17 +1,4 @@ -// Copyright (C) 2014 The Syncthing Authors. -// -// This program is free software: you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along -// with this program. If not, see . +// Copyright (C) 2014 The Protocol Authors. // Package protocol implements the Block Exchange Protocol. package protocol diff --git a/header.go b/header.go index 8da092e5c..846ee48cd 100644 --- a/header.go +++ b/header.go @@ -1,17 +1,4 @@ -// Copyright (C) 2014 The Syncthing Authors. -// -// This program is free software: you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along -// with this program. If not, see . +// Copyright (C) 2014 The Protocol Authors. package protocol diff --git a/message.go b/message.go index 0d8de8070..d85f21cc6 100644 --- a/message.go +++ b/message.go @@ -1,17 +1,4 @@ -// Copyright (C) 2014 The Syncthing Authors. -// -// This program is free software: you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along -// with this program. If not, see . +// Copyright (C) 2014 The Protocol Authors. //go:generate -command genxdr go run ../../Godeps/_workspace/src/github.com/calmh/xdr/cmd/genxdr/main.go //go:generate genxdr -o message_xdr.go message.go diff --git a/nativemodel_darwin.go b/nativemodel_darwin.go index ba30d1ae1..5b4b9be67 100644 --- a/nativemodel_darwin.go +++ b/nativemodel_darwin.go @@ -1,17 +1,4 @@ -// Copyright (C) 2014 The Syncthing Authors. -// -// This program is free software: you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along -// with this program. If not, see . +// Copyright (C) 2014 The Protocol Authors. // +build darwin diff --git a/nativemodel_unix.go b/nativemodel_unix.go index bf6499e63..2fb1654c7 100644 --- a/nativemodel_unix.go +++ b/nativemodel_unix.go @@ -1,17 +1,4 @@ -// Copyright (C) 2014 The Syncthing Authors. -// -// This program is free software: you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along -// with this program. If not, see . +// Copyright (C) 2014 The Protocol Authors. // +build !windows,!darwin diff --git a/nativemodel_windows.go b/nativemodel_windows.go index d2c079bdc..d4feea326 100644 --- a/nativemodel_windows.go +++ b/nativemodel_windows.go @@ -1,17 +1,4 @@ -// Copyright (C) 2014 The Syncthing Authors. -// -// This program is free software: you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along -// with this program. If not, see . +// Copyright (C) 2014 The Protocol Authors. // +build windows diff --git a/protocol.go b/protocol.go index 414704286..7cf59af1f 100644 --- a/protocol.go +++ b/protocol.go @@ -1,17 +1,4 @@ -// Copyright (C) 2014 The Syncthing Authors. -// -// This program is free software: you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along -// with this program. If not, see . +// Copyright (C) 2014 The Protocol Authors. package protocol diff --git a/protocol_test.go b/protocol_test.go index 75e0086ee..1ccb4525f 100644 --- a/protocol_test.go +++ b/protocol_test.go @@ -1,17 +1,4 @@ -// Copyright (C) 2014 The Syncthing Authors. -// -// This program is free software: you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along -// with this program. If not, see . +// Copyright (C) 2014 The Protocol Authors. package protocol diff --git a/wireformat.go b/wireformat.go index 84da71423..4eab3d37e 100644 --- a/wireformat.go +++ b/wireformat.go @@ -1,17 +1,4 @@ -// Copyright (C) 2014 The Syncthing Authors. -// -// This program is free software: you can redistribute it and/or modify it -// under the terms of the GNU General Public License as published by the Free -// Software Foundation, either version 3 of the License, or (at your option) -// any later version. -// -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -// more details. -// -// You should have received a copy of the GNU General Public License along -// with this program. If not, see . +// Copyright (C) 2014 The Protocol Authors. package protocol From f76b5d800208131015b14cfc1cfd3e6dd1d5e979 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Tue, 13 Jan 2015 13:47:16 +0100 Subject: [PATCH 29/94] rm '.gitignore' --- .gitignore | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .gitignore diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 2211df63d..000000000 --- a/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.txt From 3450b5f80cbf69777693cb4762685817ee2489ad Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Sun, 18 Jan 2015 01:26:52 +0100 Subject: [PATCH 30/94] Integer type policy Integers are for numbers, enabling arithmetic like subtractions and for loops without getting shot in the foot. Unsigneds are for bitfields. - "int" for numbers that will always be laughably smaller than four billion, and where we don't care about the serialization format. - "int32" for numbers that will always be laughably smaller than four billion, and will be serialized to four bytes. - "int64" for numbers that may approach four billion or will be serialized to eight bytes. - "uint32" and "uint64" for bitfields, depending on required number of bits and serialization format. Likewise "uint8" and "uint16", although rare in this project since they don't exist in XDR. - "int8", "int16" and plain "uint" are almost never useful. --- counting.go | 32 ++++++++++++++++---------------- message.go | 17 ++++++++--------- message_xdr.go | 48 ++++++++++++++++++++++++------------------------ protocol.go | 8 ++++---- 4 files changed, 52 insertions(+), 53 deletions(-) diff --git a/counting.go b/counting.go index 490b77fd1..d441ed311 100644 --- a/counting.go +++ b/counting.go @@ -10,25 +10,25 @@ import ( type countingReader struct { io.Reader - tot uint64 // bytes - last int64 // unix nanos + tot int64 // bytes + last int64 // unix nanos } var ( - totalIncoming uint64 - totalOutgoing uint64 + totalIncoming int64 + totalOutgoing int64 ) func (c *countingReader) Read(bs []byte) (int, error) { n, err := c.Reader.Read(bs) - atomic.AddUint64(&c.tot, uint64(n)) - atomic.AddUint64(&totalIncoming, uint64(n)) + atomic.AddInt64(&c.tot, int64(n)) + atomic.AddInt64(&totalIncoming, int64(n)) atomic.StoreInt64(&c.last, time.Now().UnixNano()) return n, err } -func (c *countingReader) Tot() uint64 { - return atomic.LoadUint64(&c.tot) +func (c *countingReader) Tot() int64 { + return atomic.LoadInt64(&c.tot) } func (c *countingReader) Last() time.Time { @@ -37,26 +37,26 @@ func (c *countingReader) Last() time.Time { type countingWriter struct { io.Writer - tot uint64 // bytes - last int64 // unix nanos + tot int64 // bytes + last int64 // unix nanos } func (c *countingWriter) Write(bs []byte) (int, error) { n, err := c.Writer.Write(bs) - atomic.AddUint64(&c.tot, uint64(n)) - atomic.AddUint64(&totalOutgoing, uint64(n)) + atomic.AddInt64(&c.tot, int64(n)) + atomic.AddInt64(&totalOutgoing, int64(n)) atomic.StoreInt64(&c.last, time.Now().UnixNano()) return n, err } -func (c *countingWriter) Tot() uint64 { - return atomic.LoadUint64(&c.tot) +func (c *countingWriter) Tot() int64 { + return atomic.LoadInt64(&c.tot) } func (c *countingWriter) Last() time.Time { return time.Unix(0, atomic.LoadInt64(&c.last)) } -func TotalInOut() (uint64, uint64) { - return atomic.LoadUint64(&totalIncoming), atomic.LoadUint64(&totalOutgoing) +func TotalInOut() (int64, int64) { + return atomic.LoadInt64(&totalIncoming), atomic.LoadInt64(&totalOutgoing) } diff --git a/message.go b/message.go index d85f21cc6..dbaf526ac 100644 --- a/message.go +++ b/message.go @@ -1,6 +1,5 @@ // Copyright (C) 2014 The Protocol Authors. -//go:generate -command genxdr go run ../../Godeps/_workspace/src/github.com/calmh/xdr/cmd/genxdr/main.go //go:generate genxdr -o message_xdr.go message.go package protocol @@ -18,8 +17,8 @@ type FileInfo struct { Name string // max:8192 Flags uint32 Modified int64 - Version uint64 - LocalVersion uint64 + Version int64 + LocalVersion int64 Blocks []BlockInfo } @@ -60,7 +59,7 @@ func (f FileInfo) HasPermissionBits() bool { type BlockInfo struct { Offset int64 // noencode (cache only) - Size uint32 + Size int32 Hash []byte // max:64 } @@ -71,8 +70,8 @@ func (b BlockInfo) String() string { type RequestMessage struct { Folder string // max:64 Name string // max:8192 - Offset uint64 - Size uint32 + Offset int64 + Size int32 Hash []byte // max:64 Flags uint32 Options []Option // max:64 @@ -80,7 +79,7 @@ type RequestMessage struct { type ResponseMessage struct { Data []byte - Error uint32 + Error int32 } type ClusterConfigMessage struct { @@ -107,7 +106,7 @@ type Folder struct { type Device struct { ID []byte // max:32 Flags uint32 - MaxLocalVersion uint64 + MaxLocalVersion int64 } type Option struct { @@ -117,7 +116,7 @@ type Option struct { type CloseMessage struct { Reason string // max:1024 - Code uint32 + Code int32 } type EmptyMessage struct{} diff --git a/message_xdr.go b/message_xdr.go index fa7b5802b..54d030da1 100644 --- a/message_xdr.go +++ b/message_xdr.go @@ -168,8 +168,8 @@ struct FileInfo { string Name<8192>; unsigned int Flags; hyper Modified; - unsigned hyper Version; - unsigned hyper LocalVersion; + hyper Version; + hyper LocalVersion; BlockInfo Blocks<>; } @@ -206,8 +206,8 @@ func (o FileInfo) encodeXDR(xw *xdr.Writer) (int, error) { xw.WriteString(o.Name) xw.WriteUint32(o.Flags) xw.WriteUint64(uint64(o.Modified)) - xw.WriteUint64(o.Version) - xw.WriteUint64(o.LocalVersion) + xw.WriteUint64(uint64(o.Version)) + xw.WriteUint64(uint64(o.LocalVersion)) xw.WriteUint32(uint32(len(o.Blocks))) for i := range o.Blocks { _, err := o.Blocks[i].encodeXDR(xw) @@ -233,8 +233,8 @@ func (o *FileInfo) decodeXDR(xr *xdr.Reader) error { o.Name = xr.ReadStringMax(8192) o.Flags = xr.ReadUint32() o.Modified = int64(xr.ReadUint64()) - o.Version = xr.ReadUint64() - o.LocalVersion = xr.ReadUint64() + o.Version = int64(xr.ReadUint64()) + o.LocalVersion = int64(xr.ReadUint64()) _BlocksSize := int(xr.ReadUint32()) o.Blocks = make([]BlockInfo, _BlocksSize) for i := range o.Blocks { @@ -261,7 +261,7 @@ BlockInfo Structure: struct BlockInfo { - unsigned int Size; + int Size; opaque Hash<64>; } @@ -292,7 +292,7 @@ func (o BlockInfo) AppendXDR(bs []byte) ([]byte, error) { } func (o BlockInfo) encodeXDR(xw *xdr.Writer) (int, error) { - xw.WriteUint32(o.Size) + xw.WriteUint32(uint32(o.Size)) if l := len(o.Hash); l > 64 { return xw.Tot(), xdr.ElementSizeExceeded("Hash", l, 64) } @@ -312,7 +312,7 @@ func (o *BlockInfo) UnmarshalXDR(bs []byte) error { } func (o *BlockInfo) decodeXDR(xr *xdr.Reader) error { - o.Size = xr.ReadUint32() + o.Size = int32(xr.ReadUint32()) o.Hash = xr.ReadBytesMax(64) return xr.Error() } @@ -361,8 +361,8 @@ RequestMessage Structure: struct RequestMessage { string Folder<64>; string Name<8192>; - unsigned hyper Offset; - unsigned int Size; + hyper Offset; + int Size; opaque Hash<64>; unsigned int Flags; Option Options<64>; @@ -403,8 +403,8 @@ func (o RequestMessage) encodeXDR(xw *xdr.Writer) (int, error) { return xw.Tot(), xdr.ElementSizeExceeded("Name", l, 8192) } xw.WriteString(o.Name) - xw.WriteUint64(o.Offset) - xw.WriteUint32(o.Size) + xw.WriteUint64(uint64(o.Offset)) + xw.WriteUint32(uint32(o.Size)) if l := len(o.Hash); l > 64 { return xw.Tot(), xdr.ElementSizeExceeded("Hash", l, 64) } @@ -437,8 +437,8 @@ func (o *RequestMessage) UnmarshalXDR(bs []byte) error { func (o *RequestMessage) decodeXDR(xr *xdr.Reader) error { o.Folder = xr.ReadStringMax(64) o.Name = xr.ReadStringMax(8192) - o.Offset = xr.ReadUint64() - o.Size = xr.ReadUint32() + o.Offset = int64(xr.ReadUint64()) + o.Size = int32(xr.ReadUint32()) o.Hash = xr.ReadBytesMax(64) o.Flags = xr.ReadUint32() _OptionsSize := int(xr.ReadUint32()) @@ -471,7 +471,7 @@ ResponseMessage Structure: struct ResponseMessage { opaque Data<>; - unsigned int Error; + int Error; } */ @@ -502,7 +502,7 @@ func (o ResponseMessage) AppendXDR(bs []byte) ([]byte, error) { func (o ResponseMessage) encodeXDR(xw *xdr.Writer) (int, error) { xw.WriteBytes(o.Data) - xw.WriteUint32(o.Error) + xw.WriteUint32(uint32(o.Error)) return xw.Tot(), xw.Error() } @@ -519,7 +519,7 @@ func (o *ResponseMessage) UnmarshalXDR(bs []byte) error { func (o *ResponseMessage) decodeXDR(xr *xdr.Reader) error { o.Data = xr.ReadBytes() - o.Error = xr.ReadUint32() + o.Error = int32(xr.ReadUint32()) return xr.Error() } @@ -766,7 +766,7 @@ Device Structure: struct Device { opaque ID<32>; unsigned int Flags; - unsigned hyper MaxLocalVersion; + hyper MaxLocalVersion; } */ @@ -801,7 +801,7 @@ func (o Device) encodeXDR(xw *xdr.Writer) (int, error) { } xw.WriteBytes(o.ID) xw.WriteUint32(o.Flags) - xw.WriteUint64(o.MaxLocalVersion) + xw.WriteUint64(uint64(o.MaxLocalVersion)) return xw.Tot(), xw.Error() } @@ -819,7 +819,7 @@ func (o *Device) UnmarshalXDR(bs []byte) error { func (o *Device) decodeXDR(xr *xdr.Reader) error { o.ID = xr.ReadBytesMax(32) o.Flags = xr.ReadUint32() - o.MaxLocalVersion = xr.ReadUint64() + o.MaxLocalVersion = int64(xr.ReadUint64()) return xr.Error() } @@ -923,7 +923,7 @@ CloseMessage Structure: struct CloseMessage { string Reason<1024>; - unsigned int Code; + int Code; } */ @@ -957,7 +957,7 @@ func (o CloseMessage) encodeXDR(xw *xdr.Writer) (int, error) { return xw.Tot(), xdr.ElementSizeExceeded("Reason", l, 1024) } xw.WriteString(o.Reason) - xw.WriteUint32(o.Code) + xw.WriteUint32(uint32(o.Code)) return xw.Tot(), xw.Error() } @@ -974,7 +974,7 @@ func (o *CloseMessage) UnmarshalXDR(bs []byte) error { func (o *CloseMessage) decodeXDR(xr *xdr.Reader) error { o.Reason = xr.ReadStringMax(1024) - o.Code = xr.ReadUint32() + o.Code = int32(xr.ReadUint32()) return xr.Error() } diff --git a/protocol.go b/protocol.go index 7cf59af1f..eeaf0f46b 100644 --- a/protocol.go +++ b/protocol.go @@ -222,8 +222,8 @@ func (c *rawConnection) Request(folder string, name string, offset int64, size i ok := c.send(id, messageTypeRequest, RequestMessage{ Folder: folder, Name: name, - Offset: uint64(offset), - Size: uint32(size), + Offset: offset, + Size: int32(size), }) if !ok { return nil, ErrClosed @@ -706,8 +706,8 @@ func (c *rawConnection) pingerLoop() { type Statistics struct { At time.Time - InBytesTotal uint64 - OutBytesTotal uint64 + InBytesTotal int64 + OutBytesTotal int64 } func (c *rawConnection) Statistics() Statistics { From aba915037fd1b129c258a26e45fab5e8f2c4e7bf Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Fri, 23 Jan 2015 22:25:34 +0000 Subject: [PATCH 31/94] Add FlagsAll bit mask --- protocol.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/protocol.go b/protocol.go index eeaf0f46b..7f1dd4598 100644 --- a/protocol.go +++ b/protocol.go @@ -43,6 +43,8 @@ const ( FlagSymlink = 1 << 16 FlagSymlinkMissingTarget = 1 << 17 + FlagsAll = (1 << iota) - 1 + SymlinkTypeMask = FlagDirectory | FlagSymlinkMissingTarget ) From 35f0e355bf93b3d3dd5737f908a01c31465e18b1 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Thu, 12 Feb 2015 21:59:33 +0000 Subject: [PATCH 32/94] We are not using iota --- protocol.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/protocol.go b/protocol.go index 7f1dd4598..f5c34dbff 100644 --- a/protocol.go +++ b/protocol.go @@ -43,7 +43,7 @@ const ( FlagSymlink = 1 << 16 FlagSymlinkMissingTarget = 1 << 17 - FlagsAll = (1 << iota) - 1 + FlagsAll = (1 << 18) - 1 SymlinkTypeMask = FlagDirectory | FlagSymlinkMissingTarget ) From a10c621e3340a9164221edef9ae27a8363ad28e3 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Sun, 8 Mar 2015 16:55:01 +0000 Subject: [PATCH 33/94] Remove 64 folder limit --- message.go | 4 ++-- message_xdr.go | 15 +++------------ 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/message.go b/message.go index dbaf526ac..9959fce30 100644 --- a/message.go +++ b/message.go @@ -7,7 +7,7 @@ package protocol import "fmt" type IndexMessage struct { - Folder string // max:64 + Folder string Files []FileInfo Flags uint32 Options []Option // max:64 @@ -85,7 +85,7 @@ type ResponseMessage struct { type ClusterConfigMessage struct { ClientName string // max:64 ClientVersion string // max:64 - Folders []Folder // max:64 + Folders []Folder Options []Option // max:64 } diff --git a/message_xdr.go b/message_xdr.go index 54d030da1..9e18f87b3 100644 --- a/message_xdr.go +++ b/message_xdr.go @@ -41,7 +41,7 @@ IndexMessage Structure: struct IndexMessage { - string Folder<64>; + string Folder<>; FileInfo Files<>; unsigned int Flags; Option Options<64>; @@ -74,9 +74,6 @@ func (o IndexMessage) AppendXDR(bs []byte) ([]byte, error) { } func (o IndexMessage) encodeXDR(xw *xdr.Writer) (int, error) { - if l := len(o.Folder); l > 64 { - return xw.Tot(), xdr.ElementSizeExceeded("Folder", l, 64) - } xw.WriteString(o.Folder) xw.WriteUint32(uint32(len(o.Files))) for i := range o.Files { @@ -111,7 +108,7 @@ func (o *IndexMessage) UnmarshalXDR(bs []byte) error { } func (o *IndexMessage) decodeXDR(xr *xdr.Reader) error { - o.Folder = xr.ReadStringMax(64) + o.Folder = xr.ReadString() _FilesSize := int(xr.ReadUint32()) o.Files = make([]FileInfo, _FilesSize) for i := range o.Files { @@ -559,7 +556,7 @@ ClusterConfigMessage Structure: struct ClusterConfigMessage { string ClientName<64>; string ClientVersion<64>; - Folder Folders<64>; + Folder Folders<>; Option Options<64>; } @@ -598,9 +595,6 @@ func (o ClusterConfigMessage) encodeXDR(xw *xdr.Writer) (int, error) { return xw.Tot(), xdr.ElementSizeExceeded("ClientVersion", l, 64) } xw.WriteString(o.ClientVersion) - if l := len(o.Folders); l > 64 { - return xw.Tot(), xdr.ElementSizeExceeded("Folders", l, 64) - } xw.WriteUint32(uint32(len(o.Folders))) for i := range o.Folders { _, err := o.Folders[i].encodeXDR(xw) @@ -636,9 +630,6 @@ func (o *ClusterConfigMessage) decodeXDR(xr *xdr.Reader) error { o.ClientName = xr.ReadStringMax(64) o.ClientVersion = xr.ReadStringMax(64) _FoldersSize := int(xr.ReadUint32()) - if _FoldersSize > 64 { - return xdr.ElementSizeExceeded("Folders", _FoldersSize, 64) - } o.Folders = make([]Folder, _FoldersSize) for i := range o.Folders { (&o.Folders[i]).decodeXDR(xr) From cd0cce4195105cefab80fce84664188b614032e8 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Mon, 9 Mar 2015 21:21:20 +0100 Subject: [PATCH 34/94] gofmt --- message.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/message.go b/message.go index 9959fce30..3b91161bc 100644 --- a/message.go +++ b/message.go @@ -83,8 +83,8 @@ type ResponseMessage struct { } type ClusterConfigMessage struct { - ClientName string // max:64 - ClientVersion string // max:64 + ClientName string // max:64 + ClientVersion string // max:64 Folders []Folder Options []Option // max:64 } From 108b4e2e104610bdf416f2f156f35ee769276caf Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Mon, 23 Feb 2015 09:30:47 +0100 Subject: [PATCH 35/94] Add more fine grained compression control --- compression.go | 53 +++++++++++++++++++++++++++++++++++++++++++++ compression_test.go | 51 +++++++++++++++++++++++++++++++++++++++++++ protocol.go | 38 +++++++++++++++++--------------- protocol_test.go | 20 ++++++++--------- 4 files changed, 135 insertions(+), 27 deletions(-) create mode 100644 compression.go create mode 100644 compression_test.go diff --git a/compression.go b/compression.go new file mode 100644 index 000000000..9e17213b6 --- /dev/null +++ b/compression.go @@ -0,0 +1,53 @@ +// Copyright (C) 2015 The Protocol Authors. + +package protocol + +import "fmt" + +type Compression int + +const ( + CompressMetadata Compression = iota // zero value is the default, default should be "metadata" + CompressNever + CompressAlways + + compressionThreshold = 128 // don't bother compressing messages smaller than this many bytes +) + +var compressionMarshal = map[Compression]string{ + CompressNever: "never", + CompressMetadata: "metadata", + CompressAlways: "always", +} + +var compressionUnmarshal = map[string]Compression{ + // Legacy + "false": CompressNever, + "true": CompressMetadata, + + // Current + "never": CompressNever, + "metadata": CompressMetadata, + "always": CompressAlways, +} + +func (c Compression) String() string { + s, ok := compressionMarshal[c] + if !ok { + return fmt.Sprintf("unknown:%d", c) + } + return s +} + +func (c Compression) GoString() string { + return fmt.Sprintf("%q", c.String()) +} + +func (c Compression) MarshalText() ([]byte, error) { + return []byte(compressionMarshal[c]), nil +} + +func (c *Compression) UnmarshalText(bs []byte) error { + *c = compressionUnmarshal[string(bs)] + return nil +} diff --git a/compression_test.go b/compression_test.go new file mode 100644 index 000000000..932297c32 --- /dev/null +++ b/compression_test.go @@ -0,0 +1,51 @@ +// Copyright (C) 2015 The Protocol Authors. + +package protocol + +import "testing" + +func TestCompressionMarshal(t *testing.T) { + uTestcases := []struct { + s string + c Compression + }{ + {"true", CompressMetadata}, + {"false", CompressNever}, + {"never", CompressNever}, + {"metadata", CompressMetadata}, + {"filedata", CompressFiledata}, + {"always", CompressAlways}, + {"whatever", CompressNever}, + } + + mTestcases := []struct { + s string + c Compression + }{ + {"never", CompressNever}, + {"metadata", CompressMetadata}, + {"filedata", CompressFiledata}, + {"always", CompressAlways}, + } + + var c Compression + for _, tc := range uTestcases { + err := c.UnmarshalText([]byte(tc.s)) + if err != nil { + t.Error(err) + } + if c != tc.c { + t.Errorf("%s unmarshalled to %d, not %d", tc.s, c, tc.c) + } + } + + for _, tc := range mTestcases { + bs, err := tc.c.MarshalText() + if err != nil { + t.Error(err) + } + if s := string(bs); s != tc.s { + t.Errorf("%d marshalled to %q, not %q", tc.c, s, tc.s) + } + } +} diff --git a/protocol.go b/protocol.go index f5c34dbff..e7b6fe275 100644 --- a/protocol.go +++ b/protocol.go @@ -106,7 +106,7 @@ type rawConnection struct { closed chan struct{} once sync.Once - compressionThreshold int // compress messages larger than this many bytes + compression Compression rdbuf0 []byte // used & reused by readMessage rdbuf1 []byte // used & reused by readMessage @@ -135,25 +135,21 @@ const ( pingIdleTime = 60 * time.Second ) -func NewConnection(deviceID DeviceID, reader io.Reader, writer io.Writer, receiver Model, name string, compress bool) Connection { +func NewConnection(deviceID DeviceID, reader io.Reader, writer io.Writer, receiver Model, name string, compress Compression) Connection { cr := &countingReader{Reader: reader} cw := &countingWriter{Writer: writer} - compThres := 1<<31 - 1 // compression disabled - if compress { - compThres = 128 // compress messages that are 128 bytes long or larger - } c := rawConnection{ - id: deviceID, - name: name, - receiver: nativeModel{receiver}, - state: stateInitial, - cr: cr, - cw: cw, - outbox: make(chan hdrMsg), - nextID: make(chan int), - closed: make(chan struct{}), - compressionThreshold: compThres, + id: deviceID, + name: name, + receiver: nativeModel{receiver}, + state: stateInitial, + cr: cr, + cw: cw, + outbox: make(chan hdrMsg), + nextID: make(chan int), + closed: make(chan struct{}), + compression: compress, } go c.readerLoop() @@ -571,7 +567,15 @@ func (c *rawConnection) writerLoop() { return } - if len(uncBuf) >= c.compressionThreshold { + compress := false + switch c.compression { + case CompressAlways: + compress = true + case CompressMetadata: + compress = hm.hdr.msgType != messageTypeResponse + } + + if compress && len(uncBuf) >= compressionThreshold { // Use compression for large messages hm.hdr.compression = true diff --git a/protocol_test.go b/protocol_test.go index 1ccb4525f..c1048cdcf 100644 --- a/protocol_test.go +++ b/protocol_test.go @@ -67,8 +67,8 @@ func TestPing(t *testing.T) { ar, aw := io.Pipe() br, bw := io.Pipe() - c0 := NewConnection(c0ID, ar, bw, nil, "name", true).(wireFormatConnection).next.(*rawConnection) - c1 := NewConnection(c1ID, br, aw, nil, "name", true).(wireFormatConnection).next.(*rawConnection) + c0 := NewConnection(c0ID, ar, bw, nil, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) + c1 := NewConnection(c1ID, br, aw, nil, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) if ok := c0.ping(); !ok { t.Error("c0 ping failed") @@ -91,8 +91,8 @@ func TestPingErr(t *testing.T) { eaw := &ErrPipe{PipeWriter: *aw, max: i, err: e} ebw := &ErrPipe{PipeWriter: *bw, max: j, err: e} - c0 := NewConnection(c0ID, ar, ebw, m0, "name", true).(wireFormatConnection).next.(*rawConnection) - NewConnection(c1ID, br, eaw, m1, "name", true) + c0 := NewConnection(c0ID, ar, ebw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) + NewConnection(c1ID, br, eaw, m1, "name", CompressAlways) res := c0.ping() if (i < 8 || j < 8) && res { @@ -167,8 +167,8 @@ func TestVersionErr(t *testing.T) { ar, aw := io.Pipe() br, bw := io.Pipe() - c0 := NewConnection(c0ID, ar, bw, m0, "name", true).(wireFormatConnection).next.(*rawConnection) - NewConnection(c1ID, br, aw, m1, "name", true) + c0 := NewConnection(c0ID, ar, bw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) + NewConnection(c1ID, br, aw, m1, "name", CompressAlways) w := xdr.NewWriter(c0.cw) w.WriteUint32(encodeHeader(header{ @@ -190,8 +190,8 @@ func TestTypeErr(t *testing.T) { ar, aw := io.Pipe() br, bw := io.Pipe() - c0 := NewConnection(c0ID, ar, bw, m0, "name", true).(wireFormatConnection).next.(*rawConnection) - NewConnection(c1ID, br, aw, m1, "name", true) + c0 := NewConnection(c0ID, ar, bw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) + NewConnection(c1ID, br, aw, m1, "name", CompressAlways) w := xdr.NewWriter(c0.cw) w.WriteUint32(encodeHeader(header{ @@ -213,8 +213,8 @@ func TestClose(t *testing.T) { ar, aw := io.Pipe() br, bw := io.Pipe() - c0 := NewConnection(c0ID, ar, bw, m0, "name", true).(wireFormatConnection).next.(*rawConnection) - NewConnection(c1ID, br, aw, m1, "name", true) + c0 := NewConnection(c0ID, ar, bw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) + NewConnection(c1ID, br, aw, m1, "name", CompressAlways) c0.close(nil) From 1a4398cc55c8fe82a964097eaf59f2475b020a49 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Wed, 11 Mar 2015 21:10:44 +0100 Subject: [PATCH 36/94] Tests should actually pass --- compression_test.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/compression_test.go b/compression_test.go index 932297c32..90312344c 100644 --- a/compression_test.go +++ b/compression_test.go @@ -13,9 +13,8 @@ func TestCompressionMarshal(t *testing.T) { {"false", CompressNever}, {"never", CompressNever}, {"metadata", CompressMetadata}, - {"filedata", CompressFiledata}, {"always", CompressAlways}, - {"whatever", CompressNever}, + {"whatever", CompressMetadata}, } mTestcases := []struct { @@ -24,7 +23,6 @@ func TestCompressionMarshal(t *testing.T) { }{ {"never", CompressNever}, {"metadata", CompressMetadata}, - {"filedata", CompressFiledata}, {"always", CompressAlways}, } From d2ec40bb67846f34d3c1e59714351127a2e869e9 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Wed, 25 Mar 2015 21:20:04 +0100 Subject: [PATCH 37/94] Add flags and options for future extensibility --- message.go | 5 +++- message_xdr.go | 65 ++++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 64 insertions(+), 6 deletions(-) diff --git a/message.go b/message.go index 3b91161bc..f6ed9e1a8 100644 --- a/message.go +++ b/message.go @@ -101,12 +101,15 @@ func (o *ClusterConfigMessage) GetOption(key string) string { type Folder struct { ID string // max:64 Devices []Device + Flags uint32 + Options []Option // max:64 } type Device struct { ID []byte // max:32 - Flags uint32 MaxLocalVersion int64 + Flags uint32 + Options []Option // max:64 } type Option struct { diff --git a/message_xdr.go b/message_xdr.go index 9e18f87b3..243aea879 100644 --- a/message_xdr.go +++ b/message_xdr.go @@ -664,11 +664,21 @@ Folder Structure: \ Zero or more Device Structures \ / / +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Flags | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Number of Options | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Zero or more Option Structures \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ struct Folder { string ID<64>; Device Devices<>; + unsigned int Flags; + Option Options<64>; } */ @@ -709,6 +719,17 @@ func (o Folder) encodeXDR(xw *xdr.Writer) (int, error) { return xw.Tot(), err } } + xw.WriteUint32(o.Flags) + if l := len(o.Options); l > 64 { + return xw.Tot(), xdr.ElementSizeExceeded("Options", l, 64) + } + xw.WriteUint32(uint32(len(o.Options))) + for i := range o.Options { + _, err := o.Options[i].encodeXDR(xw) + if err != nil { + return xw.Tot(), err + } + } return xw.Tot(), xw.Error() } @@ -730,6 +751,15 @@ func (o *Folder) decodeXDR(xr *xdr.Reader) error { for i := range o.Devices { (&o.Devices[i]).decodeXDR(xr) } + o.Flags = xr.ReadUint32() + _OptionsSize := int(xr.ReadUint32()) + if _OptionsSize > 64 { + return xdr.ElementSizeExceeded("Options", _OptionsSize, 64) + } + o.Options = make([]Option, _OptionsSize) + for i := range o.Options { + (&o.Options[i]).decodeXDR(xr) + } return xr.Error() } @@ -746,18 +776,25 @@ Device Structure: \ ID (variable length) \ / / +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Flags | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ | | + Max Local Version (64 bits) + | | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Flags | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Number of Options | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Zero or more Option Structures \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ struct Device { opaque ID<32>; - unsigned int Flags; hyper MaxLocalVersion; + unsigned int Flags; + Option Options<64>; } */ @@ -791,8 +828,18 @@ func (o Device) encodeXDR(xw *xdr.Writer) (int, error) { return xw.Tot(), xdr.ElementSizeExceeded("ID", l, 32) } xw.WriteBytes(o.ID) - xw.WriteUint32(o.Flags) xw.WriteUint64(uint64(o.MaxLocalVersion)) + xw.WriteUint32(o.Flags) + if l := len(o.Options); l > 64 { + return xw.Tot(), xdr.ElementSizeExceeded("Options", l, 64) + } + xw.WriteUint32(uint32(len(o.Options))) + for i := range o.Options { + _, err := o.Options[i].encodeXDR(xw) + if err != nil { + return xw.Tot(), err + } + } return xw.Tot(), xw.Error() } @@ -809,8 +856,16 @@ func (o *Device) UnmarshalXDR(bs []byte) error { func (o *Device) decodeXDR(xr *xdr.Reader) error { o.ID = xr.ReadBytesMax(32) - o.Flags = xr.ReadUint32() o.MaxLocalVersion = int64(xr.ReadUint64()) + o.Flags = xr.ReadUint32() + _OptionsSize := int(xr.ReadUint32()) + if _OptionsSize > 64 { + return xdr.ElementSizeExceeded("Options", _OptionsSize, 64) + } + o.Options = make([]Option, _OptionsSize) + for i := range o.Options { + (&o.Options[i]).decodeXDR(xr) + } return xr.Error() } From f9132cae85dcda1caba2f4ba78996d348b00ac6c Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Fri, 20 Mar 2015 09:58:32 +0100 Subject: [PATCH 38/94] Implement version vectors --- deviceid.go | 6 + message.go | 4 +- message_xdr.go | 183 +++++++++++++++--------------- vector.go | 105 +++++++++++++++++ vector_compare.go | 89 +++++++++++++++ vector_compare_test.go | 249 +++++++++++++++++++++++++++++++++++++++++ vector_test.go | 122 ++++++++++++++++++++ vector_xdr.go | 38 +++++++ 8 files changed, 704 insertions(+), 92 deletions(-) create mode 100644 vector.go create mode 100644 vector_compare.go create mode 100644 vector_compare_test.go create mode 100644 vector_test.go create mode 100644 vector_xdr.go diff --git a/deviceid.go b/deviceid.go index f3b3c5a31..2e0334a6a 100644 --- a/deviceid.go +++ b/deviceid.go @@ -6,6 +6,7 @@ import ( "bytes" "crypto/sha256" "encoding/base32" + "encoding/binary" "errors" "fmt" "regexp" @@ -67,6 +68,11 @@ func (n DeviceID) Equals(other DeviceID) bool { return bytes.Compare(n[:], other[:]) == 0 } +// Short returns an integer representing bits 0-63 of the device ID. +func (n DeviceID) Short() uint64 { + return binary.BigEndian.Uint64(n[:]) +} + func (n *DeviceID) MarshalText() ([]byte, error) { return []byte(n.String()), nil } diff --git a/message.go b/message.go index f6ed9e1a8..91c331903 100644 --- a/message.go +++ b/message.go @@ -17,13 +17,13 @@ type FileInfo struct { Name string // max:8192 Flags uint32 Modified int64 - Version int64 + Version Vector LocalVersion int64 Blocks []BlockInfo } func (f FileInfo) String() string { - return fmt.Sprintf("File{Name:%q, Flags:0%o, Modified:%d, Version:%d, Size:%d, Blocks:%v}", + return fmt.Sprintf("File{Name:%q, Flags:0%o, Modified:%d, Version:%v, Size:%d, Blocks:%v}", f.Name, f.Flags, f.Modified, f.Version, f.Size(), f.Blocks) } diff --git a/message_xdr.go b/message_xdr.go index 243aea879..95d72eb1a 100644 --- a/message_xdr.go +++ b/message_xdr.go @@ -51,7 +51,7 @@ struct IndexMessage { func (o IndexMessage) EncodeXDR(w io.Writer) (int, error) { var xw = xdr.NewWriter(w) - return o.encodeXDR(xw) + return o.EncodeXDRInto(xw) } func (o IndexMessage) MarshalXDR() ([]byte, error) { @@ -69,15 +69,15 @@ func (o IndexMessage) MustMarshalXDR() []byte { func (o IndexMessage) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - _, err := o.encodeXDR(xw) + _, err := o.EncodeXDRInto(xw) return []byte(aw), err } -func (o IndexMessage) encodeXDR(xw *xdr.Writer) (int, error) { +func (o IndexMessage) EncodeXDRInto(xw *xdr.Writer) (int, error) { xw.WriteString(o.Folder) xw.WriteUint32(uint32(len(o.Files))) for i := range o.Files { - _, err := o.Files[i].encodeXDR(xw) + _, err := o.Files[i].EncodeXDRInto(xw) if err != nil { return xw.Tot(), err } @@ -88,7 +88,7 @@ func (o IndexMessage) encodeXDR(xw *xdr.Writer) (int, error) { } xw.WriteUint32(uint32(len(o.Options))) for i := range o.Options { - _, err := o.Options[i].encodeXDR(xw) + _, err := o.Options[i].EncodeXDRInto(xw) if err != nil { return xw.Tot(), err } @@ -98,21 +98,21 @@ func (o IndexMessage) encodeXDR(xw *xdr.Writer) (int, error) { func (o *IndexMessage) DecodeXDR(r io.Reader) error { xr := xdr.NewReader(r) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } func (o *IndexMessage) UnmarshalXDR(bs []byte) error { var br = bytes.NewReader(bs) var xr = xdr.NewReader(br) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } -func (o *IndexMessage) decodeXDR(xr *xdr.Reader) error { +func (o *IndexMessage) DecodeXDRFrom(xr *xdr.Reader) error { o.Folder = xr.ReadString() _FilesSize := int(xr.ReadUint32()) o.Files = make([]FileInfo, _FilesSize) for i := range o.Files { - (&o.Files[i]).decodeXDR(xr) + (&o.Files[i]).DecodeXDRFrom(xr) } o.Flags = xr.ReadUint32() _OptionsSize := int(xr.ReadUint32()) @@ -121,7 +121,7 @@ func (o *IndexMessage) decodeXDR(xr *xdr.Reader) error { } o.Options = make([]Option, _OptionsSize) for i := range o.Options { - (&o.Options[i]).decodeXDR(xr) + (&o.Options[i]).DecodeXDRFrom(xr) } return xr.Error() } @@ -145,9 +145,9 @@ FileInfo Structure: + Modified (64 bits) + | | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| | -+ Version (64 bits) + -| | +/ / +\ Vector Structure \ +/ / +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ | | + Local Version (64 bits) + @@ -165,7 +165,7 @@ struct FileInfo { string Name<8192>; unsigned int Flags; hyper Modified; - hyper Version; + Vector Version; hyper LocalVersion; BlockInfo Blocks<>; } @@ -174,7 +174,7 @@ struct FileInfo { func (o FileInfo) EncodeXDR(w io.Writer) (int, error) { var xw = xdr.NewWriter(w) - return o.encodeXDR(xw) + return o.EncodeXDRInto(xw) } func (o FileInfo) MarshalXDR() ([]byte, error) { @@ -192,22 +192,25 @@ func (o FileInfo) MustMarshalXDR() []byte { func (o FileInfo) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - _, err := o.encodeXDR(xw) + _, err := o.EncodeXDRInto(xw) return []byte(aw), err } -func (o FileInfo) encodeXDR(xw *xdr.Writer) (int, error) { +func (o FileInfo) EncodeXDRInto(xw *xdr.Writer) (int, error) { if l := len(o.Name); l > 8192 { return xw.Tot(), xdr.ElementSizeExceeded("Name", l, 8192) } xw.WriteString(o.Name) xw.WriteUint32(o.Flags) xw.WriteUint64(uint64(o.Modified)) - xw.WriteUint64(uint64(o.Version)) + _, err := o.Version.EncodeXDRInto(xw) + if err != nil { + return xw.Tot(), err + } xw.WriteUint64(uint64(o.LocalVersion)) xw.WriteUint32(uint32(len(o.Blocks))) for i := range o.Blocks { - _, err := o.Blocks[i].encodeXDR(xw) + _, err := o.Blocks[i].EncodeXDRInto(xw) if err != nil { return xw.Tot(), err } @@ -217,25 +220,25 @@ func (o FileInfo) encodeXDR(xw *xdr.Writer) (int, error) { func (o *FileInfo) DecodeXDR(r io.Reader) error { xr := xdr.NewReader(r) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } func (o *FileInfo) UnmarshalXDR(bs []byte) error { var br = bytes.NewReader(bs) var xr = xdr.NewReader(br) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } -func (o *FileInfo) decodeXDR(xr *xdr.Reader) error { +func (o *FileInfo) DecodeXDRFrom(xr *xdr.Reader) error { o.Name = xr.ReadStringMax(8192) o.Flags = xr.ReadUint32() o.Modified = int64(xr.ReadUint64()) - o.Version = int64(xr.ReadUint64()) + (&o.Version).DecodeXDRFrom(xr) o.LocalVersion = int64(xr.ReadUint64()) _BlocksSize := int(xr.ReadUint32()) o.Blocks = make([]BlockInfo, _BlocksSize) for i := range o.Blocks { - (&o.Blocks[i]).decodeXDR(xr) + (&o.Blocks[i]).DecodeXDRFrom(xr) } return xr.Error() } @@ -266,7 +269,7 @@ struct BlockInfo { func (o BlockInfo) EncodeXDR(w io.Writer) (int, error) { var xw = xdr.NewWriter(w) - return o.encodeXDR(xw) + return o.EncodeXDRInto(xw) } func (o BlockInfo) MarshalXDR() ([]byte, error) { @@ -284,11 +287,11 @@ func (o BlockInfo) MustMarshalXDR() []byte { func (o BlockInfo) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - _, err := o.encodeXDR(xw) + _, err := o.EncodeXDRInto(xw) return []byte(aw), err } -func (o BlockInfo) encodeXDR(xw *xdr.Writer) (int, error) { +func (o BlockInfo) EncodeXDRInto(xw *xdr.Writer) (int, error) { xw.WriteUint32(uint32(o.Size)) if l := len(o.Hash); l > 64 { return xw.Tot(), xdr.ElementSizeExceeded("Hash", l, 64) @@ -299,16 +302,16 @@ func (o BlockInfo) encodeXDR(xw *xdr.Writer) (int, error) { func (o *BlockInfo) DecodeXDR(r io.Reader) error { xr := xdr.NewReader(r) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } func (o *BlockInfo) UnmarshalXDR(bs []byte) error { var br = bytes.NewReader(bs) var xr = xdr.NewReader(br) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } -func (o *BlockInfo) decodeXDR(xr *xdr.Reader) error { +func (o *BlockInfo) DecodeXDRFrom(xr *xdr.Reader) error { o.Size = int32(xr.ReadUint32()) o.Hash = xr.ReadBytesMax(64) return xr.Error() @@ -369,7 +372,7 @@ struct RequestMessage { func (o RequestMessage) EncodeXDR(w io.Writer) (int, error) { var xw = xdr.NewWriter(w) - return o.encodeXDR(xw) + return o.EncodeXDRInto(xw) } func (o RequestMessage) MarshalXDR() ([]byte, error) { @@ -387,11 +390,11 @@ func (o RequestMessage) MustMarshalXDR() []byte { func (o RequestMessage) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - _, err := o.encodeXDR(xw) + _, err := o.EncodeXDRInto(xw) return []byte(aw), err } -func (o RequestMessage) encodeXDR(xw *xdr.Writer) (int, error) { +func (o RequestMessage) EncodeXDRInto(xw *xdr.Writer) (int, error) { if l := len(o.Folder); l > 64 { return xw.Tot(), xdr.ElementSizeExceeded("Folder", l, 64) } @@ -412,7 +415,7 @@ func (o RequestMessage) encodeXDR(xw *xdr.Writer) (int, error) { } xw.WriteUint32(uint32(len(o.Options))) for i := range o.Options { - _, err := o.Options[i].encodeXDR(xw) + _, err := o.Options[i].EncodeXDRInto(xw) if err != nil { return xw.Tot(), err } @@ -422,16 +425,16 @@ func (o RequestMessage) encodeXDR(xw *xdr.Writer) (int, error) { func (o *RequestMessage) DecodeXDR(r io.Reader) error { xr := xdr.NewReader(r) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } func (o *RequestMessage) UnmarshalXDR(bs []byte) error { var br = bytes.NewReader(bs) var xr = xdr.NewReader(br) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } -func (o *RequestMessage) decodeXDR(xr *xdr.Reader) error { +func (o *RequestMessage) DecodeXDRFrom(xr *xdr.Reader) error { o.Folder = xr.ReadStringMax(64) o.Name = xr.ReadStringMax(8192) o.Offset = int64(xr.ReadUint64()) @@ -444,7 +447,7 @@ func (o *RequestMessage) decodeXDR(xr *xdr.Reader) error { } o.Options = make([]Option, _OptionsSize) for i := range o.Options { - (&o.Options[i]).decodeXDR(xr) + (&o.Options[i]).DecodeXDRFrom(xr) } return xr.Error() } @@ -475,7 +478,7 @@ struct ResponseMessage { func (o ResponseMessage) EncodeXDR(w io.Writer) (int, error) { var xw = xdr.NewWriter(w) - return o.encodeXDR(xw) + return o.EncodeXDRInto(xw) } func (o ResponseMessage) MarshalXDR() ([]byte, error) { @@ -493,11 +496,11 @@ func (o ResponseMessage) MustMarshalXDR() []byte { func (o ResponseMessage) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - _, err := o.encodeXDR(xw) + _, err := o.EncodeXDRInto(xw) return []byte(aw), err } -func (o ResponseMessage) encodeXDR(xw *xdr.Writer) (int, error) { +func (o ResponseMessage) EncodeXDRInto(xw *xdr.Writer) (int, error) { xw.WriteBytes(o.Data) xw.WriteUint32(uint32(o.Error)) return xw.Tot(), xw.Error() @@ -505,16 +508,16 @@ func (o ResponseMessage) encodeXDR(xw *xdr.Writer) (int, error) { func (o *ResponseMessage) DecodeXDR(r io.Reader) error { xr := xdr.NewReader(r) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } func (o *ResponseMessage) UnmarshalXDR(bs []byte) error { var br = bytes.NewReader(bs) var xr = xdr.NewReader(br) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } -func (o *ResponseMessage) decodeXDR(xr *xdr.Reader) error { +func (o *ResponseMessage) DecodeXDRFrom(xr *xdr.Reader) error { o.Data = xr.ReadBytes() o.Error = int32(xr.ReadUint32()) return xr.Error() @@ -564,7 +567,7 @@ struct ClusterConfigMessage { func (o ClusterConfigMessage) EncodeXDR(w io.Writer) (int, error) { var xw = xdr.NewWriter(w) - return o.encodeXDR(xw) + return o.EncodeXDRInto(xw) } func (o ClusterConfigMessage) MarshalXDR() ([]byte, error) { @@ -582,11 +585,11 @@ func (o ClusterConfigMessage) MustMarshalXDR() []byte { func (o ClusterConfigMessage) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - _, err := o.encodeXDR(xw) + _, err := o.EncodeXDRInto(xw) return []byte(aw), err } -func (o ClusterConfigMessage) encodeXDR(xw *xdr.Writer) (int, error) { +func (o ClusterConfigMessage) EncodeXDRInto(xw *xdr.Writer) (int, error) { if l := len(o.ClientName); l > 64 { return xw.Tot(), xdr.ElementSizeExceeded("ClientName", l, 64) } @@ -597,7 +600,7 @@ func (o ClusterConfigMessage) encodeXDR(xw *xdr.Writer) (int, error) { xw.WriteString(o.ClientVersion) xw.WriteUint32(uint32(len(o.Folders))) for i := range o.Folders { - _, err := o.Folders[i].encodeXDR(xw) + _, err := o.Folders[i].EncodeXDRInto(xw) if err != nil { return xw.Tot(), err } @@ -607,7 +610,7 @@ func (o ClusterConfigMessage) encodeXDR(xw *xdr.Writer) (int, error) { } xw.WriteUint32(uint32(len(o.Options))) for i := range o.Options { - _, err := o.Options[i].encodeXDR(xw) + _, err := o.Options[i].EncodeXDRInto(xw) if err != nil { return xw.Tot(), err } @@ -617,22 +620,22 @@ func (o ClusterConfigMessage) encodeXDR(xw *xdr.Writer) (int, error) { func (o *ClusterConfigMessage) DecodeXDR(r io.Reader) error { xr := xdr.NewReader(r) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } func (o *ClusterConfigMessage) UnmarshalXDR(bs []byte) error { var br = bytes.NewReader(bs) var xr = xdr.NewReader(br) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } -func (o *ClusterConfigMessage) decodeXDR(xr *xdr.Reader) error { +func (o *ClusterConfigMessage) DecodeXDRFrom(xr *xdr.Reader) error { o.ClientName = xr.ReadStringMax(64) o.ClientVersion = xr.ReadStringMax(64) _FoldersSize := int(xr.ReadUint32()) o.Folders = make([]Folder, _FoldersSize) for i := range o.Folders { - (&o.Folders[i]).decodeXDR(xr) + (&o.Folders[i]).DecodeXDRFrom(xr) } _OptionsSize := int(xr.ReadUint32()) if _OptionsSize > 64 { @@ -640,7 +643,7 @@ func (o *ClusterConfigMessage) decodeXDR(xr *xdr.Reader) error { } o.Options = make([]Option, _OptionsSize) for i := range o.Options { - (&o.Options[i]).decodeXDR(xr) + (&o.Options[i]).DecodeXDRFrom(xr) } return xr.Error() } @@ -685,7 +688,7 @@ struct Folder { func (o Folder) EncodeXDR(w io.Writer) (int, error) { var xw = xdr.NewWriter(w) - return o.encodeXDR(xw) + return o.EncodeXDRInto(xw) } func (o Folder) MarshalXDR() ([]byte, error) { @@ -703,18 +706,18 @@ func (o Folder) MustMarshalXDR() []byte { func (o Folder) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - _, err := o.encodeXDR(xw) + _, err := o.EncodeXDRInto(xw) return []byte(aw), err } -func (o Folder) encodeXDR(xw *xdr.Writer) (int, error) { +func (o Folder) EncodeXDRInto(xw *xdr.Writer) (int, error) { if l := len(o.ID); l > 64 { return xw.Tot(), xdr.ElementSizeExceeded("ID", l, 64) } xw.WriteString(o.ID) xw.WriteUint32(uint32(len(o.Devices))) for i := range o.Devices { - _, err := o.Devices[i].encodeXDR(xw) + _, err := o.Devices[i].EncodeXDRInto(xw) if err != nil { return xw.Tot(), err } @@ -725,7 +728,7 @@ func (o Folder) encodeXDR(xw *xdr.Writer) (int, error) { } xw.WriteUint32(uint32(len(o.Options))) for i := range o.Options { - _, err := o.Options[i].encodeXDR(xw) + _, err := o.Options[i].EncodeXDRInto(xw) if err != nil { return xw.Tot(), err } @@ -735,21 +738,21 @@ func (o Folder) encodeXDR(xw *xdr.Writer) (int, error) { func (o *Folder) DecodeXDR(r io.Reader) error { xr := xdr.NewReader(r) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } func (o *Folder) UnmarshalXDR(bs []byte) error { var br = bytes.NewReader(bs) var xr = xdr.NewReader(br) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } -func (o *Folder) decodeXDR(xr *xdr.Reader) error { +func (o *Folder) DecodeXDRFrom(xr *xdr.Reader) error { o.ID = xr.ReadStringMax(64) _DevicesSize := int(xr.ReadUint32()) o.Devices = make([]Device, _DevicesSize) for i := range o.Devices { - (&o.Devices[i]).decodeXDR(xr) + (&o.Devices[i]).DecodeXDRFrom(xr) } o.Flags = xr.ReadUint32() _OptionsSize := int(xr.ReadUint32()) @@ -758,7 +761,7 @@ func (o *Folder) decodeXDR(xr *xdr.Reader) error { } o.Options = make([]Option, _OptionsSize) for i := range o.Options { - (&o.Options[i]).decodeXDR(xr) + (&o.Options[i]).DecodeXDRFrom(xr) } return xr.Error() } @@ -801,7 +804,7 @@ struct Device { func (o Device) EncodeXDR(w io.Writer) (int, error) { var xw = xdr.NewWriter(w) - return o.encodeXDR(xw) + return o.EncodeXDRInto(xw) } func (o Device) MarshalXDR() ([]byte, error) { @@ -819,11 +822,11 @@ func (o Device) MustMarshalXDR() []byte { func (o Device) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - _, err := o.encodeXDR(xw) + _, err := o.EncodeXDRInto(xw) return []byte(aw), err } -func (o Device) encodeXDR(xw *xdr.Writer) (int, error) { +func (o Device) EncodeXDRInto(xw *xdr.Writer) (int, error) { if l := len(o.ID); l > 32 { return xw.Tot(), xdr.ElementSizeExceeded("ID", l, 32) } @@ -835,7 +838,7 @@ func (o Device) encodeXDR(xw *xdr.Writer) (int, error) { } xw.WriteUint32(uint32(len(o.Options))) for i := range o.Options { - _, err := o.Options[i].encodeXDR(xw) + _, err := o.Options[i].EncodeXDRInto(xw) if err != nil { return xw.Tot(), err } @@ -845,16 +848,16 @@ func (o Device) encodeXDR(xw *xdr.Writer) (int, error) { func (o *Device) DecodeXDR(r io.Reader) error { xr := xdr.NewReader(r) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } func (o *Device) UnmarshalXDR(bs []byte) error { var br = bytes.NewReader(bs) var xr = xdr.NewReader(br) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } -func (o *Device) decodeXDR(xr *xdr.Reader) error { +func (o *Device) DecodeXDRFrom(xr *xdr.Reader) error { o.ID = xr.ReadBytesMax(32) o.MaxLocalVersion = int64(xr.ReadUint64()) o.Flags = xr.ReadUint32() @@ -864,7 +867,7 @@ func (o *Device) decodeXDR(xr *xdr.Reader) error { } o.Options = make([]Option, _OptionsSize) for i := range o.Options { - (&o.Options[i]).decodeXDR(xr) + (&o.Options[i]).DecodeXDRFrom(xr) } return xr.Error() } @@ -899,7 +902,7 @@ struct Option { func (o Option) EncodeXDR(w io.Writer) (int, error) { var xw = xdr.NewWriter(w) - return o.encodeXDR(xw) + return o.EncodeXDRInto(xw) } func (o Option) MarshalXDR() ([]byte, error) { @@ -917,11 +920,11 @@ func (o Option) MustMarshalXDR() []byte { func (o Option) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - _, err := o.encodeXDR(xw) + _, err := o.EncodeXDRInto(xw) return []byte(aw), err } -func (o Option) encodeXDR(xw *xdr.Writer) (int, error) { +func (o Option) EncodeXDRInto(xw *xdr.Writer) (int, error) { if l := len(o.Key); l > 64 { return xw.Tot(), xdr.ElementSizeExceeded("Key", l, 64) } @@ -935,16 +938,16 @@ func (o Option) encodeXDR(xw *xdr.Writer) (int, error) { func (o *Option) DecodeXDR(r io.Reader) error { xr := xdr.NewReader(r) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } func (o *Option) UnmarshalXDR(bs []byte) error { var br = bytes.NewReader(bs) var xr = xdr.NewReader(br) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } -func (o *Option) decodeXDR(xr *xdr.Reader) error { +func (o *Option) DecodeXDRFrom(xr *xdr.Reader) error { o.Key = xr.ReadStringMax(64) o.Value = xr.ReadStringMax(1024) return xr.Error() @@ -976,7 +979,7 @@ struct CloseMessage { func (o CloseMessage) EncodeXDR(w io.Writer) (int, error) { var xw = xdr.NewWriter(w) - return o.encodeXDR(xw) + return o.EncodeXDRInto(xw) } func (o CloseMessage) MarshalXDR() ([]byte, error) { @@ -994,11 +997,11 @@ func (o CloseMessage) MustMarshalXDR() []byte { func (o CloseMessage) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - _, err := o.encodeXDR(xw) + _, err := o.EncodeXDRInto(xw) return []byte(aw), err } -func (o CloseMessage) encodeXDR(xw *xdr.Writer) (int, error) { +func (o CloseMessage) EncodeXDRInto(xw *xdr.Writer) (int, error) { if l := len(o.Reason); l > 1024 { return xw.Tot(), xdr.ElementSizeExceeded("Reason", l, 1024) } @@ -1009,16 +1012,16 @@ func (o CloseMessage) encodeXDR(xw *xdr.Writer) (int, error) { func (o *CloseMessage) DecodeXDR(r io.Reader) error { xr := xdr.NewReader(r) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } func (o *CloseMessage) UnmarshalXDR(bs []byte) error { var br = bytes.NewReader(bs) var xr = xdr.NewReader(br) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } -func (o *CloseMessage) decodeXDR(xr *xdr.Reader) error { +func (o *CloseMessage) DecodeXDRFrom(xr *xdr.Reader) error { o.Reason = xr.ReadStringMax(1024) o.Code = int32(xr.ReadUint32()) return xr.Error() @@ -1040,7 +1043,7 @@ struct EmptyMessage { func (o EmptyMessage) EncodeXDR(w io.Writer) (int, error) { var xw = xdr.NewWriter(w) - return o.encodeXDR(xw) + return o.EncodeXDRInto(xw) } func (o EmptyMessage) MarshalXDR() ([]byte, error) { @@ -1058,25 +1061,25 @@ func (o EmptyMessage) MustMarshalXDR() []byte { func (o EmptyMessage) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) - _, err := o.encodeXDR(xw) + _, err := o.EncodeXDRInto(xw) return []byte(aw), err } -func (o EmptyMessage) encodeXDR(xw *xdr.Writer) (int, error) { +func (o EmptyMessage) EncodeXDRInto(xw *xdr.Writer) (int, error) { return xw.Tot(), xw.Error() } func (o *EmptyMessage) DecodeXDR(r io.Reader) error { xr := xdr.NewReader(r) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } func (o *EmptyMessage) UnmarshalXDR(bs []byte) error { var br = bytes.NewReader(bs) var xr = xdr.NewReader(br) - return o.decodeXDR(xr) + return o.DecodeXDRFrom(xr) } -func (o *EmptyMessage) decodeXDR(xr *xdr.Reader) error { +func (o *EmptyMessage) DecodeXDRFrom(xr *xdr.Reader) error { return xr.Error() } diff --git a/vector.go b/vector.go new file mode 100644 index 000000000..048594522 --- /dev/null +++ b/vector.go @@ -0,0 +1,105 @@ +// Copyright (C) 2015 The Protocol Authors. + +package protocol + +// The Vector type represents a version vector. The zero value is a usable +// version vector. The vector has slice semantics and some operations on it +// are "append-like" in that they may return the same vector modified, or a +// new allocated Vector with the modified contents. +type Vector []Counter + +// Counter represents a single counter in the version vector. +type Counter struct { + ID uint64 + Value uint64 +} + +// Update returns a Vector with the index for the specific ID incremented by +// one. If it is possible, the vector v is updated and returned. If it is not, +// a copy will be created, updated and returned. +func (v Vector) Update(ID uint64) Vector { + for i := range v { + if v[i].ID == ID { + // Update an existing index + v[i].Value++ + return v + } else if v[i].ID > ID { + // Insert a new index + nv := make(Vector, len(v)+1) + copy(nv, v[:i]) + nv[i].ID = ID + nv[i].Value = 1 + copy(nv[i+1:], v[i:]) + return nv + } + } + // Append a new new index + return append(v, Counter{ID, 1}) +} + +// Merge returns the vector containing the maximum indexes from a and b. If it +// is possible, the vector a is updated and returned. If it is not, a copy +// will be created, updated and returned. +func (a Vector) Merge(b Vector) Vector { + var ai, bi int + for bi < len(b) { + if ai == len(a) { + // We've reach the end of a, all that remains are appends + return append(a, b[bi:]...) + } + + if a[ai].ID > b[bi].ID { + // The index from b should be inserted here + n := make(Vector, len(a)+1) + copy(n, a[:ai]) + n[ai] = b[bi] + copy(n[ai+1:], a[ai:]) + a = n + } + + if a[ai].ID == b[bi].ID { + if v := b[bi].Value; v > a[ai].Value { + a[ai].Value = v + } + } + + if bi < len(b) && a[ai].ID == b[bi].ID { + bi++ + } + ai++ + } + + return a +} + +// Copy returns an identical vector that is not shared with v. +func (v Vector) Copy() Vector { + nv := make(Vector, len(v)) + copy(nv, v) + return nv +} + +// Equal returns true when the two vectors are equivalent. +func (a Vector) Equal(b Vector) bool { + return a.Compare(b) == Equal +} + +// LesserEqual returns true when the two vectors are equivalent or a is Lesser +// than b. +func (a Vector) LesserEqual(b Vector) bool { + comp := a.Compare(b) + return comp == Lesser || comp == Equal +} + +// LesserEqual returns true when the two vectors are equivalent or a is Greater +// than b. +func (a Vector) GreaterEqual(b Vector) bool { + comp := a.Compare(b) + return comp == Greater || comp == Equal +} + +// Concurrent returns true when the two vectors are concrurrent. +func (a Vector) Concurrent(b Vector) bool { + comp := a.Compare(b) + return comp == ConcurrentGreater || comp == ConcurrentLesser +} diff --git a/vector_compare.go b/vector_compare.go new file mode 100644 index 000000000..9735ec9d1 --- /dev/null +++ b/vector_compare.go @@ -0,0 +1,89 @@ +// Copyright (C) 2015 The Protocol Authors. + +package protocol + +// Ordering represents the relationship between two Vectors. +type Ordering int + +const ( + Equal Ordering = iota + Greater + Lesser + ConcurrentLesser + ConcurrentGreater +) + +// There's really no such thing as "concurrent lesser" and "concurrent +// greater" in version vectors, just "concurrent". But it's useful to be able +// to get a strict ordering between versions for stable sorts and so on, so we +// return both variants. The convenience method Concurrent() can be used to +// check for either case. + +// Compare returns the Ordering that describes a's relation to b. +func (a Vector) Compare(b Vector) Ordering { + var ai, bi int // index into a and b + var av, bv Counter // value at current index + + result := Equal + + for ai < len(a) || bi < len(b) { + var aMissing, bMissing bool + + if ai < len(a) { + av = a[ai] + } else { + av = Counter{} + aMissing = true + } + + if bi < len(b) { + bv = b[bi] + } else { + bv = Counter{} + bMissing = true + } + + switch { + case av.ID == bv.ID: + // We have a counter value for each side + if av.Value > bv.Value { + if result == Lesser { + return ConcurrentLesser + } + result = Greater + } else if av.Value < bv.Value { + if result == Greater { + return ConcurrentGreater + } + result = Lesser + } + + case !aMissing && av.ID < bv.ID || bMissing: + // Value is missing on the b side + if av.Value > 0 { + if result == Lesser { + return ConcurrentLesser + } + result = Greater + } + + case !bMissing && bv.ID < av.ID || aMissing: + // Value is missing on the a side + if bv.Value > 0 { + if result == Greater { + return ConcurrentGreater + } + result = Lesser + } + } + + if ai < len(a) && (av.ID <= bv.ID || bMissing) { + ai++ + } + if bi < len(b) && (bv.ID <= av.ID || aMissing) { + bi++ + } + } + + return result +} diff --git a/vector_compare_test.go b/vector_compare_test.go new file mode 100644 index 000000000..78b6abe43 --- /dev/null +++ b/vector_compare_test.go @@ -0,0 +1,249 @@ +// Copyright (C) 2015 The Protocol Authors. + +package protocol + +import ( + "math" + "testing" +) + +func TestCompare(t *testing.T) { + testcases := []struct { + a, b Vector + r Ordering + }{ + // Empty vectors are identical + {Vector{}, Vector{}, Equal}, + {Vector{}, nil, Equal}, + {nil, Vector{}, Equal}, + {nil, Vector{Counter{42, 0}}, Equal}, + {Vector{}, Vector{Counter{42, 0}}, Equal}, + {Vector{Counter{42, 0}}, nil, Equal}, + {Vector{Counter{42, 0}}, Vector{}, Equal}, + + // Zero is the implied value for a missing Counter + { + Vector{Counter{42, 0}}, + Vector{Counter{77, 0}}, + Equal, + }, + + // Equal vectors are equal + { + Vector{Counter{42, 33}}, + Vector{Counter{42, 33}}, + Equal, + }, + { + Vector{Counter{42, 33}, Counter{77, 24}}, + Vector{Counter{42, 33}, Counter{77, 24}}, + Equal, + }, + + // These a-vectors are all greater than the b-vector + { + Vector{Counter{42, 1}}, + nil, + Greater, + }, + { + Vector{Counter{42, 1}}, + Vector{}, + Greater, + }, + { + Vector{Counter{0, 1}}, + Vector{Counter{0, 0}}, + Greater, + }, + { + Vector{Counter{42, 1}}, + Vector{Counter{42, 0}}, + Greater, + }, + { + Vector{Counter{math.MaxUint64, 1}}, + Vector{Counter{math.MaxUint64, 0}}, + Greater, + }, + { + Vector{Counter{0, math.MaxUint64}}, + Vector{Counter{0, 0}}, + Greater, + }, + { + Vector{Counter{42, math.MaxUint64}}, + Vector{Counter{42, 0}}, + Greater, + }, + { + Vector{Counter{math.MaxUint64, math.MaxUint64}}, + Vector{Counter{math.MaxUint64, 0}}, + Greater, + }, + { + Vector{Counter{0, math.MaxUint64}}, + Vector{Counter{0, math.MaxUint64 - 1}}, + Greater, + }, + { + Vector{Counter{42, math.MaxUint64}}, + Vector{Counter{42, math.MaxUint64 - 1}}, + Greater, + }, + { + Vector{Counter{math.MaxUint64, math.MaxUint64}}, + Vector{Counter{math.MaxUint64, math.MaxUint64 - 1}}, + Greater, + }, + { + Vector{Counter{42, 2}}, + Vector{Counter{42, 1}}, + Greater, + }, + { + Vector{Counter{22, 22}, Counter{42, 2}}, + Vector{Counter{22, 22}, Counter{42, 1}}, + Greater, + }, + { + Vector{Counter{42, 2}, Counter{77, 3}}, + Vector{Counter{42, 1}, Counter{77, 3}}, + Greater, + }, + { + Vector{Counter{22, 22}, Counter{42, 2}, Counter{77, 3}}, + Vector{Counter{22, 22}, Counter{42, 1}, Counter{77, 3}}, + Greater, + }, + { + Vector{Counter{22, 23}, Counter{42, 2}, Counter{77, 4}}, + Vector{Counter{22, 22}, Counter{42, 1}, Counter{77, 3}}, + Greater, + }, + + // These a-vectors are all lesser than the b-vector + {nil, Vector{Counter{42, 1}}, Lesser}, + {Vector{}, Vector{Counter{42, 1}}, Lesser}, + { + Vector{Counter{42, 0}}, + Vector{Counter{42, 1}}, + Lesser, + }, + { + Vector{Counter{42, 1}}, + Vector{Counter{42, 2}}, + Lesser, + }, + { + Vector{Counter{22, 22}, Counter{42, 1}}, + Vector{Counter{22, 22}, Counter{42, 2}}, + Lesser, + }, + { + Vector{Counter{42, 1}, Counter{77, 3}}, + Vector{Counter{42, 2}, Counter{77, 3}}, + Lesser, + }, + { + Vector{Counter{22, 22}, Counter{42, 1}, Counter{77, 3}}, + Vector{Counter{22, 22}, Counter{42, 2}, Counter{77, 3}}, + Lesser, + }, + { + Vector{Counter{22, 22}, Counter{42, 1}, Counter{77, 3}}, + Vector{Counter{22, 23}, Counter{42, 2}, Counter{77, 4}}, + Lesser, + }, + + // These are all in conflict + { + Vector{Counter{42, 2}}, + Vector{Counter{43, 1}}, + ConcurrentGreater, + }, + { + Vector{Counter{43, 1}}, + Vector{Counter{42, 2}}, + ConcurrentLesser, + }, + { + Vector{Counter{22, 23}, Counter{42, 1}}, + Vector{Counter{22, 22}, Counter{42, 2}}, + ConcurrentGreater, + }, + { + Vector{Counter{22, 21}, Counter{42, 2}}, + Vector{Counter{22, 22}, Counter{42, 1}}, + ConcurrentLesser, + }, + { + Vector{Counter{22, 21}, Counter{42, 2}, Counter{43, 1}}, + Vector{Counter{20, 1}, Counter{22, 22}, Counter{42, 1}}, + ConcurrentLesser, + }, + } + + for i, tc := range testcases { + // Test real Compare + if r := tc.a.Compare(tc.b); r != tc.r { + t.Errorf("%d: %+v.Compare(%+v) == %v (expected %v)", i, tc.a, tc.b, r, tc.r) + } + + // Test convenience functions + switch tc.r { + case Greater: + if tc.a.Equal(tc.b) { + t.Errorf("%+v == %+v", tc.a, tc.b) + } + if tc.a.Concurrent(tc.b) { + t.Errorf("%+v concurrent %+v", tc.a, tc.b) + } + if !tc.a.GreaterEqual(tc.b) { + t.Errorf("%+v not >= %+v", tc.a, tc.b) + } + if tc.a.LesserEqual(tc.b) { + t.Errorf("%+v <= %+v", tc.a, tc.b) + } + case Lesser: + if tc.a.Concurrent(tc.b) { + t.Errorf("%+v concurrent %+v", tc.a, tc.b) + } + if tc.a.Equal(tc.b) { + t.Errorf("%+v == %+v", tc.a, tc.b) + } + if tc.a.GreaterEqual(tc.b) { + t.Errorf("%+v >= %+v", tc.a, tc.b) + } + if !tc.a.LesserEqual(tc.b) { + t.Errorf("%+v not <= %+v", tc.a, tc.b) + } + case Equal: + if tc.a.Concurrent(tc.b) { + t.Errorf("%+v concurrent %+v", tc.a, tc.b) + } + if !tc.a.Equal(tc.b) { + t.Errorf("%+v not == %+v", tc.a, tc.b) + } + if !tc.a.GreaterEqual(tc.b) { + t.Errorf("%+v not <= %+v", tc.a, tc.b) + } + if !tc.a.LesserEqual(tc.b) { + t.Errorf("%+v not <= %+v", tc.a, tc.b) + } + case ConcurrentLesser, ConcurrentGreater: + if !tc.a.Concurrent(tc.b) { + t.Errorf("%+v not concurrent %+v", tc.a, tc.b) + } + if tc.a.Equal(tc.b) { + t.Errorf("%+v == %+v", tc.a, tc.b) + } + if tc.a.GreaterEqual(tc.b) { + t.Errorf("%+v >= %+v", tc.a, tc.b) + } + if tc.a.LesserEqual(tc.b) { + t.Errorf("%+v <= %+v", tc.a, tc.b) + } + } + } +} diff --git a/vector_test.go b/vector_test.go new file mode 100644 index 000000000..7815412c2 --- /dev/null +++ b/vector_test.go @@ -0,0 +1,122 @@ +// Copyright (C) 2015 The Protocol Authors. + +package protocol + +import "testing" + +func TestUpdate(t *testing.T) { + var v Vector + + // Append + + v = v.Update(42) + expected := Vector{Counter{42, 1}} + + if v.Compare(expected) != Equal { + t.Errorf("Update error, %+v != %+v", v, expected) + } + + // Insert at front + + v = v.Update(36) + expected = Vector{Counter{36, 1}, Counter{42, 1}} + + if v.Compare(expected) != Equal { + t.Errorf("Update error, %+v != %+v", v, expected) + } + + // Insert in moddle + + v = v.Update(37) + expected = Vector{Counter{36, 1}, Counter{37, 1}, Counter{42, 1}} + + if v.Compare(expected) != Equal { + t.Errorf("Update error, %+v != %+v", v, expected) + } + + // Update existing + + v = v.Update(37) + expected = Vector{Counter{36, 1}, Counter{37, 2}, Counter{42, 1}} + + if v.Compare(expected) != Equal { + t.Errorf("Update error, %+v != %+v", v, expected) + } +} + +func TestCopy(t *testing.T) { + v0 := Vector{Counter{42, 1}} + v1 := v0.Copy() + v1.Update(42) + if v0.Compare(v1) != Lesser { + t.Errorf("Copy error, %+v should be ancestor of %+v", v0, v1) + } +} + +func TestMerge(t *testing.T) { + testcases := []struct { + a, b, m Vector + }{ + // No-ops + { + Vector{}, + Vector{}, + Vector{}, + }, + { + Vector{Counter{22, 1}, Counter{42, 1}}, + Vector{Counter{22, 1}, Counter{42, 1}}, + Vector{Counter{22, 1}, Counter{42, 1}}, + }, + + // Appends + { + Vector{}, + Vector{Counter{22, 1}, Counter{42, 1}}, + Vector{Counter{22, 1}, Counter{42, 1}}, + }, + { + Vector{Counter{22, 1}}, + Vector{Counter{42, 1}}, + Vector{Counter{22, 1}, Counter{42, 1}}, + }, + { + Vector{Counter{22, 1}}, + Vector{Counter{22, 1}, Counter{42, 1}}, + Vector{Counter{22, 1}, Counter{42, 1}}, + }, + + // Insert + { + Vector{Counter{22, 1}, Counter{42, 1}}, + Vector{Counter{22, 1}, Counter{23, 2}, Counter{42, 1}}, + Vector{Counter{22, 1}, Counter{23, 2}, Counter{42, 1}}, + }, + { + Vector{Counter{42, 1}}, + Vector{Counter{22, 1}}, + Vector{Counter{22, 1}, Counter{42, 1}}, + }, + + // Update + { + Vector{Counter{22, 1}, Counter{42, 2}}, + Vector{Counter{22, 2}, Counter{42, 1}}, + Vector{Counter{22, 2}, Counter{42, 2}}, + }, + + // All of the above + { + Vector{Counter{10, 1}, Counter{20, 2}, Counter{30, 1}}, + Vector{Counter{5, 1}, Counter{10, 2}, Counter{15, 1}, Counter{20, 1}, Counter{25, 1}, Counter{35, 1}}, + Vector{Counter{5, 1}, Counter{10, 2}, Counter{15, 1}, Counter{20, 2}, Counter{25, 1}, Counter{30, 1}, Counter{35, 1}}, + }, + } + + for i, tc := range testcases { + if m := tc.a.Merge(tc.b); m.Compare(tc.m) != Equal { + t.Errorf("%d: %+v.Merge(%+v) == %+v (expected %+v)", i, tc.a, tc.b, m, tc.m) + } + } + +} diff --git a/vector_xdr.go b/vector_xdr.go new file mode 100644 index 000000000..a4b6b132b --- /dev/null +++ b/vector_xdr.go @@ -0,0 +1,38 @@ +// Copyright (C) 2015 The Protocol Authors. + +package protocol + +// This stuff is hacked up manually because genxdr doesn't support 'type +// Vector []Counter' declarations and it was tricky when I tried to add it... + +type xdrWriter interface { + WriteUint32(uint32) (int, error) + WriteUint64(uint64) (int, error) +} +type xdrReader interface { + ReadUint32() uint32 + ReadUint64() uint64 +} + +// EncodeXDRInto encodes the vector as an XDR object into the given XDR +// encoder. +func (v Vector) EncodeXDRInto(w xdrWriter) (int, error) { + w.WriteUint32(uint32(len(v))) + for i := range v { + w.WriteUint64(v[i].ID) + w.WriteUint64(v[i].Value) + } + return 4 + 16*len(v), nil +} + +// DecodeXDRFrom decodes the XDR objects from the given reader into itself. +func (v *Vector) DecodeXDRFrom(r xdrReader) error { + l := int(r.ReadUint32()) + n := make(Vector, l) + for i := range n { + n[i].ID = r.ReadUint64() + n[i].Value = r.ReadUint64() + } + *v = n + return nil +} From 1a59a5478f8c8bdb0c50043a2cfccaa597ee8374 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Sat, 24 Jan 2015 21:56:12 +0000 Subject: [PATCH 39/94] Expose hash, flags, options in Request --- common_test.go | 8 +++++++- nativemodel_darwin.go | 4 ++-- nativemodel_unix.go | 4 ++-- nativemodel_windows.go | 4 ++-- protocol.go | 19 +++++++++++-------- protocol_test.go | 2 +- wireformat.go | 4 ++-- 7 files changed, 27 insertions(+), 18 deletions(-) diff --git a/common_test.go b/common_test.go index f67fb4812..0f3795d5b 100644 --- a/common_test.go +++ b/common_test.go @@ -13,6 +13,9 @@ type TestModel struct { name string offset int64 size int + hash []byte + flags uint32 + options []Option closedCh chan bool } @@ -28,11 +31,14 @@ func (t *TestModel) Index(deviceID DeviceID, folder string, files []FileInfo) { func (t *TestModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) { } -func (t *TestModel) Request(deviceID DeviceID, folder, name string, offset int64, size int) ([]byte, error) { +func (t *TestModel) Request(deviceID DeviceID, folder, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) { t.folder = folder t.name = name t.offset = offset t.size = size + t.hash = hash + t.flags = flags + t.options = options return t.data, nil } diff --git a/nativemodel_darwin.go b/nativemodel_darwin.go index 5b4b9be67..6001af694 100644 --- a/nativemodel_darwin.go +++ b/nativemodel_darwin.go @@ -26,9 +26,9 @@ func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileI m.next.IndexUpdate(deviceID, folder, files) } -func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int) ([]byte, error) { +func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) { name = norm.NFD.String(name) - return m.next.Request(deviceID, folder, name, offset, size) + return m.next.Request(deviceID, folder, name, offset, size, hash, flags, options) } func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) { diff --git a/nativemodel_unix.go b/nativemodel_unix.go index 2fb1654c7..5f7b2e2c4 100644 --- a/nativemodel_unix.go +++ b/nativemodel_unix.go @@ -18,8 +18,8 @@ func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileI m.next.IndexUpdate(deviceID, folder, files) } -func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int) ([]byte, error) { - return m.next.Request(deviceID, folder, name, offset, size) +func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) { + return m.next.Request(deviceID, folder, name, offset, size, hash, flags, options) } func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) { diff --git a/nativemodel_windows.go b/nativemodel_windows.go index d4feea326..4859cbaab 100644 --- a/nativemodel_windows.go +++ b/nativemodel_windows.go @@ -56,9 +56,9 @@ func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileI m.next.IndexUpdate(deviceID, folder, files) } -func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int) ([]byte, error) { +func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) { name = filepath.FromSlash(name) - return m.next.Request(deviceID, folder, name, offset, size) + return m.next.Request(deviceID, folder, name, offset, size, hash, flags, options) } func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) { diff --git a/protocol.go b/protocol.go index e7b6fe275..7e2658775 100644 --- a/protocol.go +++ b/protocol.go @@ -70,7 +70,7 @@ type Model interface { // An index update was received from the peer device IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) // A request was made by the peer device - Request(deviceID DeviceID, folder string, name string, offset int64, size int) ([]byte, error) + Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) // A cluster configuration message was received ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) // The peer device closed the connection @@ -82,7 +82,7 @@ type Connection interface { Name() string Index(folder string, files []FileInfo) error IndexUpdate(folder string, files []FileInfo) error - Request(folder string, name string, offset int64, size int) ([]byte, error) + Request(folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) ClusterConfig(config ClusterConfigMessage) Statistics() Statistics } @@ -201,7 +201,7 @@ func (c *rawConnection) IndexUpdate(folder string, idx []FileInfo) error { } // Request returns the bytes for the specified block after fetching them from the connected peer. -func (c *rawConnection) Request(folder string, name string, offset int64, size int) ([]byte, error) { +func (c *rawConnection) Request(folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) { var id int select { case id = <-c.nextID: @@ -218,10 +218,13 @@ func (c *rawConnection) Request(folder string, name string, offset int64, size i c.awaitingMut.Unlock() ok := c.send(id, messageTypeRequest, RequestMessage{ - Folder: folder, - Name: name, - Offset: offset, - Size: int32(size), + Folder: folder, + Name: name, + Offset: offset, + Size: int32(size), + Hash: hash, + Flags: flags, + Options: options, }) if !ok { return nil, ErrClosed @@ -499,7 +502,7 @@ func filterIndexMessageFiles(fs []FileInfo) []FileInfo { } func (c *rawConnection) handleRequest(msgID int, req RequestMessage) { - data, _ := c.receiver.Request(c.id, req.Folder, req.Name, int64(req.Offset), int(req.Size)) + data, _ := c.receiver.Request(c.id, req.Folder, req.Name, int64(req.Offset), int(req.Size), req.Hash, req.Flags, req.Options) c.send(msgID, messageTypeResponse, ResponseMessage{ Data: data, diff --git a/protocol_test.go b/protocol_test.go index c1048cdcf..c660ba327 100644 --- a/protocol_test.go +++ b/protocol_test.go @@ -232,7 +232,7 @@ func TestClose(t *testing.T) { c0.Index("default", nil) c0.Index("default", nil) - if _, err := c0.Request("default", "foo", 0, 0); err == nil { + if _, err := c0.Request("default", "foo", 0, 0, nil, 0, nil); err == nil { t.Error("Request should return an error") } } diff --git a/wireformat.go b/wireformat.go index 4eab3d37e..23d347e1b 100644 --- a/wireformat.go +++ b/wireformat.go @@ -42,9 +42,9 @@ func (c wireFormatConnection) IndexUpdate(folder string, fs []FileInfo) error { return c.next.IndexUpdate(folder, myFs) } -func (c wireFormatConnection) Request(folder, name string, offset int64, size int) ([]byte, error) { +func (c wireFormatConnection) Request(folder, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) { name = norm.NFC.String(filepath.ToSlash(name)) - return c.next.Request(folder, name, offset, size) + return c.next.Request(folder, name, offset, size, hash, flags, options) } func (c wireFormatConnection) ClusterConfig(config ClusterConfigMessage) { From 1cb5875b200f07f0a768198c2425d59905957bae Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Wed, 4 Feb 2015 22:15:17 +0000 Subject: [PATCH 40/94] Expose flags, options in Index{,Update} --- common_test.go | 4 ++-- nativemodel_darwin.go | 8 ++++---- nativemodel_unix.go | 8 ++++---- nativemodel_windows.go | 8 ++++---- protocol.go | 32 ++++++++++++++++++-------------- protocol_test.go | 4 ++-- wireformat.go | 8 ++++---- 7 files changed, 38 insertions(+), 34 deletions(-) diff --git a/common_test.go b/common_test.go index 0f3795d5b..f46b6a8da 100644 --- a/common_test.go +++ b/common_test.go @@ -25,10 +25,10 @@ func newTestModel() *TestModel { } } -func (t *TestModel) Index(deviceID DeviceID, folder string, files []FileInfo) { +func (t *TestModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { } -func (t *TestModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) { +func (t *TestModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { } func (t *TestModel) Request(deviceID DeviceID, folder, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) { diff --git a/nativemodel_darwin.go b/nativemodel_darwin.go index 6001af694..502a71f23 100644 --- a/nativemodel_darwin.go +++ b/nativemodel_darwin.go @@ -12,18 +12,18 @@ type nativeModel struct { next Model } -func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo) { +func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { for i := range files { files[i].Name = norm.NFD.String(files[i].Name) } - m.next.Index(deviceID, folder, files) + m.next.Index(deviceID, folder, files, flags, options) } -func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) { +func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { for i := range files { files[i].Name = norm.NFD.String(files[i].Name) } - m.next.IndexUpdate(deviceID, folder, files) + m.next.IndexUpdate(deviceID, folder, files, flags, options) } func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) { diff --git a/nativemodel_unix.go b/nativemodel_unix.go index 5f7b2e2c4..21585e308 100644 --- a/nativemodel_unix.go +++ b/nativemodel_unix.go @@ -10,12 +10,12 @@ type nativeModel struct { next Model } -func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo) { - m.next.Index(deviceID, folder, files) +func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { + m.next.Index(deviceID, folder, files, flags, options) } -func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) { - m.next.IndexUpdate(deviceID, folder, files) +func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { + m.next.IndexUpdate(deviceID, folder, files, flags, options) } func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) { diff --git a/nativemodel_windows.go b/nativemodel_windows.go index 4859cbaab..951f5b7e6 100644 --- a/nativemodel_windows.go +++ b/nativemodel_windows.go @@ -24,7 +24,7 @@ type nativeModel struct { next Model } -func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo) { +func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { for i, f := range files { if strings.ContainsAny(f.Name, disallowedCharacters) { if f.IsDeleted() { @@ -37,10 +37,10 @@ func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo) { } files[i].Name = filepath.FromSlash(f.Name) } - m.next.Index(deviceID, folder, files) + m.next.Index(deviceID, folder, files, flags, options) } -func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) { +func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { for i, f := range files { if strings.ContainsAny(f.Name, disallowedCharacters) { if f.IsDeleted() { @@ -53,7 +53,7 @@ func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileI } files[i].Name = filepath.FromSlash(files[i].Name) } - m.next.IndexUpdate(deviceID, folder, files) + m.next.IndexUpdate(deviceID, folder, files, flags, options) } func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) { diff --git a/protocol.go b/protocol.go index 7e2658775..7d742a7fa 100644 --- a/protocol.go +++ b/protocol.go @@ -66,9 +66,9 @@ type pongMessage struct{ EmptyMessage } type Model interface { // An index was received from the peer device - Index(deviceID DeviceID, folder string, files []FileInfo) + Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) // An index update was received from the peer device - IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) + IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) // A request was made by the peer device Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) // A cluster configuration message was received @@ -80,8 +80,8 @@ type Model interface { type Connection interface { ID() DeviceID Name() string - Index(folder string, files []FileInfo) error - IndexUpdate(folder string, files []FileInfo) error + Index(folder string, files []FileInfo, flags uint32, options []Option) error + IndexUpdate(folder string, files []FileInfo, flags uint32, options []Option) error Request(folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) ClusterConfig(config ClusterConfigMessage) Statistics() Statistics @@ -169,7 +169,7 @@ func (c *rawConnection) Name() string { } // Index writes the list of file information to the connected peer device -func (c *rawConnection) Index(folder string, idx []FileInfo) error { +func (c *rawConnection) Index(folder string, idx []FileInfo, flags uint32, options []Option) error { select { case <-c.closed: return ErrClosed @@ -177,15 +177,17 @@ func (c *rawConnection) Index(folder string, idx []FileInfo) error { } c.idxMut.Lock() c.send(-1, messageTypeIndex, IndexMessage{ - Folder: folder, - Files: idx, + Folder: folder, + Files: idx, + Flags: flags, + Options: options, }) c.idxMut.Unlock() return nil } // IndexUpdate writes the list of file information to the connected peer device as an update -func (c *rawConnection) IndexUpdate(folder string, idx []FileInfo) error { +func (c *rawConnection) IndexUpdate(folder string, idx []FileInfo, flags uint32, options []Option) error { select { case <-c.closed: return ErrClosed @@ -193,8 +195,10 @@ func (c *rawConnection) IndexUpdate(folder string, idx []FileInfo) error { } c.idxMut.Lock() c.send(-1, messageTypeIndexUpdate, IndexMessage{ - Folder: folder, - Files: idx, + Folder: folder, + Files: idx, + Flags: flags, + Options: options, }) c.idxMut.Unlock() return nil @@ -463,16 +467,16 @@ func (c *rawConnection) readMessage() (hdr header, msg encodable, err error) { func (c *rawConnection) handleIndex(im IndexMessage) { if debug { - l.Debugf("Index(%v, %v, %d files)", c.id, im.Folder, len(im.Files)) + l.Debugf("Index(%v, %v, %d file, flags %x, opts: %s)", c.id, im.Folder, len(im.Files), im.Flags, im.Options) } - c.receiver.Index(c.id, im.Folder, filterIndexMessageFiles(im.Files)) + c.receiver.Index(c.id, im.Folder, filterIndexMessageFiles(im.Files), im.Flags, im.Options) } func (c *rawConnection) handleIndexUpdate(im IndexMessage) { if debug { - l.Debugf("queueing IndexUpdate(%v, %v, %d files)", c.id, im.Folder, len(im.Files)) + l.Debugf("queueing IndexUpdate(%v, %v, %d files, flags %x, opts: %s)", c.id, im.Folder, len(im.Files), im.Flags, im.Options) } - c.receiver.IndexUpdate(c.id, im.Folder, filterIndexMessageFiles(im.Files)) + c.receiver.IndexUpdate(c.id, im.Folder, filterIndexMessageFiles(im.Files), im.Flags, im.Options) } func filterIndexMessageFiles(fs []FileInfo) []FileInfo { diff --git a/protocol_test.go b/protocol_test.go index c660ba327..3ff1042cb 100644 --- a/protocol_test.go +++ b/protocol_test.go @@ -229,8 +229,8 @@ func TestClose(t *testing.T) { t.Error("Ping should not return true") } - c0.Index("default", nil) - c0.Index("default", nil) + c0.Index("default", nil, 0, nil) + c0.Index("default", nil, 0, nil) if _, err := c0.Request("default", "foo", 0, 0, nil, 0, nil); err == nil { t.Error("Request should return an error") diff --git a/wireformat.go b/wireformat.go index 23d347e1b..9411955ba 100644 --- a/wireformat.go +++ b/wireformat.go @@ -20,7 +20,7 @@ func (c wireFormatConnection) Name() string { return c.next.Name() } -func (c wireFormatConnection) Index(folder string, fs []FileInfo) error { +func (c wireFormatConnection) Index(folder string, fs []FileInfo, flags uint32, options []Option) error { var myFs = make([]FileInfo, len(fs)) copy(myFs, fs) @@ -28,10 +28,10 @@ func (c wireFormatConnection) Index(folder string, fs []FileInfo) error { myFs[i].Name = norm.NFC.String(filepath.ToSlash(myFs[i].Name)) } - return c.next.Index(folder, myFs) + return c.next.Index(folder, myFs, flags, options) } -func (c wireFormatConnection) IndexUpdate(folder string, fs []FileInfo) error { +func (c wireFormatConnection) IndexUpdate(folder string, fs []FileInfo, flags uint32, options []Option) error { var myFs = make([]FileInfo, len(fs)) copy(myFs, fs) @@ -39,7 +39,7 @@ func (c wireFormatConnection) IndexUpdate(folder string, fs []FileInfo) error { myFs[i].Name = norm.NFC.String(filepath.ToSlash(myFs[i].Name)) } - return c.next.IndexUpdate(folder, myFs) + return c.next.IndexUpdate(folder, myFs, flags, options) } func (c wireFormatConnection) Request(folder, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) { From fdf15f3ca323b647291e13b4970a0525926f9e4f Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Thu, 5 Feb 2015 23:01:17 +0000 Subject: [PATCH 41/94] Flag checking is now responsibility of the model --- protocol.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/protocol.go b/protocol.go index 7d742a7fa..48d43995a 100644 --- a/protocol.go +++ b/protocol.go @@ -287,11 +287,6 @@ func (c *rawConnection) readerLoop() (err error) { switch msg := msg.(type) { case IndexMessage: - if msg.Flags != 0 { - // We don't currently support or expect any flags. - return fmt.Errorf("protocol error: unknown flags 0x%x in Index(Update) message", msg.Flags) - } - switch hdr.msgType { case messageTypeIndex: if c.state < stateCCRcvd { @@ -308,10 +303,6 @@ func (c *rawConnection) readerLoop() (err error) { } case RequestMessage: - if msg.Flags != 0 { - // We don't currently support or expect any flags. - return fmt.Errorf("protocol error: unknown flags 0x%x in Request message", msg.Flags) - } if c.state < stateIdxRcvd { return fmt.Errorf("protocol error: request message in state %d", c.state) } From bf7fea9a0ac34db6c08db594e911b0dff2bd55e7 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Fri, 6 Feb 2015 21:34:51 +0000 Subject: [PATCH 42/94] Rename error to code, update xdr path --- message.go | 5 +++-- message_xdr.go | 8 ++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/message.go b/message.go index 91c331903..c2d898944 100644 --- a/message.go +++ b/message.go @@ -1,5 +1,6 @@ // Copyright (C) 2014 The Protocol Authors. +//go:generate -command genxdr go run ../syncthing/Godeps/_workspace/src/github.com/calmh/xdr/cmd/genxdr/main.go //go:generate genxdr -o message_xdr.go message.go package protocol @@ -78,8 +79,8 @@ type RequestMessage struct { } type ResponseMessage struct { - Data []byte - Error int32 + Data []byte + Code int32 } type ClusterConfigMessage struct { diff --git a/message_xdr.go b/message_xdr.go index 95d72eb1a..c179de769 100644 --- a/message_xdr.go +++ b/message_xdr.go @@ -465,13 +465,13 @@ ResponseMessage Structure: \ Data (variable length) \ / / +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Error | +| Code | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ struct ResponseMessage { opaque Data<>; - int Error; + int Code; } */ @@ -502,7 +502,7 @@ func (o ResponseMessage) AppendXDR(bs []byte) ([]byte, error) { func (o ResponseMessage) EncodeXDRInto(xw *xdr.Writer) (int, error) { xw.WriteBytes(o.Data) - xw.WriteUint32(uint32(o.Error)) + xw.WriteUint32(uint32(o.Code)) return xw.Tot(), xw.Error() } @@ -519,7 +519,7 @@ func (o *ResponseMessage) UnmarshalXDR(bs []byte) error { func (o *ResponseMessage) DecodeXDRFrom(xr *xdr.Reader) error { o.Data = xr.ReadBytes() - o.Error = int32(xr.ReadUint32()) + o.Code = int32(xr.ReadUint32()) return xr.Error() } From 34c2c1ec16729b4f1090db2a48893b129251e733 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Sun, 8 Feb 2015 11:04:01 +0000 Subject: [PATCH 43/94] Send and receive Request error codes --- errors.go | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ protocol.go | 5 +++-- 2 files changed, 54 insertions(+), 2 deletions(-) create mode 100755 errors.go diff --git a/errors.go b/errors.go new file mode 100755 index 000000000..31d27af0d --- /dev/null +++ b/errors.go @@ -0,0 +1,51 @@ +// Copyright (C) 2014 The Protocol Authors. + +package protocol + +import ( + "errors" +) + +const ( + ecNoError int32 = iota + ecGeneric + ecNoSuchFile + ecInvalid +) + +var ( + ErrNoError error = nil + ErrGeneric = errors.New("generic error") + ErrNoSuchFile = errors.New("no such file") + ErrInvalid = errors.New("file is invalid") +) + +var lookupError = map[int32]error{ + ecNoError: ErrNoError, + ecGeneric: ErrGeneric, + ecNoSuchFile: ErrNoSuchFile, + ecInvalid: ErrInvalid, +} + +var lookupCode = map[error]int32{ + ErrNoError: ecNoError, + ErrGeneric: ecGeneric, + ErrNoSuchFile: ecNoSuchFile, + ErrInvalid: ecInvalid, +} + +func codeToError(errcode int32) error { + err, ok := lookupError[errcode] + if !ok { + return ErrGeneric + } + return err +} + +func errorToCode(err error) int32 { + code, ok := lookupCode[err] + if !ok { + return ecGeneric + } + return code +} diff --git a/protocol.go b/protocol.go index 48d43995a..d46d70036 100644 --- a/protocol.go +++ b/protocol.go @@ -497,10 +497,11 @@ func filterIndexMessageFiles(fs []FileInfo) []FileInfo { } func (c *rawConnection) handleRequest(msgID int, req RequestMessage) { - data, _ := c.receiver.Request(c.id, req.Folder, req.Name, int64(req.Offset), int(req.Size), req.Hash, req.Flags, req.Options) + data, err := c.receiver.Request(c.id, req.Folder, req.Name, int64(req.Offset), int(req.Size), req.Hash, req.Flags, req.Options) c.send(msgID, messageTypeResponse, ResponseMessage{ Data: data, + Code: errorToCode(err), }) } @@ -508,7 +509,7 @@ func (c *rawConnection) handleResponse(msgID int, resp ResponseMessage) { c.awaitingMut.Lock() if rc := c.awaiting[msgID]; rc != nil { c.awaiting[msgID] = nil - rc <- asyncResult{resp.Data, nil} + rc <- asyncResult{resp.Data, codeToError(resp.Code)} close(rc) } c.awaitingMut.Unlock() From 1d76efcbcd24b408f4cdcea3f305aa9291985ba1 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Wed, 11 Feb 2015 22:11:44 +0000 Subject: [PATCH 44/94] Remove duplication --- nativemodel_windows.go | 41 +++++++++++++++++------------------------ 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/nativemodel_windows.go b/nativemodel_windows.go index 951f5b7e6..072e1278b 100644 --- a/nativemodel_windows.go +++ b/nativemodel_windows.go @@ -25,34 +25,12 @@ type nativeModel struct { } func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { - for i, f := range files { - if strings.ContainsAny(f.Name, disallowedCharacters) { - if f.IsDeleted() { - // Don't complain if the file is marked as deleted, since it - // can't possibly exist here anyway. - continue - } - files[i].Flags |= FlagInvalid - l.Warnf("File name %q contains invalid characters; marked as invalid.", f.Name) - } - files[i].Name = filepath.FromSlash(f.Name) - } + fixupFiles(files) m.next.Index(deviceID, folder, files, flags, options) } func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { - for i, f := range files { - if strings.ContainsAny(f.Name, disallowedCharacters) { - if f.IsDeleted() { - // Don't complain if the file is marked as deleted, since it - // can't possibly exist here anyway. - continue - } - files[i].Flags |= FlagInvalid - l.Warnf("File name %q contains invalid characters; marked as invalid.", f.Name) - } - files[i].Name = filepath.FromSlash(files[i].Name) - } + fixupFiles(files) m.next.IndexUpdate(deviceID, folder, files, flags, options) } @@ -68,3 +46,18 @@ func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessag func (m nativeModel) Close(deviceID DeviceID, err error) { m.next.Close(deviceID, err) } + +func fixupFiles(files []FileInfo) { + for i, f := range files { + if strings.ContainsAny(f.Name, disallowedCharacters) { + if f.IsDeleted() { + // Don't complain if the file is marked as deleted, since it + // can't possibly exist here anyway. + continue + } + files[i].Flags |= FlagInvalid + l.Warnf("File name %q contains invalid characters; marked as invalid.", f.Name) + } + files[i].Name = filepath.FromSlash(files[i].Name) + } +} From aa9eda197930d3c6f64a680f3e92f95763c1ed63 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Fri, 13 Feb 2015 23:27:01 +0000 Subject: [PATCH 45/94] Add IndexTemporary and RequestTemporary flags --- protocol.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/protocol.go b/protocol.go index d46d70036..b9859203a 100644 --- a/protocol.go +++ b/protocol.go @@ -35,6 +35,7 @@ const ( stateIdxRcvd ) +// FileInfo flags const ( FlagDeleted uint32 = 1 << 12 FlagInvalid = 1 << 13 @@ -48,6 +49,17 @@ const ( SymlinkTypeMask = FlagDirectory | FlagSymlinkMissingTarget ) +// IndexMessage message flags (for IndexUpdate) +const ( + FlagIndexTemporary uint32 = 1 << iota +) + +// Request message flags +const ( + FlagRequestTemporary uint32 = 1 << iota +) + +// ClusterConfigMessage.Folders.Devices flags const ( FlagShareTrusted uint32 = 1 << 0 FlagShareReadOnly = 1 << 1 From 3d8a71fdb205fe2401a341a739208bc9d1e79a1b Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Wed, 8 Apr 2015 14:46:08 +0200 Subject: [PATCH 46/94] Generate with updated XDR package --- message_xdr.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/message_xdr.go b/message_xdr.go index c179de769..68d01b696 100644 --- a/message_xdr.go +++ b/message_xdr.go @@ -110,12 +110,18 @@ func (o *IndexMessage) UnmarshalXDR(bs []byte) error { func (o *IndexMessage) DecodeXDRFrom(xr *xdr.Reader) error { o.Folder = xr.ReadString() _FilesSize := int(xr.ReadUint32()) + if _FilesSize < 0 { + return xdr.ElementSizeExceeded("Files", _FilesSize, 0) + } o.Files = make([]FileInfo, _FilesSize) for i := range o.Files { (&o.Files[i]).DecodeXDRFrom(xr) } o.Flags = xr.ReadUint32() _OptionsSize := int(xr.ReadUint32()) + if _OptionsSize < 0 { + return xdr.ElementSizeExceeded("Options", _OptionsSize, 64) + } if _OptionsSize > 64 { return xdr.ElementSizeExceeded("Options", _OptionsSize, 64) } @@ -236,6 +242,9 @@ func (o *FileInfo) DecodeXDRFrom(xr *xdr.Reader) error { (&o.Version).DecodeXDRFrom(xr) o.LocalVersion = int64(xr.ReadUint64()) _BlocksSize := int(xr.ReadUint32()) + if _BlocksSize < 0 { + return xdr.ElementSizeExceeded("Blocks", _BlocksSize, 0) + } o.Blocks = make([]BlockInfo, _BlocksSize) for i := range o.Blocks { (&o.Blocks[i]).DecodeXDRFrom(xr) @@ -442,6 +451,9 @@ func (o *RequestMessage) DecodeXDRFrom(xr *xdr.Reader) error { o.Hash = xr.ReadBytesMax(64) o.Flags = xr.ReadUint32() _OptionsSize := int(xr.ReadUint32()) + if _OptionsSize < 0 { + return xdr.ElementSizeExceeded("Options", _OptionsSize, 64) + } if _OptionsSize > 64 { return xdr.ElementSizeExceeded("Options", _OptionsSize, 64) } @@ -633,11 +645,17 @@ func (o *ClusterConfigMessage) DecodeXDRFrom(xr *xdr.Reader) error { o.ClientName = xr.ReadStringMax(64) o.ClientVersion = xr.ReadStringMax(64) _FoldersSize := int(xr.ReadUint32()) + if _FoldersSize < 0 { + return xdr.ElementSizeExceeded("Folders", _FoldersSize, 0) + } o.Folders = make([]Folder, _FoldersSize) for i := range o.Folders { (&o.Folders[i]).DecodeXDRFrom(xr) } _OptionsSize := int(xr.ReadUint32()) + if _OptionsSize < 0 { + return xdr.ElementSizeExceeded("Options", _OptionsSize, 64) + } if _OptionsSize > 64 { return xdr.ElementSizeExceeded("Options", _OptionsSize, 64) } @@ -750,12 +768,18 @@ func (o *Folder) UnmarshalXDR(bs []byte) error { func (o *Folder) DecodeXDRFrom(xr *xdr.Reader) error { o.ID = xr.ReadStringMax(64) _DevicesSize := int(xr.ReadUint32()) + if _DevicesSize < 0 { + return xdr.ElementSizeExceeded("Devices", _DevicesSize, 0) + } o.Devices = make([]Device, _DevicesSize) for i := range o.Devices { (&o.Devices[i]).DecodeXDRFrom(xr) } o.Flags = xr.ReadUint32() _OptionsSize := int(xr.ReadUint32()) + if _OptionsSize < 0 { + return xdr.ElementSizeExceeded("Options", _OptionsSize, 64) + } if _OptionsSize > 64 { return xdr.ElementSizeExceeded("Options", _OptionsSize, 64) } @@ -862,6 +886,9 @@ func (o *Device) DecodeXDRFrom(xr *xdr.Reader) error { o.MaxLocalVersion = int64(xr.ReadUint64()) o.Flags = xr.ReadUint32() _OptionsSize := int(xr.ReadUint32()) + if _OptionsSize < 0 { + return xdr.ElementSizeExceeded("Options", _OptionsSize, 64) + } if _OptionsSize > 64 { return xdr.ElementSizeExceeded("Options", _OptionsSize, 64) } From e7db2648034fb71b051902a02bc25d4468ed492e Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Thu, 9 Apr 2015 12:51:21 +0200 Subject: [PATCH 47/94] Extract counter value from vector --- vector.go | 10 ++++++++++ vector_test.go | 14 +++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/vector.go b/vector.go index 048594522..edd156143 100644 --- a/vector.go +++ b/vector.go @@ -103,3 +103,13 @@ func (a Vector) Concurrent(b Vector) bool { comp := a.Compare(b) return comp == ConcurrentGreater || comp == ConcurrentLesser } + +// Counter returns the current value of the given counter ID. +func (v Vector) Counter(id uint64) uint64 { + for _, c := range v { + if c.ID == id { + return c.Value + } + } + return 0 +} diff --git a/vector_test.go b/vector_test.go index 7815412c2..c01255e7a 100644 --- a/vector_test.go +++ b/vector_test.go @@ -118,5 +118,17 @@ func TestMerge(t *testing.T) { t.Errorf("%d: %+v.Merge(%+v) == %+v (expected %+v)", i, tc.a, tc.b, m, tc.m) } } - +} + +func TestCounterValue(t *testing.T) { + v0 := Vector{Counter{42, 1}, Counter{64, 5}} + if v0.Counter(42) != 1 { + t.Error("Counter error, %d != %d", v0.Counter(42), 1) + } + if v0.Counter(64) != 5 { + t.Error("Counter error, %d != %d", v0.Counter(64), 5) + } + if v0.Counter(72) != 0 { + t.Error("Counter error, %d != %d", v0.Counter(72), 0) + } } From 19d742b9e4b1f4f338c70c94547ed163406e66ae Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Wed, 24 Jun 2015 00:34:16 +0100 Subject: [PATCH 48/94] Initial commit --- .gitignore | 24 ++++++++++++++++++++++++ LICENSE | 22 ++++++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..daf913b1b --- /dev/null +++ b/.gitignore @@ -0,0 +1,24 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..581a17054 --- /dev/null +++ b/LICENSE @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2015 The Syncthing Project + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + From 8e191c8e6bd16847cd5599c2625d685a4c1bef8e Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Wed, 24 Jun 2015 12:39:46 +0100 Subject: [PATCH 49/94] Add initial code --- CONTRIBUTORS | 0 main.go | 88 +++++++++ protocol/packets.go | 45 +++++ protocol/packets_xdr.go | 415 ++++++++++++++++++++++++++++++++++++++++ protocol_listener.go | 230 ++++++++++++++++++++++ session.go | 173 +++++++++++++++++ session_listener.go | 59 ++++++ utils.go | 53 +++++ 8 files changed, 1063 insertions(+) create mode 100644 CONTRIBUTORS create mode 100644 main.go create mode 100644 protocol/packets.go create mode 100644 protocol/packets_xdr.go create mode 100644 protocol_listener.go create mode 100644 session.go create mode 100644 session_listener.go create mode 100644 utils.go diff --git a/CONTRIBUTORS b/CONTRIBUTORS new file mode 100644 index 000000000..e69de29bb diff --git a/main.go b/main.go new file mode 100644 index 000000000..3c4d533ed --- /dev/null +++ b/main.go @@ -0,0 +1,88 @@ +// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file). + +package main + +import ( + "crypto/tls" + "flag" + "log" + "os" + "path/filepath" + "sync" + "time" + + syncthingprotocol "github.com/syncthing/protocol" + "github.com/syncthing/relaysrv/protocol" +) + +var ( + listenProtocol string + listenSession string + debug bool + + sessionAddress []byte + sessionPort uint16 + + networkTimeout time.Duration + pingInterval time.Duration + messageTimeout time.Duration + + pingMessage message + + mut = sync.RWMutex{} + outbox = make(map[syncthingprotocol.DeviceID]chan message) +) + +func main() { + var dir, extAddress string + + pingPayload := protocol.Ping{}.MustMarshalXDR() + pingMessage = message{ + header: protocol.Header{ + Magic: protocol.Magic, + MessageType: protocol.MessageTypePing, + MessageLength: int32(len(pingPayload)), + }, + payload: pingPayload, + } + + flag.StringVar(&listenProtocol, "protocol-listen", ":22067", "Protocol listen address") + flag.StringVar(&listenSession, "session-listen", ":22068", "Session listen address") + flag.StringVar(&extAddress, "external-address", "", "External address to advertise, defaults no IP and session-listen port, causing clients to use the remote IP from the protocol connection") + flag.StringVar(&dir, "keys", ".", "Directory where cert.pem and key.pem is stored") + flag.DurationVar(&networkTimeout, "network-timeout", 2*time.Minute, "Timeout for network operations") + flag.DurationVar(&pingInterval, "ping-interval", time.Minute, "How often pings are sent") + flag.DurationVar(&messageTimeout, "message-timeout", time.Minute, "Maximum amount of time we wait for relevant messages to arrive") + + flag.BoolVar(&debug, "debug", false, "Enable debug output") + flag.Parse() + + certFile, keyFile := filepath.Join(dir, "cert.pem"), filepath.Join(dir, "key.pem") + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + log.Fatalln("Failed to load X509 key pair:", err) + } + + tlsCfg := &tls.Config{ + Certificates: []tls.Certificate{cert}, + NextProtos: []string{protocol.ProtocolName}, + ClientAuth: tls.RequestClientCert, + SessionTicketsDisabled: true, + InsecureSkipVerify: true, + MinVersion: tls.VersionTLS12, + CipherSuites: []uint16{ + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, + tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, + tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, + }, + } + + log.SetOutput(os.Stdout) + + go sessionListener(listenSession) + + protocolListener(listenProtocol, tlsCfg) +} diff --git a/protocol/packets.go b/protocol/packets.go new file mode 100644 index 000000000..4675d1cf4 --- /dev/null +++ b/protocol/packets.go @@ -0,0 +1,45 @@ +// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file). + +//go:generate -command genxdr go run ../../syncthing/Godeps/_workspace/src/github.com/calmh/xdr/cmd/genxdr/main.go +//go:generate genxdr -o packets_xdr.go packets.go + +package protocol + +import ( + "unsafe" +) + +const ( + Magic = 0x9E79BC40 + HeaderSize = unsafe.Sizeof(&Header{}) + ProtocolName = "bep-relay" +) + +const ( + MessageTypePing int32 = iota + MessageTypePong + MessageTypeJoinRequest + MessageTypeConnectRequest + MessageTypeSessionInvitation +) + +type Header struct { + Magic uint32 + MessageType int32 + MessageLength int32 +} + +type Ping struct{} +type Pong struct{} +type JoinRequest struct{} + +type ConnectRequest struct { + ID []byte // max:32 +} + +type SessionInvitation struct { + Key []byte // max:32 + Address []byte // max:32 + Port uint16 + ServerSocket bool +} diff --git a/protocol/packets_xdr.go b/protocol/packets_xdr.go new file mode 100644 index 000000000..ca547e007 --- /dev/null +++ b/protocol/packets_xdr.go @@ -0,0 +1,415 @@ +// ************************************************************ +// This file is automatically generated by genxdr. Do not edit. +// ************************************************************ + +package protocol + +import ( + "bytes" + "io" + + "github.com/calmh/xdr" +) + +/* + +Header Structure: + + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Magic | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Message Type | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Message Length | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + +struct Header { + unsigned int Magic; + int MessageType; + int MessageLength; +} + +*/ + +func (o Header) EncodeXDR(w io.Writer) (int, error) { + var xw = xdr.NewWriter(w) + return o.EncodeXDRInto(xw) +} + +func (o Header) MarshalXDR() ([]byte, error) { + return o.AppendXDR(make([]byte, 0, 128)) +} + +func (o Header) MustMarshalXDR() []byte { + bs, err := o.MarshalXDR() + if err != nil { + panic(err) + } + return bs +} + +func (o Header) AppendXDR(bs []byte) ([]byte, error) { + var aw = xdr.AppendWriter(bs) + var xw = xdr.NewWriter(&aw) + _, err := o.EncodeXDRInto(xw) + return []byte(aw), err +} + +func (o Header) EncodeXDRInto(xw *xdr.Writer) (int, error) { + xw.WriteUint32(o.Magic) + xw.WriteUint32(uint32(o.MessageType)) + xw.WriteUint32(uint32(o.MessageLength)) + return xw.Tot(), xw.Error() +} + +func (o *Header) DecodeXDR(r io.Reader) error { + xr := xdr.NewReader(r) + return o.DecodeXDRFrom(xr) +} + +func (o *Header) UnmarshalXDR(bs []byte) error { + var br = bytes.NewReader(bs) + var xr = xdr.NewReader(br) + return o.DecodeXDRFrom(xr) +} + +func (o *Header) DecodeXDRFrom(xr *xdr.Reader) error { + o.Magic = xr.ReadUint32() + o.MessageType = int32(xr.ReadUint32()) + o.MessageLength = int32(xr.ReadUint32()) + return xr.Error() +} + +/* + +Ping Structure: + + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + +struct Ping { +} + +*/ + +func (o Ping) EncodeXDR(w io.Writer) (int, error) { + var xw = xdr.NewWriter(w) + return o.EncodeXDRInto(xw) +} + +func (o Ping) MarshalXDR() ([]byte, error) { + return o.AppendXDR(make([]byte, 0, 128)) +} + +func (o Ping) MustMarshalXDR() []byte { + bs, err := o.MarshalXDR() + if err != nil { + panic(err) + } + return bs +} + +func (o Ping) AppendXDR(bs []byte) ([]byte, error) { + var aw = xdr.AppendWriter(bs) + var xw = xdr.NewWriter(&aw) + _, err := o.EncodeXDRInto(xw) + return []byte(aw), err +} + +func (o Ping) EncodeXDRInto(xw *xdr.Writer) (int, error) { + return xw.Tot(), xw.Error() +} + +func (o *Ping) DecodeXDR(r io.Reader) error { + xr := xdr.NewReader(r) + return o.DecodeXDRFrom(xr) +} + +func (o *Ping) UnmarshalXDR(bs []byte) error { + var br = bytes.NewReader(bs) + var xr = xdr.NewReader(br) + return o.DecodeXDRFrom(xr) +} + +func (o *Ping) DecodeXDRFrom(xr *xdr.Reader) error { + return xr.Error() +} + +/* + +Pong Structure: + + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + +struct Pong { +} + +*/ + +func (o Pong) EncodeXDR(w io.Writer) (int, error) { + var xw = xdr.NewWriter(w) + return o.EncodeXDRInto(xw) +} + +func (o Pong) MarshalXDR() ([]byte, error) { + return o.AppendXDR(make([]byte, 0, 128)) +} + +func (o Pong) MustMarshalXDR() []byte { + bs, err := o.MarshalXDR() + if err != nil { + panic(err) + } + return bs +} + +func (o Pong) AppendXDR(bs []byte) ([]byte, error) { + var aw = xdr.AppendWriter(bs) + var xw = xdr.NewWriter(&aw) + _, err := o.EncodeXDRInto(xw) + return []byte(aw), err +} + +func (o Pong) EncodeXDRInto(xw *xdr.Writer) (int, error) { + return xw.Tot(), xw.Error() +} + +func (o *Pong) DecodeXDR(r io.Reader) error { + xr := xdr.NewReader(r) + return o.DecodeXDRFrom(xr) +} + +func (o *Pong) UnmarshalXDR(bs []byte) error { + var br = bytes.NewReader(bs) + var xr = xdr.NewReader(br) + return o.DecodeXDRFrom(xr) +} + +func (o *Pong) DecodeXDRFrom(xr *xdr.Reader) error { + return xr.Error() +} + +/* + +JoinRequest Structure: + + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + +struct JoinRequest { +} + +*/ + +func (o JoinRequest) EncodeXDR(w io.Writer) (int, error) { + var xw = xdr.NewWriter(w) + return o.EncodeXDRInto(xw) +} + +func (o JoinRequest) MarshalXDR() ([]byte, error) { + return o.AppendXDR(make([]byte, 0, 128)) +} + +func (o JoinRequest) MustMarshalXDR() []byte { + bs, err := o.MarshalXDR() + if err != nil { + panic(err) + } + return bs +} + +func (o JoinRequest) AppendXDR(bs []byte) ([]byte, error) { + var aw = xdr.AppendWriter(bs) + var xw = xdr.NewWriter(&aw) + _, err := o.EncodeXDRInto(xw) + return []byte(aw), err +} + +func (o JoinRequest) EncodeXDRInto(xw *xdr.Writer) (int, error) { + return xw.Tot(), xw.Error() +} + +func (o *JoinRequest) DecodeXDR(r io.Reader) error { + xr := xdr.NewReader(r) + return o.DecodeXDRFrom(xr) +} + +func (o *JoinRequest) UnmarshalXDR(bs []byte) error { + var br = bytes.NewReader(bs) + var xr = xdr.NewReader(br) + return o.DecodeXDRFrom(xr) +} + +func (o *JoinRequest) DecodeXDRFrom(xr *xdr.Reader) error { + return xr.Error() +} + +/* + +ConnectRequest Structure: + + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Length of ID | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ ID (variable length) \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + +struct ConnectRequest { + opaque ID<32>; +} + +*/ + +func (o ConnectRequest) EncodeXDR(w io.Writer) (int, error) { + var xw = xdr.NewWriter(w) + return o.EncodeXDRInto(xw) +} + +func (o ConnectRequest) MarshalXDR() ([]byte, error) { + return o.AppendXDR(make([]byte, 0, 128)) +} + +func (o ConnectRequest) MustMarshalXDR() []byte { + bs, err := o.MarshalXDR() + if err != nil { + panic(err) + } + return bs +} + +func (o ConnectRequest) AppendXDR(bs []byte) ([]byte, error) { + var aw = xdr.AppendWriter(bs) + var xw = xdr.NewWriter(&aw) + _, err := o.EncodeXDRInto(xw) + return []byte(aw), err +} + +func (o ConnectRequest) EncodeXDRInto(xw *xdr.Writer) (int, error) { + if l := len(o.ID); l > 32 { + return xw.Tot(), xdr.ElementSizeExceeded("ID", l, 32) + } + xw.WriteBytes(o.ID) + return xw.Tot(), xw.Error() +} + +func (o *ConnectRequest) DecodeXDR(r io.Reader) error { + xr := xdr.NewReader(r) + return o.DecodeXDRFrom(xr) +} + +func (o *ConnectRequest) UnmarshalXDR(bs []byte) error { + var br = bytes.NewReader(bs) + var xr = xdr.NewReader(br) + return o.DecodeXDRFrom(xr) +} + +func (o *ConnectRequest) DecodeXDRFrom(xr *xdr.Reader) error { + o.ID = xr.ReadBytesMax(32) + return xr.Error() +} + +/* + +SessionInvitation Structure: + + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Length of Key | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Key (variable length) \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Length of Address | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Address (variable length) \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| 0x0000 | Port | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Server Socket (V=0 or 1) |V| ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + +struct SessionInvitation { + opaque Key<32>; + opaque Address<32>; + unsigned int Port; + bool ServerSocket; +} + +*/ + +func (o SessionInvitation) EncodeXDR(w io.Writer) (int, error) { + var xw = xdr.NewWriter(w) + return o.EncodeXDRInto(xw) +} + +func (o SessionInvitation) MarshalXDR() ([]byte, error) { + return o.AppendXDR(make([]byte, 0, 128)) +} + +func (o SessionInvitation) MustMarshalXDR() []byte { + bs, err := o.MarshalXDR() + if err != nil { + panic(err) + } + return bs +} + +func (o SessionInvitation) AppendXDR(bs []byte) ([]byte, error) { + var aw = xdr.AppendWriter(bs) + var xw = xdr.NewWriter(&aw) + _, err := o.EncodeXDRInto(xw) + return []byte(aw), err +} + +func (o SessionInvitation) EncodeXDRInto(xw *xdr.Writer) (int, error) { + if l := len(o.Key); l > 32 { + return xw.Tot(), xdr.ElementSizeExceeded("Key", l, 32) + } + xw.WriteBytes(o.Key) + if l := len(o.Address); l > 32 { + return xw.Tot(), xdr.ElementSizeExceeded("Address", l, 32) + } + xw.WriteBytes(o.Address) + xw.WriteUint16(o.Port) + xw.WriteBool(o.ServerSocket) + return xw.Tot(), xw.Error() +} + +func (o *SessionInvitation) DecodeXDR(r io.Reader) error { + xr := xdr.NewReader(r) + return o.DecodeXDRFrom(xr) +} + +func (o *SessionInvitation) UnmarshalXDR(bs []byte) error { + var br = bytes.NewReader(bs) + var xr = xdr.NewReader(br) + return o.DecodeXDRFrom(xr) +} + +func (o *SessionInvitation) DecodeXDRFrom(xr *xdr.Reader) error { + o.Key = xr.ReadBytesMax(32) + o.Address = xr.ReadBytesMax(32) + o.Port = xr.ReadUint16() + o.ServerSocket = xr.ReadBool() + return xr.Error() +} diff --git a/protocol_listener.go b/protocol_listener.go new file mode 100644 index 000000000..b6d89b226 --- /dev/null +++ b/protocol_listener.go @@ -0,0 +1,230 @@ +// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file). + +package main + +import ( + "crypto/tls" + "io" + "log" + "net" + "time" + + syncthingprotocol "github.com/syncthing/protocol" + + "github.com/syncthing/relaysrv/protocol" +) + +type message struct { + header protocol.Header + payload []byte +} + +func protocolListener(addr string, config *tls.Config) { + listener, err := net.Listen("tcp", addr) + if err != nil { + log.Fatalln(err) + } + + for { + conn, err := listener.Accept() + if err != nil { + if debug { + log.Println(err) + } + continue + } + + if debug { + log.Println("Protocol listener accepted connection from", conn.RemoteAddr()) + } + + go protocolConnectionHandler(conn, config) + } +} + +func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config) { + err := setTCPOptions(tcpConn) + if err != nil && debug { + log.Println("Failed to set TCP options on protocol connection", tcpConn.RemoteAddr(), err) + } + + conn := tls.Server(tcpConn, config) + err = conn.Handshake() + if err != nil { + log.Println("Protocol connection TLS handshake:", conn.RemoteAddr(), err) + conn.Close() + return + } + + state := conn.ConnectionState() + if (!state.NegotiatedProtocolIsMutual || state.NegotiatedProtocol != protocol.ProtocolName) && debug { + log.Println("Protocol negotiation error") + } + + certs := state.PeerCertificates + if len(certs) != 1 { + log.Println("Certificate list error") + conn.Close() + return + } + + deviceId := syncthingprotocol.NewDeviceID(certs[0].Raw) + + mut.RLock() + _, ok := outbox[deviceId] + mut.RUnlock() + if ok { + log.Println("Already have a peer with the same ID", deviceId, conn.RemoteAddr()) + conn.Close() + return + } + + errorChannel := make(chan error) + messageChannel := make(chan message) + outboxChannel := make(chan message) + + go readerLoop(conn, messageChannel, errorChannel) + + pingTicker := time.NewTicker(pingInterval) + timeoutTicker := time.NewTimer(messageTimeout * 2) + joined := false + + for { + select { + case msg := <-messageChannel: + switch msg.header.MessageType { + case protocol.MessageTypeJoinRequest: + mut.Lock() + outbox[deviceId] = outboxChannel + mut.Unlock() + joined = true + case protocol.MessageTypeConnectRequest: + // We will disconnect after this message, no matter what, + // because, we've either sent out an invitation, or we don't + // have the peer available. + var fmsg protocol.ConnectRequest + err := fmsg.UnmarshalXDR(msg.payload) + if err != nil { + log.Println(err) + conn.Close() + continue + } + + requestedPeer := syncthingprotocol.DeviceIDFromBytes(fmsg.ID) + mut.RLock() + peerOutbox, ok := outbox[requestedPeer] + mut.RUnlock() + if !ok { + if debug { + log.Println("Do not have", requestedPeer) + } + conn.Close() + continue + } + + ses := newSession() + + smsg, err := ses.GetServerInvitationMessage() + if err != nil { + log.Println("Error getting server invitation", requestedPeer) + conn.Close() + continue + } + cmsg, err := ses.GetClientInvitationMessage() + if err != nil { + log.Println("Error getting client invitation", requestedPeer) + conn.Close() + continue + } + + go ses.Serve() + + if err := sendMessage(cmsg, conn); err != nil { + log.Println("Failed to send invitation message", err) + } else { + peerOutbox <- smsg + if debug { + log.Println("Sent invitation from", deviceId, "to", requestedPeer) + } + } + conn.Close() + case protocol.MessageTypePong: + timeoutTicker.Reset(messageTimeout) + } + case err := <-errorChannel: + log.Println("Closing connection:", err) + return + case <-pingTicker.C: + if !joined { + log.Println(deviceId, "didn't join within", messageTimeout) + conn.Close() + continue + } + + if err := sendMessage(pingMessage, conn); err != nil { + log.Println(err) + conn.Close() + continue + } + case <-timeoutTicker.C: + // We should receive a error, which will cause us to quit the + // loop. + conn.Close() + case msg := <-outboxChannel: + if debug { + log.Println("Sending message to", deviceId, msg) + } + if err := sendMessage(msg, conn); err == nil { + log.Println(err) + conn.Close() + continue + } + } + } +} + +func readerLoop(conn *tls.Conn, messages chan<- message, errors chan<- error) { + header := make([]byte, protocol.HeaderSize) + data := make([]byte, 0, 0) + for { + _, err := io.ReadFull(conn, header) + if err != nil { + errors <- err + conn.Close() + return + } + + var hdr protocol.Header + err = hdr.UnmarshalXDR(header) + if err != nil { + conn.Close() + return + } + + if hdr.Magic != protocol.Magic { + conn.Close() + return + } + + if hdr.MessageLength > int32(cap(data)) { + data = make([]byte, 0, hdr.MessageLength) + } else { + data = data[:hdr.MessageLength] + } + + _, err = io.ReadFull(conn, data) + if err != nil { + errors <- err + conn.Close() + return + } + + msg := message{ + header: hdr, + payload: make([]byte, hdr.MessageLength), + } + copy(msg.payload, data[:hdr.MessageLength]) + + messages <- msg + } +} diff --git a/session.go b/session.go new file mode 100644 index 000000000..3466bd535 --- /dev/null +++ b/session.go @@ -0,0 +1,173 @@ +// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file). + +package main + +import ( + "crypto/rand" + "net" + "sync" + "time" + + "github.com/syncthing/relaysrv/protocol" +) + +var ( + sessionmut = sync.Mutex{} + sessions = make(map[string]*session, 0) +) + +type session struct { + serverkey string + clientkey string + + mut sync.RWMutex + conns chan net.Conn +} + +func newSession() *session { + serverkey := make([]byte, 32) + _, err := rand.Read(serverkey) + if err != nil { + return nil + } + + clientkey := make([]byte, 32) + _, err = rand.Read(clientkey) + if err != nil { + return nil + } + + return &session{ + serverkey: string(serverkey), + clientkey: string(clientkey), + conns: make(chan net.Conn), + } +} + +func findSession(key string) *session { + sessionmut.Lock() + defer sessionmut.Unlock() + lob, ok := sessions[key] + if !ok { + return nil + + } + delete(sessions, key) + return lob +} + +func (l *session) AddConnection(conn net.Conn) { + select { + case l.conns <- conn: + default: + } +} + +func (l *session) Serve() { + + timedout := time.After(messageTimeout) + + sessionmut.Lock() + sessions[l.serverkey] = l + sessions[l.clientkey] = l + sessionmut.Unlock() + + conns := make([]net.Conn, 0, 2) + for { + select { + case conn := <-l.conns: + conns = append(conns, conn) + if len(conns) < 2 { + continue + } + + close(l.conns) + + wg := sync.WaitGroup{} + + wg.Add(2) + + go proxy(conns[0], conns[1], wg) + go proxy(conns[1], conns[0], wg) + + wg.Wait() + + break + case <-timedout: + sessionmut.Lock() + delete(sessions, l.serverkey) + delete(sessions, l.clientkey) + sessionmut.Unlock() + + for _, conn := range conns { + conn.Close() + } + + break + } + } +} + +func (l *session) GetClientInvitationMessage() (message, error) { + invitation := protocol.SessionInvitation{ + Key: []byte(l.clientkey), + Address: nil, + Port: 123, + ServerSocket: false, + } + data, err := invitation.MarshalXDR() + if err != nil { + return message{}, err + } + + return message{ + header: protocol.Header{ + Magic: protocol.Magic, + MessageType: protocol.MessageTypeSessionInvitation, + MessageLength: int32(len(data)), + }, + payload: data, + }, nil +} + +func (l *session) GetServerInvitationMessage() (message, error) { + invitation := protocol.SessionInvitation{ + Key: []byte(l.serverkey), + Address: nil, + Port: 123, + ServerSocket: true, + } + data, err := invitation.MarshalXDR() + if err != nil { + return message{}, err + } + + return message{ + header: protocol.Header{ + Magic: protocol.Magic, + MessageType: protocol.MessageTypeSessionInvitation, + MessageLength: int32(len(data)), + }, + payload: data, + }, nil +} + +func proxy(c1, c2 net.Conn, wg sync.WaitGroup) { + for { + buf := make([]byte, 1024) + c1.SetReadDeadline(time.Now().Add(networkTimeout)) + n, err := c1.Read(buf) + if err != nil { + break + } + + c2.SetWriteDeadline(time.Now().Add(networkTimeout)) + _, err = c2.Write(buf[:n]) + if err != nil { + break + } + } + c1.Close() + c2.Close() + wg.Done() +} diff --git a/session_listener.go b/session_listener.go new file mode 100644 index 000000000..b78c4f4b6 --- /dev/null +++ b/session_listener.go @@ -0,0 +1,59 @@ +// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file). + +package main + +import ( + "io" + "log" + "net" + "time" +) + +func sessionListener(addr string) { + listener, err := net.Listen("tcp", addr) + if err != nil { + log.Fatalln(err) + } + + for { + conn, err := listener.Accept() + if err != nil { + if debug { + log.Println(err) + } + continue + } + + if debug { + log.Println("Session listener accepted connection from", conn.RemoteAddr()) + } + + go sessionConnectionHandler(conn) + } +} + +func sessionConnectionHandler(conn net.Conn) { + conn.SetReadDeadline(time.Now().Add(messageTimeout)) + key := make([]byte, 32) + + _, err := io.ReadFull(conn, key) + if err != nil { + if debug { + log.Println("Failed to read key", err, conn.RemoteAddr()) + } + conn.Close() + return + } + + ses := findSession(string(key)) + if debug { + log.Println("Key", key, "by", conn.RemoteAddr(), "session", ses) + } + + if ses != nil { + ses.AddConnection(conn) + } else { + conn.Close() + return + } +} diff --git a/utils.go b/utils.go new file mode 100644 index 000000000..5388ba32e --- /dev/null +++ b/utils.go @@ -0,0 +1,53 @@ +// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file). + +package main + +import ( + "errors" + "net" + "time" +) + +func setTCPOptions(conn net.Conn) error { + tcpConn, ok := conn.(*net.TCPConn) + if !ok { + return errors.New("Not a TCP connection") + } + if err := tcpConn.SetLinger(0); err != nil { + return err + } + if err := tcpConn.SetNoDelay(true); err != nil { + return err + } + if err := tcpConn.SetKeepAlivePeriod(60 * time.Second); err != nil { + return err + } + if err := tcpConn.SetKeepAlive(true); err != nil { + return err + } + return nil +} + +func sendMessage(msg message, conn net.Conn) error { + header, err := msg.header.MarshalXDR() + if err != nil { + return err + } + + err = conn.SetWriteDeadline(time.Now().Add(networkTimeout)) + if err != nil { + return err + } + + _, err = conn.Write(header) + if err != nil { + return err + } + + _, err = conn.Write(msg.payload) + if err != nil { + return err + } + + return nil +} From cbe44e1fff4bb67cf7761e1f5b2ec4501bb1aa26 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Fri, 26 Jun 2015 15:38:56 +0200 Subject: [PATCH 50/94] Enforce ClusterConfiguration at start, then no ordering --- protocol.go | 45 +++++++++++++++++++++++++-------------------- protocol_test.go | 30 +++++++++++++++++++++--------- 2 files changed, 46 insertions(+), 29 deletions(-) diff --git a/protocol.go b/protocol.go index b9859203a..605de4781 100644 --- a/protocol.go +++ b/protocol.go @@ -31,8 +31,7 @@ const ( const ( stateInitial = iota - stateCCRcvd - stateIdxRcvd + stateReady ) // FileInfo flags @@ -103,7 +102,6 @@ type rawConnection struct { id DeviceID name string receiver Model - state int cr *countingReader cw *countingWriter @@ -155,7 +153,6 @@ func NewConnection(deviceID DeviceID, reader io.Reader, writer io.Writer, receiv id: deviceID, name: name, receiver: nativeModel{receiver}, - state: stateInitial, cr: cr, cw: cw, outbox: make(chan hdrMsg), @@ -285,6 +282,7 @@ func (c *rawConnection) readerLoop() (err error) { c.close(err) }() + state := stateInitial for { select { case <-c.closed: @@ -298,47 +296,54 @@ func (c *rawConnection) readerLoop() (err error) { } switch msg := msg.(type) { + case ClusterConfigMessage: + if state != stateInitial { + return fmt.Errorf("protocol error: cluster config message in state %d", state) + } + go c.receiver.ClusterConfig(c.id, msg) + state = stateReady + case IndexMessage: switch hdr.msgType { case messageTypeIndex: - if c.state < stateCCRcvd { - return fmt.Errorf("protocol error: index message in state %d", c.state) + if state != stateReady { + return fmt.Errorf("protocol error: index message in state %d", state) } c.handleIndex(msg) - c.state = stateIdxRcvd + state = stateReady case messageTypeIndexUpdate: - if c.state < stateIdxRcvd { - return fmt.Errorf("protocol error: index update message in state %d", c.state) + if state != stateReady { + return fmt.Errorf("protocol error: index update message in state %d", state) } c.handleIndexUpdate(msg) + state = stateReady } case RequestMessage: - if c.state < stateIdxRcvd { - return fmt.Errorf("protocol error: request message in state %d", c.state) + if state != stateReady { + return fmt.Errorf("protocol error: request message in state %d", state) } // Requests are handled asynchronously go c.handleRequest(hdr.msgID, msg) case ResponseMessage: - if c.state < stateIdxRcvd { - return fmt.Errorf("protocol error: response message in state %d", c.state) + if state != stateReady { + return fmt.Errorf("protocol error: response message in state %d", state) } c.handleResponse(hdr.msgID, msg) case pingMessage: + if state != stateReady { + return fmt.Errorf("protocol error: ping message in state %d", state) + } c.send(hdr.msgID, messageTypePong, pongMessage{}) case pongMessage: - c.handlePong(hdr.msgID) - - case ClusterConfigMessage: - if c.state != stateInitial { - return fmt.Errorf("protocol error: cluster config message in state %d", c.state) + if state != stateReady { + return fmt.Errorf("protocol error: pong message in state %d", state) } - go c.receiver.ClusterConfig(c.id, msg) - c.state = stateCCRcvd + c.handlePong(hdr.msgID) case CloseMessage: return errors.New(msg.Reason) diff --git a/protocol_test.go b/protocol_test.go index 3ff1042cb..bb4fe7d95 100644 --- a/protocol_test.go +++ b/protocol_test.go @@ -67,8 +67,10 @@ func TestPing(t *testing.T) { ar, aw := io.Pipe() br, bw := io.Pipe() - c0 := NewConnection(c0ID, ar, bw, nil, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) - c1 := NewConnection(c1ID, br, aw, nil, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) + c0 := NewConnection(c0ID, ar, bw, newTestModel(), "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) + c1 := NewConnection(c1ID, br, aw, newTestModel(), "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) + c0.ClusterConfig(ClusterConfigMessage{}) + c1.ClusterConfig(ClusterConfigMessage{}) if ok := c0.ping(); !ok { t.Error("c0 ping failed") @@ -81,8 +83,8 @@ func TestPing(t *testing.T) { func TestPingErr(t *testing.T) { e := errors.New("something broke") - for i := 0; i < 16; i++ { - for j := 0; j < 16; j++ { + for i := 0; i < 32; i++ { + for j := 0; j < 32; j++ { m0 := newTestModel() m1 := newTestModel() @@ -92,12 +94,16 @@ func TestPingErr(t *testing.T) { ebw := &ErrPipe{PipeWriter: *bw, max: j, err: e} c0 := NewConnection(c0ID, ar, ebw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) - NewConnection(c1ID, br, eaw, m1, "name", CompressAlways) + c1 := NewConnection(c1ID, br, eaw, m1, "name", CompressAlways) + c0.ClusterConfig(ClusterConfigMessage{}) + c1.ClusterConfig(ClusterConfigMessage{}) res := c0.ping() if (i < 8 || j < 8) && res { + // This should have resulted in failure, as there is no way an empty ClusterConfig plus a Ping message fits in eight bytes. t.Errorf("Unexpected ping success; i=%d, j=%d", i, j) - } else if (i >= 12 && j >= 12) && !res { + } else if (i >= 28 && j >= 28) && !res { + // This should have worked though, as 28 bytes is plenty for both. t.Errorf("Unexpected ping fail; i=%d, j=%d", i, j) } } @@ -168,7 +174,9 @@ func TestVersionErr(t *testing.T) { br, bw := io.Pipe() c0 := NewConnection(c0ID, ar, bw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) - NewConnection(c1ID, br, aw, m1, "name", CompressAlways) + c1 := NewConnection(c1ID, br, aw, m1, "name", CompressAlways) + c0.ClusterConfig(ClusterConfigMessage{}) + c1.ClusterConfig(ClusterConfigMessage{}) w := xdr.NewWriter(c0.cw) w.WriteUint32(encodeHeader(header{ @@ -191,7 +199,9 @@ func TestTypeErr(t *testing.T) { br, bw := io.Pipe() c0 := NewConnection(c0ID, ar, bw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) - NewConnection(c1ID, br, aw, m1, "name", CompressAlways) + c1 := NewConnection(c1ID, br, aw, m1, "name", CompressAlways) + c0.ClusterConfig(ClusterConfigMessage{}) + c1.ClusterConfig(ClusterConfigMessage{}) w := xdr.NewWriter(c0.cw) w.WriteUint32(encodeHeader(header{ @@ -214,7 +224,9 @@ func TestClose(t *testing.T) { br, bw := io.Pipe() c0 := NewConnection(c0ID, ar, bw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) - NewConnection(c1ID, br, aw, m1, "name", CompressAlways) + c1 := NewConnection(c1ID, br, aw, m1, "name", CompressAlways) + c0.ClusterConfig(ClusterConfigMessage{}) + c1.ClusterConfig(ClusterConfigMessage{}) c0.close(nil) From 9f871a372629080e07662db3ccf4f075bdac7a6f Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Sat, 27 Jun 2015 11:07:44 +0100 Subject: [PATCH 51/94] Expose timeouts in protocol --- protocol.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/protocol.go b/protocol.go index 605de4781..50115c09c 100644 --- a/protocol.go +++ b/protocol.go @@ -140,9 +140,9 @@ type isEofer interface { IsEOF() bool } -const ( - pingTimeout = 30 * time.Second - pingIdleTime = 60 * time.Second +var ( + PingTimeout = 30 * time.Second + PingIdleTime = 60 * time.Second ) func NewConnection(deviceID DeviceID, reader io.Reader, writer io.Writer, receiver Model, name string, compress Compression) Connection { @@ -684,17 +684,17 @@ func (c *rawConnection) idGenerator() { func (c *rawConnection) pingerLoop() { var rc = make(chan bool, 1) - ticker := time.Tick(pingIdleTime / 2) + ticker := time.Tick(PingIdleTime / 2) for { select { case <-ticker: - if d := time.Since(c.cr.Last()); d < pingIdleTime { + if d := time.Since(c.cr.Last()); d < PingIdleTime { if debug { l.Debugln(c.id, "ping skipped after rd", d) } continue } - if d := time.Since(c.cw.Last()); d < pingIdleTime { + if d := time.Since(c.cw.Last()); d < PingIdleTime { if debug { l.Debugln(c.id, "ping skipped after wr", d) } @@ -714,7 +714,7 @@ func (c *rawConnection) pingerLoop() { if !ok { c.close(fmt.Errorf("ping failure")) } - case <-time.After(pingTimeout): + case <-time.After(PingTimeout): c.close(fmt.Errorf("ping timeout")) case <-c.closed: return From b72d31f87fa34fa77a0a5a73712ec019bdf2bf61 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Sun, 28 Jun 2015 01:52:01 +0100 Subject: [PATCH 52/94] Progress --- README.md | 6 + client/client.go | 249 ++++++++++++++++++++++++++++++++++++++++ client/debug.go | 15 +++ client/methods.go | 113 ++++++++++++++++++ main.go | 39 ++++--- protocol/packets.go | 42 +++---- protocol/packets_xdr.go | 216 ++++++++++++++++++++++++++++------ protocol/protocol.go | 114 ++++++++++++++++++ protocol_listener.go | 233 +++++++++++++++++-------------------- session.go | 179 ++++++++++++++++------------- session_listener.go | 58 +++++++--- testutil/main.go | 142 +++++++++++++++++++++++ utils.go | 27 +---- 13 files changed, 1114 insertions(+), 319 deletions(-) create mode 100644 README.md create mode 100644 client/client.go create mode 100644 client/debug.go create mode 100644 client/methods.go create mode 100644 protocol/protocol.go create mode 100644 testutil/main.go diff --git a/README.md b/README.md new file mode 100644 index 000000000..e88929280 --- /dev/null +++ b/README.md @@ -0,0 +1,6 @@ +relaysrv +======== + +This is the relay server for the `syncthing` project. + +`go get github.com/syncthing/relaysrv` diff --git a/client/client.go b/client/client.go new file mode 100644 index 000000000..b48320fd2 --- /dev/null +++ b/client/client.go @@ -0,0 +1,249 @@ +// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file). + +package client + +import ( + "crypto/tls" + "fmt" + "log" + "net" + "net/url" + "time" + + syncthingprotocol "github.com/syncthing/protocol" + "github.com/syncthing/relaysrv/protocol" +) + +func NewProtocolClient(uri *url.URL, certs []tls.Certificate, invitations chan protocol.SessionInvitation) ProtocolClient { + closeInvitationsOnFinish := false + if invitations == nil { + closeInvitationsOnFinish = true + invitations = make(chan protocol.SessionInvitation) + } + return ProtocolClient{ + URI: uri, + Invitations: invitations, + + closeInvitationsOnFinish: closeInvitationsOnFinish, + + config: configForCerts(certs), + + timeout: time.Minute * 2, + + stop: make(chan struct{}), + stopped: make(chan struct{}), + } +} + +type ProtocolClient struct { + URI *url.URL + Invitations chan protocol.SessionInvitation + + closeInvitationsOnFinish bool + + config *tls.Config + + timeout time.Duration + + stop chan struct{} + stopped chan struct{} + + conn *tls.Conn +} + +func (c *ProtocolClient) connect() error { + conn, err := tls.Dial("tcp", c.URI.Host, c.config) + if err != nil { + return err + } + + conn.SetDeadline(time.Now().Add(10 * time.Second)) + + if err := performHandshakeAndValidation(conn, c.URI); err != nil { + return err + } + + c.conn = conn + return nil +} + +func (c *ProtocolClient) Serve() { + if err := c.connect(); err != nil { + panic(err) + } + + if debug { + l.Debugln(c, "connected", c.conn.RemoteAddr()) + } + + if err := c.join(); err != nil { + c.conn.Close() + panic(err) + } + + c.conn.SetDeadline(time.Time{}) + + if debug { + l.Debugln(c, "joined", c.conn.RemoteAddr(), "via", c.conn.LocalAddr()) + } + + c.stop = make(chan struct{}) + c.stopped = make(chan struct{}) + + defer c.cleanup() + + messages := make(chan interface{}) + errors := make(chan error, 1) + + go func(conn net.Conn, message chan<- interface{}, errors chan<- error) { + for { + msg, err := protocol.ReadMessage(conn) + if err != nil { + errors <- err + return + } + messages <- msg + } + }(c.conn, messages, errors) + + timeout := time.NewTimer(c.timeout) + for { + select { + case message := <-messages: + timeout.Reset(c.timeout) + if debug { + log.Printf("%s received message %T", c, message) + } + switch msg := message.(type) { + case protocol.Ping: + if err := protocol.WriteMessage(c.conn, protocol.Pong{}); err != nil { + panic(err) + } + if debug { + l.Debugln(c, "sent pong") + } + case protocol.SessionInvitation: + ip := net.IP(msg.Address) + if len(ip) == 0 || ip.IsUnspecified() { + msg.Address = c.conn.RemoteAddr().(*net.TCPAddr).IP[:] + } + c.Invitations <- msg + default: + panic(fmt.Errorf("protocol error: unexpected message %v", msg)) + } + case <-c.stop: + if debug { + l.Debugln(c, "stopping") + } + break + case err := <-errors: + panic(err) + case <-timeout.C: + if debug { + l.Debugln(c, "timed out") + } + return + } + } + + c.stopped <- struct{}{} +} + +func (c *ProtocolClient) Stop() { + if c.stop == nil { + return + } + + c.stop <- struct{}{} + <-c.stopped +} + +func (c *ProtocolClient) String() string { + return fmt.Sprintf("ProtocolClient@%p", c) +} + +func (c *ProtocolClient) cleanup() { + if c.closeInvitationsOnFinish { + close(c.Invitations) + c.Invitations = make(chan protocol.SessionInvitation) + } + + if debug { + l.Debugln(c, "cleaning up") + } + + if c.stop != nil { + close(c.stop) + c.stop = nil + } + + if c.stopped != nil { + close(c.stopped) + c.stopped = nil + } + + if c.conn != nil { + c.conn.Close() + } +} + +func (c *ProtocolClient) join() error { + err := protocol.WriteMessage(c.conn, protocol.JoinRelayRequest{}) + if err != nil { + return err + } + + message, err := protocol.ReadMessage(c.conn) + if err != nil { + return err + } + + switch msg := message.(type) { + case protocol.Response: + if msg.Code != 0 { + return fmt.Errorf("Incorrect response code %d: %s", msg.Code, msg.Message) + } + default: + return fmt.Errorf("protocol error: expecting response got %v", msg) + } + + return nil +} + +func performHandshakeAndValidation(conn *tls.Conn, uri *url.URL) error { + err := conn.Handshake() + if err != nil { + conn.Close() + return err + } + + cs := conn.ConnectionState() + if !cs.NegotiatedProtocolIsMutual || cs.NegotiatedProtocol != protocol.ProtocolName { + conn.Close() + return fmt.Errorf("protocol negotiation error") + } + + q := uri.Query() + relayIDs := q.Get("id") + if relayIDs != "" { + relayID, err := syncthingprotocol.DeviceIDFromString(relayIDs) + if err != nil { + conn.Close() + return fmt.Errorf("relay address contains invalid verification id: %s", err) + } + + certs := cs.PeerCertificates + if cl := len(certs); cl != 1 { + conn.Close() + return fmt.Errorf("unexpected certificate count: %d", cl) + } + + remoteID := syncthingprotocol.NewDeviceID(certs[0].Raw) + if remoteID != relayID { + conn.Close() + return fmt.Errorf("relay id does not match. Expected %v got %v", relayID, remoteID) + } + } + + return nil +} diff --git a/client/debug.go b/client/debug.go new file mode 100644 index 000000000..4a3608dec --- /dev/null +++ b/client/debug.go @@ -0,0 +1,15 @@ +// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file). + +package client + +import ( + "os" + "strings" + + "github.com/calmh/logger" +) + +var ( + debug = strings.Contains(os.Getenv("STTRACE"), "relay") || os.Getenv("STTRACE") == "all" + l = logger.DefaultLogger +) diff --git a/client/methods.go b/client/methods.go new file mode 100644 index 000000000..1d457e294 --- /dev/null +++ b/client/methods.go @@ -0,0 +1,113 @@ +// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file). + +package client + +import ( + "crypto/tls" + "fmt" + "net" + "net/url" + "strconv" + "time" + + syncthingprotocol "github.com/syncthing/protocol" + "github.com/syncthing/relaysrv/protocol" +) + +func GetInvitationFromRelay(uri *url.URL, id syncthingprotocol.DeviceID, certs []tls.Certificate) (protocol.SessionInvitation, error) { + conn, err := tls.Dial("tcp", uri.Host, configForCerts(certs)) + conn.SetDeadline(time.Now().Add(10 * time.Second)) + if err != nil { + return protocol.SessionInvitation{}, err + } + + if err := performHandshakeAndValidation(conn, uri); err != nil { + return protocol.SessionInvitation{}, err + } + + defer conn.Close() + + request := protocol.ConnectRequest{ + ID: id[:], + } + + if err := protocol.WriteMessage(conn, request); err != nil { + return protocol.SessionInvitation{}, err + } + + message, err := protocol.ReadMessage(conn) + if err != nil { + return protocol.SessionInvitation{}, err + } + + switch msg := message.(type) { + case protocol.Response: + return protocol.SessionInvitation{}, fmt.Errorf("Incorrect response code %d: %s", msg.Code, msg.Message) + case protocol.SessionInvitation: + if debug { + l.Debugln("Received invitation via", conn.LocalAddr()) + } + ip := net.IP(msg.Address) + if len(ip) == 0 || ip.IsUnspecified() { + msg.Address = conn.RemoteAddr().(*net.TCPAddr).IP[:] + } + return msg, nil + default: + return protocol.SessionInvitation{}, fmt.Errorf("protocol error: unexpected message %v", msg) + } +} + +func JoinSession(invitation protocol.SessionInvitation) (net.Conn, error) { + addr := net.JoinHostPort(net.IP(invitation.Address).String(), strconv.Itoa(int(invitation.Port))) + + conn, err := net.Dial("tcp", addr) + if err != nil { + return nil, err + } + + request := protocol.JoinSessionRequest{ + Key: invitation.Key, + } + + conn.SetDeadline(time.Now().Add(10 * time.Second)) + err = protocol.WriteMessage(conn, request) + if err != nil { + return nil, err + } + + message, err := protocol.ReadMessage(conn) + if err != nil { + return nil, err + } + + conn.SetDeadline(time.Time{}) + + switch msg := message.(type) { + case protocol.Response: + if msg.Code != 0 { + return nil, fmt.Errorf("Incorrect response code %d: %s", msg.Code, msg.Message) + } + return conn, nil + default: + return nil, fmt.Errorf("protocol error: expecting response got %v", msg) + } +} + +func configForCerts(certs []tls.Certificate) *tls.Config { + return &tls.Config{ + Certificates: certs, + NextProtos: []string{protocol.ProtocolName}, + ClientAuth: tls.RequestClientCert, + SessionTicketsDisabled: true, + InsecureSkipVerify: true, + MinVersion: tls.VersionTLS12, + CipherSuites: []uint16{ + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, + tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, + tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, + }, + } +} diff --git a/main.go b/main.go index 3c4d533ed..5ca060689 100644 --- a/main.go +++ b/main.go @@ -6,13 +6,13 @@ import ( "crypto/tls" "flag" "log" - "os" + "net" "path/filepath" - "sync" "time" - syncthingprotocol "github.com/syncthing/protocol" "github.com/syncthing/relaysrv/protocol" + + syncthingprotocol "github.com/syncthing/protocol" ) var ( @@ -26,26 +26,11 @@ var ( networkTimeout time.Duration pingInterval time.Duration messageTimeout time.Duration - - pingMessage message - - mut = sync.RWMutex{} - outbox = make(map[syncthingprotocol.DeviceID]chan message) ) func main() { var dir, extAddress string - pingPayload := protocol.Ping{}.MustMarshalXDR() - pingMessage = message{ - header: protocol.Header{ - Magic: protocol.Magic, - MessageType: protocol.MessageTypePing, - MessageLength: int32(len(pingPayload)), - }, - payload: pingPayload, - } - flag.StringVar(&listenProtocol, "protocol-listen", ":22067", "Protocol listen address") flag.StringVar(&listenSession, "session-listen", ":22068", "Session listen address") flag.StringVar(&extAddress, "external-address", "", "External address to advertise, defaults no IP and session-listen port, causing clients to use the remote IP from the protocol connection") @@ -54,7 +39,20 @@ func main() { flag.DurationVar(&pingInterval, "ping-interval", time.Minute, "How often pings are sent") flag.DurationVar(&messageTimeout, "message-timeout", time.Minute, "Maximum amount of time we wait for relevant messages to arrive") + if extAddress == "" { + extAddress = listenSession + } + + addr, err := net.ResolveTCPAddr("tcp", extAddress) + if err != nil { + log.Fatal(err) + } + + sessionAddress = addr.IP[:] + sessionPort = uint16(addr.Port) + flag.BoolVar(&debug, "debug", false, "Enable debug output") + flag.Parse() certFile, keyFile := filepath.Join(dir, "cert.pem"), filepath.Join(dir, "key.pem") @@ -80,7 +78,10 @@ func main() { }, } - log.SetOutput(os.Stdout) + id := syncthingprotocol.NewDeviceID(cert.Certificate[0]) + if debug { + log.Println("ID:", id) + } go sessionListener(listenSession) diff --git a/protocol/packets.go b/protocol/packets.go index 4675d1cf4..658bc536d 100644 --- a/protocol/packets.go +++ b/protocol/packets.go @@ -5,39 +5,41 @@ package protocol -import ( - "unsafe" -) - const ( - Magic = 0x9E79BC40 - HeaderSize = unsafe.Sizeof(&Header{}) - ProtocolName = "bep-relay" + messageTypePing int32 = iota + messageTypePong + messageTypeJoinRelayRequest + messageTypeJoinSessionRequest + messageTypeResponse + messageTypeConnectRequest + messageTypeSessionInvitation ) -const ( - MessageTypePing int32 = iota - MessageTypePong - MessageTypeJoinRequest - MessageTypeConnectRequest - MessageTypeSessionInvitation -) - -type Header struct { - Magic uint32 - MessageType int32 - MessageLength int32 +type header struct { + magic uint32 + messageType int32 + messageLength int32 } type Ping struct{} type Pong struct{} -type JoinRequest struct{} +type JoinRelayRequest struct{} + +type JoinSessionRequest struct { + Key []byte // max:32 +} + +type Response struct { + Code int32 + Message string +} type ConnectRequest struct { ID []byte // max:32 } type SessionInvitation struct { + From []byte // max:32 Key []byte // max:32 Address []byte // max:32 Port uint16 diff --git a/protocol/packets_xdr.go b/protocol/packets_xdr.go index ca547e007..f18e18c18 100644 --- a/protocol/packets_xdr.go +++ b/protocol/packets_xdr.go @@ -13,37 +13,37 @@ import ( /* -Header Structure: +header Structure: 0 1 2 3 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Magic | +| magic | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Message Type | +| message Type | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Message Length | +| message Length | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -struct Header { - unsigned int Magic; - int MessageType; - int MessageLength; +struct header { + unsigned int magic; + int messageType; + int messageLength; } */ -func (o Header) EncodeXDR(w io.Writer) (int, error) { +func (o header) EncodeXDR(w io.Writer) (int, error) { var xw = xdr.NewWriter(w) return o.EncodeXDRInto(xw) } -func (o Header) MarshalXDR() ([]byte, error) { +func (o header) MarshalXDR() ([]byte, error) { return o.AppendXDR(make([]byte, 0, 128)) } -func (o Header) MustMarshalXDR() []byte { +func (o header) MustMarshalXDR() []byte { bs, err := o.MarshalXDR() if err != nil { panic(err) @@ -51,35 +51,35 @@ func (o Header) MustMarshalXDR() []byte { return bs } -func (o Header) AppendXDR(bs []byte) ([]byte, error) { +func (o header) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) _, err := o.EncodeXDRInto(xw) return []byte(aw), err } -func (o Header) EncodeXDRInto(xw *xdr.Writer) (int, error) { - xw.WriteUint32(o.Magic) - xw.WriteUint32(uint32(o.MessageType)) - xw.WriteUint32(uint32(o.MessageLength)) +func (o header) EncodeXDRInto(xw *xdr.Writer) (int, error) { + xw.WriteUint32(o.magic) + xw.WriteUint32(uint32(o.messageType)) + xw.WriteUint32(uint32(o.messageLength)) return xw.Tot(), xw.Error() } -func (o *Header) DecodeXDR(r io.Reader) error { +func (o *header) DecodeXDR(r io.Reader) error { xr := xdr.NewReader(r) return o.DecodeXDRFrom(xr) } -func (o *Header) UnmarshalXDR(bs []byte) error { +func (o *header) UnmarshalXDR(bs []byte) error { var br = bytes.NewReader(bs) var xr = xdr.NewReader(br) return o.DecodeXDRFrom(xr) } -func (o *Header) DecodeXDRFrom(xr *xdr.Reader) error { - o.Magic = xr.ReadUint32() - o.MessageType = int32(xr.ReadUint32()) - o.MessageLength = int32(xr.ReadUint32()) +func (o *header) DecodeXDRFrom(xr *xdr.Reader) error { + o.magic = xr.ReadUint32() + o.messageType = int32(xr.ReadUint32()) + o.messageLength = int32(xr.ReadUint32()) return xr.Error() } @@ -199,28 +199,28 @@ func (o *Pong) DecodeXDRFrom(xr *xdr.Reader) error { /* -JoinRequest Structure: +JoinRelayRequest Structure: 0 1 2 3 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -struct JoinRequest { +struct JoinRelayRequest { } */ -func (o JoinRequest) EncodeXDR(w io.Writer) (int, error) { +func (o JoinRelayRequest) EncodeXDR(w io.Writer) (int, error) { var xw = xdr.NewWriter(w) return o.EncodeXDRInto(xw) } -func (o JoinRequest) MarshalXDR() ([]byte, error) { +func (o JoinRelayRequest) MarshalXDR() ([]byte, error) { return o.AppendXDR(make([]byte, 0, 128)) } -func (o JoinRequest) MustMarshalXDR() []byte { +func (o JoinRelayRequest) MustMarshalXDR() []byte { bs, err := o.MarshalXDR() if err != nil { panic(err) @@ -228,29 +228,169 @@ func (o JoinRequest) MustMarshalXDR() []byte { return bs } -func (o JoinRequest) AppendXDR(bs []byte) ([]byte, error) { +func (o JoinRelayRequest) AppendXDR(bs []byte) ([]byte, error) { var aw = xdr.AppendWriter(bs) var xw = xdr.NewWriter(&aw) _, err := o.EncodeXDRInto(xw) return []byte(aw), err } -func (o JoinRequest) EncodeXDRInto(xw *xdr.Writer) (int, error) { +func (o JoinRelayRequest) EncodeXDRInto(xw *xdr.Writer) (int, error) { return xw.Tot(), xw.Error() } -func (o *JoinRequest) DecodeXDR(r io.Reader) error { +func (o *JoinRelayRequest) DecodeXDR(r io.Reader) error { xr := xdr.NewReader(r) return o.DecodeXDRFrom(xr) } -func (o *JoinRequest) UnmarshalXDR(bs []byte) error { +func (o *JoinRelayRequest) UnmarshalXDR(bs []byte) error { var br = bytes.NewReader(bs) var xr = xdr.NewReader(br) return o.DecodeXDRFrom(xr) } -func (o *JoinRequest) DecodeXDRFrom(xr *xdr.Reader) error { +func (o *JoinRelayRequest) DecodeXDRFrom(xr *xdr.Reader) error { + return xr.Error() +} + +/* + +JoinSessionRequest Structure: + + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Length of Key | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Key (variable length) \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + +struct JoinSessionRequest { + opaque Key<32>; +} + +*/ + +func (o JoinSessionRequest) EncodeXDR(w io.Writer) (int, error) { + var xw = xdr.NewWriter(w) + return o.EncodeXDRInto(xw) +} + +func (o JoinSessionRequest) MarshalXDR() ([]byte, error) { + return o.AppendXDR(make([]byte, 0, 128)) +} + +func (o JoinSessionRequest) MustMarshalXDR() []byte { + bs, err := o.MarshalXDR() + if err != nil { + panic(err) + } + return bs +} + +func (o JoinSessionRequest) AppendXDR(bs []byte) ([]byte, error) { + var aw = xdr.AppendWriter(bs) + var xw = xdr.NewWriter(&aw) + _, err := o.EncodeXDRInto(xw) + return []byte(aw), err +} + +func (o JoinSessionRequest) EncodeXDRInto(xw *xdr.Writer) (int, error) { + if l := len(o.Key); l > 32 { + return xw.Tot(), xdr.ElementSizeExceeded("Key", l, 32) + } + xw.WriteBytes(o.Key) + return xw.Tot(), xw.Error() +} + +func (o *JoinSessionRequest) DecodeXDR(r io.Reader) error { + xr := xdr.NewReader(r) + return o.DecodeXDRFrom(xr) +} + +func (o *JoinSessionRequest) UnmarshalXDR(bs []byte) error { + var br = bytes.NewReader(bs) + var xr = xdr.NewReader(br) + return o.DecodeXDRFrom(xr) +} + +func (o *JoinSessionRequest) DecodeXDRFrom(xr *xdr.Reader) error { + o.Key = xr.ReadBytesMax(32) + return xr.Error() +} + +/* + +Response Structure: + + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Code | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Length of Message | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ Message (variable length) \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + +struct Response { + int Code; + string Message<>; +} + +*/ + +func (o Response) EncodeXDR(w io.Writer) (int, error) { + var xw = xdr.NewWriter(w) + return o.EncodeXDRInto(xw) +} + +func (o Response) MarshalXDR() ([]byte, error) { + return o.AppendXDR(make([]byte, 0, 128)) +} + +func (o Response) MustMarshalXDR() []byte { + bs, err := o.MarshalXDR() + if err != nil { + panic(err) + } + return bs +} + +func (o Response) AppendXDR(bs []byte) ([]byte, error) { + var aw = xdr.AppendWriter(bs) + var xw = xdr.NewWriter(&aw) + _, err := o.EncodeXDRInto(xw) + return []byte(aw), err +} + +func (o Response) EncodeXDRInto(xw *xdr.Writer) (int, error) { + xw.WriteUint32(uint32(o.Code)) + xw.WriteString(o.Message) + return xw.Tot(), xw.Error() +} + +func (o *Response) DecodeXDR(r io.Reader) error { + xr := xdr.NewReader(r) + return o.DecodeXDRFrom(xr) +} + +func (o *Response) UnmarshalXDR(bs []byte) error { + var br = bytes.NewReader(bs) + var xr = xdr.NewReader(br) + return o.DecodeXDRFrom(xr) +} + +func (o *Response) DecodeXDRFrom(xr *xdr.Reader) error { + o.Code = int32(xr.ReadUint32()) + o.Message = xr.ReadString() return xr.Error() } @@ -330,6 +470,12 @@ SessionInvitation Structure: 0 1 2 3 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| Length of From | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +/ / +\ From (variable length) \ +/ / ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ | Length of Key | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ / / @@ -349,6 +495,7 @@ SessionInvitation Structure: struct SessionInvitation { + opaque From<32>; opaque Key<32>; opaque Address<32>; unsigned int Port; @@ -382,6 +529,10 @@ func (o SessionInvitation) AppendXDR(bs []byte) ([]byte, error) { } func (o SessionInvitation) EncodeXDRInto(xw *xdr.Writer) (int, error) { + if l := len(o.From); l > 32 { + return xw.Tot(), xdr.ElementSizeExceeded("From", l, 32) + } + xw.WriteBytes(o.From) if l := len(o.Key); l > 32 { return xw.Tot(), xdr.ElementSizeExceeded("Key", l, 32) } @@ -407,6 +558,7 @@ func (o *SessionInvitation) UnmarshalXDR(bs []byte) error { } func (o *SessionInvitation) DecodeXDRFrom(xr *xdr.Reader) error { + o.From = xr.ReadBytesMax(32) o.Key = xr.ReadBytesMax(32) o.Address = xr.ReadBytesMax(32) o.Port = xr.ReadUint16() diff --git a/protocol/protocol.go b/protocol/protocol.go new file mode 100644 index 000000000..57a967ac8 --- /dev/null +++ b/protocol/protocol.go @@ -0,0 +1,114 @@ +// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file). + +package protocol + +import ( + "fmt" + "io" +) + +const ( + magic = 0x9E79BC40 + ProtocolName = "bep-relay" +) + +var ( + ResponseSuccess = Response{0, "success"} + ResponseNotFound = Response{1, "not found"} + ResponseAlreadyConnected = Response{2, "already connected"} + ResponseInternalError = Response{99, "internal error"} + ResponseUnexpectedMessage = Response{100, "unexpected message"} +) + +func WriteMessage(w io.Writer, message interface{}) error { + header := header{ + magic: magic, + } + + var payload []byte + var err error + + switch msg := message.(type) { + case Ping: + payload, err = msg.MarshalXDR() + header.messageType = messageTypePing + case Pong: + payload, err = msg.MarshalXDR() + header.messageType = messageTypePong + case JoinRelayRequest: + payload, err = msg.MarshalXDR() + header.messageType = messageTypeJoinRelayRequest + case JoinSessionRequest: + payload, err = msg.MarshalXDR() + header.messageType = messageTypeJoinSessionRequest + case Response: + payload, err = msg.MarshalXDR() + header.messageType = messageTypeResponse + case ConnectRequest: + payload, err = msg.MarshalXDR() + header.messageType = messageTypeConnectRequest + case SessionInvitation: + payload, err = msg.MarshalXDR() + header.messageType = messageTypeSessionInvitation + default: + err = fmt.Errorf("Unknown message type") + } + + if err != nil { + return err + } + + header.messageLength = int32(len(payload)) + + headerpayload, err := header.MarshalXDR() + if err != nil { + return err + } + + _, err = w.Write(append(headerpayload, payload...)) + return err +} + +func ReadMessage(r io.Reader) (interface{}, error) { + var header header + if err := header.DecodeXDR(r); err != nil { + return nil, err + } + + if header.magic != magic { + return nil, fmt.Errorf("magic mismatch") + } + + switch header.messageType { + case messageTypePing: + var msg Ping + err := msg.DecodeXDR(r) + return msg, err + case messageTypePong: + var msg Pong + err := msg.DecodeXDR(r) + return msg, err + case messageTypeJoinRelayRequest: + var msg JoinRelayRequest + err := msg.DecodeXDR(r) + return msg, err + case messageTypeJoinSessionRequest: + var msg JoinSessionRequest + err := msg.DecodeXDR(r) + return msg, err + case messageTypeResponse: + var msg Response + err := msg.DecodeXDR(r) + return msg, err + case messageTypeConnectRequest: + var msg ConnectRequest + err := msg.DecodeXDR(r) + return msg, err + case messageTypeSessionInvitation: + var msg SessionInvitation + err := msg.DecodeXDR(r) + return msg, err + } + + return nil, fmt.Errorf("Unknown message type") +} diff --git a/protocol_listener.go b/protocol_listener.go index b6d89b226..1e18b156e 100644 --- a/protocol_listener.go +++ b/protocol_listener.go @@ -4,9 +4,9 @@ package main import ( "crypto/tls" - "io" "log" "net" + "sync" "time" syncthingprotocol "github.com/syncthing/protocol" @@ -14,10 +14,10 @@ import ( "github.com/syncthing/relaysrv/protocol" ) -type message struct { - header protocol.Header - payload []byte -} +var ( + outboxesMut = sync.RWMutex{} + outboxes = make(map[syncthingprotocol.DeviceID]chan interface{}) +) func protocolListener(addr string, config *tls.Config) { listener, err := net.Listen("tcp", addr) @@ -27,6 +27,7 @@ func protocolListener(addr string, config *tls.Config) { for { conn, err := listener.Accept() + setTCPOptions(conn) if err != nil { if debug { log.Println(err) @@ -43,15 +44,12 @@ func protocolListener(addr string, config *tls.Config) { } func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config) { - err := setTCPOptions(tcpConn) - if err != nil && debug { - log.Println("Failed to set TCP options on protocol connection", tcpConn.RemoteAddr(), err) - } - conn := tls.Server(tcpConn, config) - err = conn.Handshake() + err := conn.Handshake() if err != nil { - log.Println("Protocol connection TLS handshake:", conn.RemoteAddr(), err) + if debug { + log.Println("Protocol connection TLS handshake:", conn.RemoteAddr(), err) + } conn.Close() return } @@ -63,168 +61,147 @@ func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config) { certs := state.PeerCertificates if len(certs) != 1 { - log.Println("Certificate list error") + if debug { + log.Println("Certificate list error") + } conn.Close() return } - deviceId := syncthingprotocol.NewDeviceID(certs[0].Raw) + id := syncthingprotocol.NewDeviceID(certs[0].Raw) - mut.RLock() - _, ok := outbox[deviceId] - mut.RUnlock() - if ok { - log.Println("Already have a peer with the same ID", deviceId, conn.RemoteAddr()) - conn.Close() - return - } + messages := make(chan interface{}) + errors := make(chan error, 1) + outbox := make(chan interface{}) - errorChannel := make(chan error) - messageChannel := make(chan message) - outboxChannel := make(chan message) - - go readerLoop(conn, messageChannel, errorChannel) + go func(conn net.Conn, message chan<- interface{}, errors chan<- error) { + for { + msg, err := protocol.ReadMessage(conn) + if err != nil { + errors <- err + return + } + messages <- msg + } + }(conn, messages, errors) pingTicker := time.NewTicker(pingInterval) - timeoutTicker := time.NewTimer(messageTimeout * 2) + timeoutTicker := time.NewTimer(networkTimeout) joined := false for { select { - case msg := <-messageChannel: - switch msg.header.MessageType { - case protocol.MessageTypeJoinRequest: - mut.Lock() - outbox[deviceId] = outboxChannel - mut.Unlock() - joined = true - case protocol.MessageTypeConnectRequest: - // We will disconnect after this message, no matter what, - // because, we've either sent out an invitation, or we don't - // have the peer available. - var fmsg protocol.ConnectRequest - err := fmsg.UnmarshalXDR(msg.payload) - if err != nil { - log.Println(err) + case message := <-messages: + timeoutTicker.Reset(networkTimeout) + if debug { + log.Printf("Message %T from %s", message, id) + } + switch msg := message.(type) { + case protocol.JoinRelayRequest: + outboxesMut.RLock() + _, ok := outboxes[id] + outboxesMut.RUnlock() + if ok { + protocol.WriteMessage(conn, protocol.ResponseAlreadyConnected) + if debug { + log.Println("Already have a peer with the same ID", id, conn.RemoteAddr()) + } conn.Close() continue } - requestedPeer := syncthingprotocol.DeviceIDFromBytes(fmsg.ID) - mut.RLock() - peerOutbox, ok := outbox[requestedPeer] - mut.RUnlock() + outboxesMut.Lock() + outboxes[id] = outbox + outboxesMut.Unlock() + joined = true + + protocol.WriteMessage(conn, protocol.ResponseSuccess) + case protocol.ConnectRequest: + requestedPeer := syncthingprotocol.DeviceIDFromBytes(msg.ID) + outboxesMut.RLock() + peerOutbox, ok := outboxes[requestedPeer] + outboxesMut.RUnlock() if !ok { if debug { - log.Println("Do not have", requestedPeer) + log.Println(id, "is looking", requestedPeer, "which does not exist") } + protocol.WriteMessage(conn, protocol.ResponseNotFound) conn.Close() continue } ses := newSession() - smsg, err := ses.GetServerInvitationMessage() - if err != nil { - log.Println("Error getting server invitation", requestedPeer) - conn.Close() - continue - } - cmsg, err := ses.GetClientInvitationMessage() - if err != nil { - log.Println("Error getting client invitation", requestedPeer) - conn.Close() - continue - } - go ses.Serve() - if err := sendMessage(cmsg, conn); err != nil { - log.Println("Failed to send invitation message", err) - } else { - peerOutbox <- smsg + clientInvitation := ses.GetClientInvitationMessage(requestedPeer) + serverInvitation := ses.GetServerInvitationMessage(id) + + if err := protocol.WriteMessage(conn, clientInvitation); err != nil { if debug { - log.Println("Sent invitation from", deviceId, "to", requestedPeer) + log.Printf("Error sending invitation from %s to client: %s", id, err) } + conn.Close() + continue + } + + peerOutbox <- serverInvitation + + if debug { + log.Println("Sent invitation from", id, "to", requestedPeer) } conn.Close() - case protocol.MessageTypePong: - timeoutTicker.Reset(messageTimeout) + case protocol.Pong: + default: + if debug { + log.Printf("Unknown message %s: %T", id, message) + } + protocol.WriteMessage(conn, protocol.ResponseUnexpectedMessage) + conn.Close() } - case err := <-errorChannel: - log.Println("Closing connection:", err) + case err := <-errors: + if debug { + log.Printf("Closing connection %s: %s", id, err) + } + // Potentially closing a second time. + close(outbox) + conn.Close() + outboxesMut.Lock() + delete(outboxes, id) + outboxesMut.Unlock() return case <-pingTicker.C: if !joined { - log.Println(deviceId, "didn't join within", messageTimeout) + if debug { + log.Println(id, "didn't join within", pingInterval) + } conn.Close() continue } - if err := sendMessage(pingMessage, conn); err != nil { - log.Println(err) + if err := protocol.WriteMessage(conn, protocol.Ping{}); err != nil { + if debug { + log.Println(id, err) + } conn.Close() - continue } case <-timeoutTicker.C: - // We should receive a error, which will cause us to quit the - // loop. - conn.Close() - case msg := <-outboxChannel: + // We should receive a error from the reader loop, which will cause + // us to quit this loop. if debug { - log.Println("Sending message to", deviceId, msg) + log.Printf("%s timed out", id) } - if err := sendMessage(msg, conn); err == nil { - log.Println(err) + conn.Close() + case msg := <-outbox: + if debug { + log.Printf("Sending message %T to %s", msg, id) + } + if err := protocol.WriteMessage(conn, msg); err != nil { + if debug { + log.Println(id, err) + } conn.Close() - continue } } } } - -func readerLoop(conn *tls.Conn, messages chan<- message, errors chan<- error) { - header := make([]byte, protocol.HeaderSize) - data := make([]byte, 0, 0) - for { - _, err := io.ReadFull(conn, header) - if err != nil { - errors <- err - conn.Close() - return - } - - var hdr protocol.Header - err = hdr.UnmarshalXDR(header) - if err != nil { - conn.Close() - return - } - - if hdr.Magic != protocol.Magic { - conn.Close() - return - } - - if hdr.MessageLength > int32(cap(data)) { - data = make([]byte, 0, hdr.MessageLength) - } else { - data = data[:hdr.MessageLength] - } - - _, err = io.ReadFull(conn, data) - if err != nil { - errors <- err - conn.Close() - return - } - - msg := message{ - header: hdr, - payload: make([]byte, hdr.MessageLength), - } - copy(msg.payload, data[:hdr.MessageLength]) - - messages <- msg - } -} diff --git a/session.go b/session.go index 3466bd535..c5a091952 100644 --- a/session.go +++ b/session.go @@ -4,23 +4,27 @@ package main import ( "crypto/rand" + "encoding/hex" + "fmt" + "log" "net" "sync" "time" "github.com/syncthing/relaysrv/protocol" + + syncthingprotocol "github.com/syncthing/protocol" ) var ( - sessionmut = sync.Mutex{} + sessionMut = sync.Mutex{} sessions = make(map[string]*session, 0) ) type session struct { - serverkey string - clientkey string + serverkey []byte + clientkey []byte - mut sync.RWMutex conns chan net.Conn } @@ -37,16 +41,27 @@ func newSession() *session { return nil } - return &session{ - serverkey: string(serverkey), - clientkey: string(clientkey), + ses := &session{ + serverkey: serverkey, + clientkey: clientkey, conns: make(chan net.Conn), } + + if debug { + log.Println("New session", ses) + } + + sessionMut.Lock() + sessions[string(ses.serverkey)] = ses + sessions[string(ses.clientkey)] = ses + sessionMut.Unlock() + + return ses } func findSession(key string) *session { - sessionmut.Lock() - defer sessionmut.Unlock() + sessionMut.Lock() + defer sessionMut.Unlock() lob, ok := sessions[key] if !ok { return nil @@ -56,118 +71,128 @@ func findSession(key string) *session { return lob } -func (l *session) AddConnection(conn net.Conn) { +func (s *session) AddConnection(conn net.Conn) bool { + if debug { + log.Println("New connection for", s, "from", conn.RemoteAddr()) + } + select { - case l.conns <- conn: + case s.conns <- conn: + return true default: } + return false } -func (l *session) Serve() { - +func (s *session) Serve() { timedout := time.After(messageTimeout) - sessionmut.Lock() - sessions[l.serverkey] = l - sessions[l.clientkey] = l - sessionmut.Unlock() + if debug { + log.Println("Session", s, "serving") + } conns := make([]net.Conn, 0, 2) for { select { - case conn := <-l.conns: + case conn := <-s.conns: conns = append(conns, conn) if len(conns) < 2 { continue } - close(l.conns) + close(s.conns) + + if debug { + log.Println("Session", s, "starting between", conns[0].RemoteAddr(), conns[1].RemoteAddr()) + } wg := sync.WaitGroup{} - wg.Add(2) - go proxy(conns[0], conns[1], wg) - go proxy(conns[1], conns[0], wg) + errors := make(chan error, 2) + + go func() { + errors <- proxy(conns[0], conns[1]) + wg.Done() + }() + + go func() { + errors <- proxy(conns[1], conns[0]) + wg.Done() + }() wg.Wait() - break - case <-timedout: - sessionmut.Lock() - delete(sessions, l.serverkey) - delete(sessions, l.clientkey) - sessionmut.Unlock() - - for _, conn := range conns { - conn.Close() + if debug { + log.Println("Session", s, "ended, outcomes:", <-errors, <-errors) } - - break + goto done + case <-timedout: + if debug { + log.Println("Session", s, "timed out") + } + goto done } } +done: + sessionMut.Lock() + delete(sessions, string(s.serverkey)) + delete(sessions, string(s.clientkey)) + sessionMut.Unlock() + + for _, conn := range conns { + conn.Close() + } + + if debug { + log.Println("Session", s, "stopping") + } } -func (l *session) GetClientInvitationMessage() (message, error) { - invitation := protocol.SessionInvitation{ - Key: []byte(l.clientkey), - Address: nil, - Port: 123, +func (s *session) GetClientInvitationMessage(from syncthingprotocol.DeviceID) protocol.SessionInvitation { + return protocol.SessionInvitation{ + From: from[:], + Key: []byte(s.clientkey), + Address: sessionAddress, + Port: sessionPort, ServerSocket: false, } - data, err := invitation.MarshalXDR() - if err != nil { - return message{}, err - } - - return message{ - header: protocol.Header{ - Magic: protocol.Magic, - MessageType: protocol.MessageTypeSessionInvitation, - MessageLength: int32(len(data)), - }, - payload: data, - }, nil } -func (l *session) GetServerInvitationMessage() (message, error) { - invitation := protocol.SessionInvitation{ - Key: []byte(l.serverkey), - Address: nil, - Port: 123, +func (s *session) GetServerInvitationMessage(from syncthingprotocol.DeviceID) protocol.SessionInvitation { + return protocol.SessionInvitation{ + From: from[:], + Key: []byte(s.serverkey), + Address: sessionAddress, + Port: sessionPort, ServerSocket: true, } - data, err := invitation.MarshalXDR() - if err != nil { - return message{}, err - } - - return message{ - header: protocol.Header{ - Magic: protocol.Magic, - MessageType: protocol.MessageTypeSessionInvitation, - MessageLength: int32(len(data)), - }, - payload: data, - }, nil } -func proxy(c1, c2 net.Conn, wg sync.WaitGroup) { +func proxy(c1, c2 net.Conn) error { + if debug { + log.Println("Proxy", c1.RemoteAddr(), "->", c2.RemoteAddr()) + } + buf := make([]byte, 1024) for { - buf := make([]byte, 1024) c1.SetReadDeadline(time.Now().Add(networkTimeout)) - n, err := c1.Read(buf) + n, err := c1.Read(buf[0:]) if err != nil { - break + return err + } + + if debug { + log.Printf("%d bytes from %s to %s", n, c1.RemoteAddr(), c2.RemoteAddr()) } c2.SetWriteDeadline(time.Now().Add(networkTimeout)) _, err = c2.Write(buf[:n]) if err != nil { - break + return err } } - c1.Close() - c2.Close() - wg.Done() +} + +func (s *session) String() string { + return fmt.Sprintf("<%s/%s>", hex.EncodeToString(s.clientkey)[:5], hex.EncodeToString(s.serverkey)[:5]) } diff --git a/session_listener.go b/session_listener.go index b78c4f4b6..6159ceef5 100644 --- a/session_listener.go +++ b/session_listener.go @@ -3,10 +3,11 @@ package main import ( - "io" "log" "net" "time" + + "github.com/syncthing/relaysrv/protocol" ) func sessionListener(addr string) { @@ -17,6 +18,7 @@ func sessionListener(addr string) { for { conn, err := listener.Accept() + setTCPOptions(conn) if err != nil { if debug { log.Println(err) @@ -33,27 +35,49 @@ func sessionListener(addr string) { } func sessionConnectionHandler(conn net.Conn) { - conn.SetReadDeadline(time.Now().Add(messageTimeout)) - key := make([]byte, 32) - - _, err := io.ReadFull(conn, key) + conn.SetDeadline(time.Now().Add(messageTimeout)) + message, err := protocol.ReadMessage(conn) if err != nil { + conn.Close() + return + } + + switch msg := message.(type) { + case protocol.JoinSessionRequest: + ses := findSession(string(msg.Key)) if debug { - log.Println("Failed to read key", err, conn.RemoteAddr()) + log.Println(conn.RemoteAddr(), "session lookup", ses) } - conn.Close() - return - } - ses := findSession(string(key)) - if debug { - log.Println("Key", key, "by", conn.RemoteAddr(), "session", ses) - } + if ses == nil { + protocol.WriteMessage(conn, protocol.ResponseNotFound) + conn.Close() + return + } - if ses != nil { - ses.AddConnection(conn) - } else { + if !ses.AddConnection(conn) { + if debug { + log.Println("Failed to add", conn.RemoteAddr(), "to session", ses) + } + protocol.WriteMessage(conn, protocol.ResponseAlreadyConnected) + conn.Close() + return + } + + err := protocol.WriteMessage(conn, protocol.ResponseSuccess) + if err != nil { + if debug { + log.Println("Failed to send session join response to ", conn.RemoteAddr(), "for", ses) + } + conn.Close() + return + } + conn.SetDeadline(time.Time{}) + default: + if debug { + log.Println("Unexpected message from", conn.RemoteAddr(), message) + } + protocol.WriteMessage(conn, protocol.ResponseUnexpectedMessage) conn.Close() - return } } diff --git a/testutil/main.go b/testutil/main.go new file mode 100644 index 000000000..10c222457 --- /dev/null +++ b/testutil/main.go @@ -0,0 +1,142 @@ +// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file). + +package main + +import ( + "bufio" + "crypto/tls" + "flag" + "log" + "net" + "net/url" + "os" + "path/filepath" + "time" + + syncthingprotocol "github.com/syncthing/protocol" + "github.com/syncthing/relaysrv/client" + "github.com/syncthing/relaysrv/protocol" +) + +func main() { + log.SetOutput(os.Stdout) + log.SetFlags(log.LstdFlags | log.Lshortfile) + + var connect, relay, dir string + var join bool + + flag.StringVar(&connect, "connect", "", "Device ID to which to connect to") + flag.BoolVar(&join, "join", false, "Join relay") + flag.StringVar(&relay, "relay", "relay://127.0.0.1:22067", "Relay address") + flag.StringVar(&dir, "keys", ".", "Directory where cert.pem and key.pem is stored") + + flag.Parse() + + certFile, keyFile := filepath.Join(dir, "cert.pem"), filepath.Join(dir, "key.pem") + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + log.Fatalln("Failed to load X509 key pair:", err) + } + + id := syncthingprotocol.NewDeviceID(cert.Certificate[0]) + log.Println("ID:", id) + + uri, err := url.Parse(relay) + if err != nil { + log.Fatal(err) + } + + stdin := make(chan string) + + go stdinReader(stdin) + + if join { + log.Printf("Creating client") + relay := client.NewProtocolClient(uri, []tls.Certificate{cert}, nil) + log.Printf("Created client") + + go relay.Serve() + + recv := make(chan protocol.SessionInvitation) + + go func() { + log.Println("Starting invitation receiver") + for invite := range relay.Invitations { + select { + case recv <- invite: + log.Printf("Received invitation from %s on %s:%d", syncthingprotocol.DeviceIDFromBytes(invite.From), net.IP(invite.Address), invite.Port) + default: + log.Printf("Discarding invitation", invite) + } + } + }() + + for { + conn, err := client.JoinSession(<-recv) + if err != nil { + log.Fatalln("Failed to join", err) + } + log.Println("Joined", conn.RemoteAddr(), conn.LocalAddr()) + connectToStdio(stdin, conn) + log.Println("Finished", conn.RemoteAddr(), conn.LocalAddr()) + } + } else if connect != "" { + id, err := syncthingprotocol.DeviceIDFromString(connect) + if err != nil { + log.Fatal(err) + } + + invite, err := client.GetInvitationFromRelay(uri, id, []tls.Certificate{cert}) + if err != nil { + log.Fatal(err) + } + + log.Printf("Received invitation from %s on %s:%d", syncthingprotocol.DeviceIDFromBytes(invite.From), net.IP(invite.Address), invite.Port) + conn, err := client.JoinSession(invite) + if err != nil { + log.Fatalln("Failed to join", err) + } + log.Println("Joined", conn.RemoteAddr(), conn.LocalAddr()) + connectToStdio(stdin, conn) + log.Println("Finished", conn.RemoteAddr(), conn.LocalAddr()) + } else { + log.Fatal("Requires either join or connect") + } +} + +func stdinReader(c chan<- string) { + scanner := bufio.NewScanner(os.Stdin) + for scanner.Scan() { + c <- scanner.Text() + c <- "\n" + } +} + +func connectToStdio(stdin <-chan string, conn net.Conn) { + go func() { + + }() + + buf := make([]byte, 1024) + for { + conn.SetReadDeadline(time.Now().Add(time.Millisecond)) + n, err := conn.Read(buf[0:]) + if err != nil { + nerr, ok := err.(net.Error) + if !ok || !nerr.Timeout() { + log.Println(err) + return + } + } + os.Stdout.Write(buf[:n]) + + select { + case msg := <-stdin: + _, err := conn.Write([]byte(msg)) + if err != nil { + return + } + default: + } + } +} diff --git a/utils.go b/utils.go index 5388ba32e..7d1f6bfa4 100644 --- a/utils.go +++ b/utils.go @@ -5,7 +5,6 @@ package main import ( "errors" "net" - "time" ) func setTCPOptions(conn net.Conn) error { @@ -19,7 +18,7 @@ func setTCPOptions(conn net.Conn) error { if err := tcpConn.SetNoDelay(true); err != nil { return err } - if err := tcpConn.SetKeepAlivePeriod(60 * time.Second); err != nil { + if err := tcpConn.SetKeepAlivePeriod(networkTimeout); err != nil { return err } if err := tcpConn.SetKeepAlive(true); err != nil { @@ -27,27 +26,3 @@ func setTCPOptions(conn net.Conn) error { } return nil } - -func sendMessage(msg message, conn net.Conn) error { - header, err := msg.header.MarshalXDR() - if err != nil { - return err - } - - err = conn.SetWriteDeadline(time.Now().Add(networkTimeout)) - if err != nil { - return err - } - - _, err = conn.Write(header) - if err != nil { - return err - } - - _, err = conn.Write(msg.payload) - if err != nil { - return err - } - - return nil -} From c68c78d4120ecde3eb24f4431dcf2bab820dcb5d Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Sun, 28 Jun 2015 20:34:28 +0100 Subject: [PATCH 53/94] Do scheme validation in the client --- client/client.go | 4 ++++ client/methods.go | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/client/client.go b/client/client.go index b48320fd2..d05944aca 100644 --- a/client/client.go +++ b/client/client.go @@ -52,6 +52,10 @@ type ProtocolClient struct { } func (c *ProtocolClient) connect() error { + if c.URI.Scheme != "relay" { + return fmt.Errorf("Unsupported relay schema:", c.URI.Scheme) + } + conn, err := tls.Dial("tcp", c.URI.Host, c.config) if err != nil { return err diff --git a/client/methods.go b/client/methods.go index 1d457e294..c9b7a265c 100644 --- a/client/methods.go +++ b/client/methods.go @@ -15,6 +15,10 @@ import ( ) func GetInvitationFromRelay(uri *url.URL, id syncthingprotocol.DeviceID, certs []tls.Certificate) (protocol.SessionInvitation, error) { + if uri.Scheme != "relay" { + return protocol.SessionInvitation{}, fmt.Errorf("Unsupported relay schema:", uri.Scheme) + } + conn, err := tls.Dial("tcp", uri.Host, configForCerts(certs)) conn.SetDeadline(time.Now().Add(10 * time.Second)) if err != nil { From e1959afb6beca5c15575cf1573e9c0809b93eb49 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Sun, 28 Jun 2015 21:18:38 +0100 Subject: [PATCH 54/94] Change EOL --- client/debug.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/client/debug.go b/client/debug.go index 4a3608dec..935e9fe62 100644 --- a/client/debug.go +++ b/client/debug.go @@ -1,15 +1,15 @@ -// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file). - -package client - -import ( - "os" - "strings" - - "github.com/calmh/logger" -) - -var ( - debug = strings.Contains(os.Getenv("STTRACE"), "relay") || os.Getenv("STTRACE") == "all" - l = logger.DefaultLogger -) +// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file). + +package client + +import ( + "os" + "strings" + + "github.com/calmh/logger" +) + +var ( + debug = strings.Contains(os.Getenv("STTRACE"), "relay") || os.Getenv("STTRACE") == "all" + l = logger.DefaultLogger +) From 9dd6f848bdcd3550606158a33d6aa98de6ea0cdc Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Thu, 9 Jul 2015 22:38:21 +0100 Subject: [PATCH 55/94] Name the folder in error messages --- nativemodel_windows.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nativemodel_windows.go b/nativemodel_windows.go index 072e1278b..f1a24898c 100644 --- a/nativemodel_windows.go +++ b/nativemodel_windows.go @@ -25,12 +25,12 @@ type nativeModel struct { } func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { - fixupFiles(files) + fixupFiles(folder, files) m.next.Index(deviceID, folder, files, flags, options) } func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { - fixupFiles(files) + fixupFiles(folder, files) m.next.IndexUpdate(deviceID, folder, files, flags, options) } @@ -47,7 +47,7 @@ func (m nativeModel) Close(deviceID DeviceID, err error) { m.next.Close(deviceID, err) } -func fixupFiles(files []FileInfo) { +func fixupFiles(folder string, files []FileInfo) { for i, f := range files { if strings.ContainsAny(f.Name, disallowedCharacters) { if f.IsDeleted() { @@ -56,7 +56,7 @@ func fixupFiles(files []FileInfo) { continue } files[i].Flags |= FlagInvalid - l.Warnf("File name %q contains invalid characters; marked as invalid.", f.Name) + l.Warnf("File name %q (folder %q) contains invalid characters; marked as invalid.", f.Name, folder) } files[i].Name = filepath.FromSlash(files[i].Name) } From b05c1a5bb9b30edb7691482befc2288f7fb7ce7b Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Fri, 10 Jul 2015 16:34:54 +1000 Subject: [PATCH 56/94] Connection now needs explicit Start() --- protocol.go | 9 +++++++-- protocol_test.go | 10 ++++++++++ wireformat.go | 4 ++++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/protocol.go b/protocol.go index 50115c09c..d0e23055d 100644 --- a/protocol.go +++ b/protocol.go @@ -89,6 +89,7 @@ type Model interface { } type Connection interface { + Start() ID() DeviceID Name() string Index(folder string, files []FileInfo, flags uint32, options []Option) error @@ -161,12 +162,16 @@ func NewConnection(deviceID DeviceID, reader io.Reader, writer io.Writer, receiv compression: compress, } + return wireFormatConnection{&c} +} + +// Start creates the goroutines for sending and receiving of messages. It must +// be called exactly once after creating a connection. +func (c *rawConnection) Start() { go c.readerLoop() go c.writerLoop() go c.pingerLoop() go c.idGenerator() - - return wireFormatConnection{&c} } func (c *rawConnection) ID() DeviceID { diff --git a/protocol_test.go b/protocol_test.go index bb4fe7d95..051672411 100644 --- a/protocol_test.go +++ b/protocol_test.go @@ -68,7 +68,9 @@ func TestPing(t *testing.T) { br, bw := io.Pipe() c0 := NewConnection(c0ID, ar, bw, newTestModel(), "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) + c0.Start() c1 := NewConnection(c1ID, br, aw, newTestModel(), "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) + c1.Start() c0.ClusterConfig(ClusterConfigMessage{}) c1.ClusterConfig(ClusterConfigMessage{}) @@ -94,7 +96,9 @@ func TestPingErr(t *testing.T) { ebw := &ErrPipe{PipeWriter: *bw, max: j, err: e} c0 := NewConnection(c0ID, ar, ebw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) + c0.Start() c1 := NewConnection(c1ID, br, eaw, m1, "name", CompressAlways) + c1.Start() c0.ClusterConfig(ClusterConfigMessage{}) c1.ClusterConfig(ClusterConfigMessage{}) @@ -174,7 +178,9 @@ func TestVersionErr(t *testing.T) { br, bw := io.Pipe() c0 := NewConnection(c0ID, ar, bw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) + c0.Start() c1 := NewConnection(c1ID, br, aw, m1, "name", CompressAlways) + c1.Start() c0.ClusterConfig(ClusterConfigMessage{}) c1.ClusterConfig(ClusterConfigMessage{}) @@ -199,7 +205,9 @@ func TestTypeErr(t *testing.T) { br, bw := io.Pipe() c0 := NewConnection(c0ID, ar, bw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) + c0.Start() c1 := NewConnection(c1ID, br, aw, m1, "name", CompressAlways) + c1.Start() c0.ClusterConfig(ClusterConfigMessage{}) c1.ClusterConfig(ClusterConfigMessage{}) @@ -224,7 +232,9 @@ func TestClose(t *testing.T) { br, bw := io.Pipe() c0 := NewConnection(c0ID, ar, bw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) + c0.Start() c1 := NewConnection(c1ID, br, aw, m1, "name", CompressAlways) + c1.Start() c0.ClusterConfig(ClusterConfigMessage{}) c1.ClusterConfig(ClusterConfigMessage{}) diff --git a/wireformat.go b/wireformat.go index 9411955ba..66b02ed6f 100644 --- a/wireformat.go +++ b/wireformat.go @@ -12,6 +12,10 @@ type wireFormatConnection struct { next Connection } +func (c wireFormatConnection) Start() { + c.next.Start() +} + func (c wireFormatConnection) ID() DeviceID { return c.next.ID() } From 2505f82ce5f923480dfe9a63d16a5a87a9fa9753 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Fri, 17 Jul 2015 20:17:49 +0100 Subject: [PATCH 57/94] General cleanup --- client/methods.go | 4 ++-- protocol/packets.go | 14 ++++++++++++++ testutil/main.go | 10 +++++----- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/client/methods.go b/client/methods.go index c9b7a265c..ef6145e9c 100644 --- a/client/methods.go +++ b/client/methods.go @@ -16,7 +16,7 @@ import ( func GetInvitationFromRelay(uri *url.URL, id syncthingprotocol.DeviceID, certs []tls.Certificate) (protocol.SessionInvitation, error) { if uri.Scheme != "relay" { - return protocol.SessionInvitation{}, fmt.Errorf("Unsupported relay schema:", uri.Scheme) + return protocol.SessionInvitation{}, fmt.Errorf("Unsupported relay scheme:", uri.Scheme) } conn, err := tls.Dial("tcp", uri.Host, configForCerts(certs)) @@ -49,7 +49,7 @@ func GetInvitationFromRelay(uri *url.URL, id syncthingprotocol.DeviceID, certs [ return protocol.SessionInvitation{}, fmt.Errorf("Incorrect response code %d: %s", msg.Code, msg.Message) case protocol.SessionInvitation: if debug { - l.Debugln("Received invitation via", conn.LocalAddr()) + l.Debugln("Received invitation", msg, "via", conn.LocalAddr()) } ip := net.IP(msg.Address) if len(ip) == 0 || ip.IsUnspecified() { diff --git a/protocol/packets.go b/protocol/packets.go index 658bc536d..84316da98 100644 --- a/protocol/packets.go +++ b/protocol/packets.go @@ -5,6 +5,12 @@ package protocol +import ( + "fmt" + syncthingprotocol "github.com/syncthing/protocol" + "net" +) + const ( messageTypePing int32 = iota messageTypePong @@ -45,3 +51,11 @@ type SessionInvitation struct { Port uint16 ServerSocket bool } + +func (i *SessionInvitation) String() string { + return fmt.Sprintf("%s@%s", syncthingprotocol.DeviceIDFromBytes(i.From), i.AddressString()) +} + +func (i *SessionInvitation) AddressString() string { + return fmt.Sprintf("%s:%d", net.IP(i.Address), i.Port) +} diff --git a/testutil/main.go b/testutil/main.go index 10c222457..69dbb00a1 100644 --- a/testutil/main.go +++ b/testutil/main.go @@ -51,9 +51,9 @@ func main() { go stdinReader(stdin) if join { - log.Printf("Creating client") + log.Println("Creating client") relay := client.NewProtocolClient(uri, []tls.Certificate{cert}, nil) - log.Printf("Created client") + log.Println("Created client") go relay.Serve() @@ -64,9 +64,9 @@ func main() { for invite := range relay.Invitations { select { case recv <- invite: - log.Printf("Received invitation from %s on %s:%d", syncthingprotocol.DeviceIDFromBytes(invite.From), net.IP(invite.Address), invite.Port) + log.Println("Received invitation", invite) default: - log.Printf("Discarding invitation", invite) + log.Println("Discarding invitation", invite) } } }() @@ -91,7 +91,7 @@ func main() { log.Fatal(err) } - log.Printf("Received invitation from %s on %s:%d", syncthingprotocol.DeviceIDFromBytes(invite.From), net.IP(invite.Address), invite.Port) + log.Println("Received invitation", invite) conn, err := client.JoinSession(invite) if err != nil { log.Fatalln("Failed to join", err) From e97f75cad50e728b6938b1142781465aa9d1a6c4 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Fri, 17 Jul 2015 21:49:45 +0100 Subject: [PATCH 58/94] Change receiver type, add GoStringer --- protocol/packets.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/protocol/packets.go b/protocol/packets.go index 84316da98..7ff020115 100644 --- a/protocol/packets.go +++ b/protocol/packets.go @@ -52,10 +52,14 @@ type SessionInvitation struct { ServerSocket bool } -func (i *SessionInvitation) String() string { +func (i SessionInvitation) String() string { return fmt.Sprintf("%s@%s", syncthingprotocol.DeviceIDFromBytes(i.From), i.AddressString()) } -func (i *SessionInvitation) AddressString() string { +func (i SessionInvitation) GoString() string { + return i.String() +} + +func (i SessionInvitation) AddressString() string { return fmt.Sprintf("%s:%d", net.IP(i.Address), i.Port) } From f86946c6dfe5acec0a73e0ad4741b313de496ac4 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Fri, 17 Jul 2015 22:04:02 +0100 Subject: [PATCH 59/94] Fix bugs --- protocol_listener.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/protocol_listener.go b/protocol_listener.go index 1e18b156e..c3321aa50 100644 --- a/protocol_listener.go +++ b/protocol_listener.go @@ -166,9 +166,13 @@ func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config) { // Potentially closing a second time. close(outbox) conn.Close() - outboxesMut.Lock() - delete(outboxes, id) - outboxesMut.Unlock() + // Only delete the outbox if the client join, as it migth be a + // lookup request coming from the same client. + if joined { + outboxesMut.Lock() + delete(outboxes, id) + outboxesMut.Unlock() + } return case <-pingTicker.C: if !joined { From dab1c4cfc9cb55c00cf5c83e505931c9a1f2c9a6 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Mon, 20 Jul 2015 12:11:06 +0200 Subject: [PATCH 60/94] Build script from discosrv --- .gitignore | 2 ++ build.sh | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100755 build.sh diff --git a/.gitignore b/.gitignore index daf913b1b..b7006615a 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,5 @@ _testmain.go *.exe *.test *.prof +*.tar.gz +*.zip diff --git a/build.sh b/build.sh new file mode 100755 index 000000000..5f605e1b9 --- /dev/null +++ b/build.sh @@ -0,0 +1,37 @@ +#!/bin/bash +set -euo pipefail +set nullglob + +echo Get dependencies +go get -d + +rm -rf relaysrv-*-* + +build() { + export GOOS="$1" + export GOARCH="$2" + target="relaysrv-$GOOS-$GOARCH" + go build -v + mkdir "$target" + if [ -f relaysrv ] ; then + mv relaysrv "$target" + tar zcvf "$target.tar.gz" "$target" + fi + if [ -f relaysrv.exe ] ; then + mv relaysrv.exe "$target" + zip -r "$target.zip" "$target" + fi +} + +for goos in linux darwin windows freebsd openbsd netbsd solaris ; do + build "$goos" amd64 +done +for goos in linux windows freebsd openbsd netbsd ; do + build "$goos" 386 +done +build linux arm + +# Hack used because we run as root under Docker +if [[ ${CHOWN_USER:-} != "" ]] ; then + chown -R $CHOWN_USER . +fi From 35d20a19bc7d389d0367b9eae4769d323e5ecb13 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Mon, 20 Jul 2015 13:25:08 +0200 Subject: [PATCH 61/94] Implement global and per session rate limiting --- main.go | 22 +++++++++++++--- protocol_listener.go | 2 +- session.go | 63 +++++++++++++++++++++++++++++++++++++++----- 3 files changed, 76 insertions(+), 11 deletions(-) diff --git a/main.go b/main.go index 5ca060689..b429d94a3 100644 --- a/main.go +++ b/main.go @@ -10,6 +10,7 @@ import ( "path/filepath" "time" + "github.com/juju/ratelimit" "github.com/syncthing/relaysrv/protocol" syncthingprotocol "github.com/syncthing/protocol" @@ -26,6 +27,11 @@ var ( networkTimeout time.Duration pingInterval time.Duration messageTimeout time.Duration + + sessionLimitBps int + globalLimitBps int + sessionLimiter *ratelimit.Bucket + globalLimiter *ratelimit.Bucket ) func main() { @@ -38,6 +44,11 @@ func main() { flag.DurationVar(&networkTimeout, "network-timeout", 2*time.Minute, "Timeout for network operations") flag.DurationVar(&pingInterval, "ping-interval", time.Minute, "How often pings are sent") flag.DurationVar(&messageTimeout, "message-timeout", time.Minute, "Maximum amount of time we wait for relevant messages to arrive") + flag.IntVar(&sessionLimitBps, "per-session-rate", sessionLimitBps, "Per session rate limit, in bytes/s") + flag.IntVar(&globalLimitBps, "global-rate", globalLimitBps, "Global rate limit, in bytes/s") + flag.BoolVar(&debug, "debug", false, "Enable debug output") + + flag.Parse() if extAddress == "" { extAddress = listenSession @@ -51,10 +62,6 @@ func main() { sessionAddress = addr.IP[:] sessionPort = uint16(addr.Port) - flag.BoolVar(&debug, "debug", false, "Enable debug output") - - flag.Parse() - certFile, keyFile := filepath.Join(dir, "cert.pem"), filepath.Join(dir, "key.pem") cert, err := tls.LoadX509KeyPair(certFile, keyFile) if err != nil { @@ -83,6 +90,13 @@ func main() { log.Println("ID:", id) } + if sessionLimitBps > 0 { + sessionLimiter = ratelimit.NewBucketWithRate(float64(sessionLimitBps), int64(2*sessionLimitBps)) + } + if globalLimitBps > 0 { + globalLimiter = ratelimit.NewBucketWithRate(float64(globalLimitBps), int64(2*globalLimitBps)) + } + go sessionListener(listenSession) protocolListener(listenProtocol, tlsCfg) diff --git a/protocol_listener.go b/protocol_listener.go index c3321aa50..8825af827 100644 --- a/protocol_listener.go +++ b/protocol_listener.go @@ -130,7 +130,7 @@ func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config) { continue } - ses := newSession() + ses := newSession(sessionLimiter, globalLimiter) go ses.Serve() diff --git a/session.go b/session.go index c5a091952..c526ed5d3 100644 --- a/session.go +++ b/session.go @@ -11,6 +11,7 @@ import ( "sync" "time" + "github.com/juju/ratelimit" "github.com/syncthing/relaysrv/protocol" syncthingprotocol "github.com/syncthing/protocol" @@ -25,10 +26,12 @@ type session struct { serverkey []byte clientkey []byte + rateLimit func(bytes int64) + conns chan net.Conn } -func newSession() *session { +func newSession(sessionRateLimit, globalRateLimit *ratelimit.Bucket) *session { serverkey := make([]byte, 32) _, err := rand.Read(serverkey) if err != nil { @@ -44,6 +47,7 @@ func newSession() *session { ses := &session{ serverkey: serverkey, clientkey: clientkey, + rateLimit: makeRateLimitFunc(sessionRateLimit, globalRateLimit), conns: make(chan net.Conn), } @@ -112,12 +116,12 @@ func (s *session) Serve() { errors := make(chan error, 2) go func() { - errors <- proxy(conns[0], conns[1]) + errors <- s.proxy(conns[0], conns[1]) wg.Done() }() go func() { - errors <- proxy(conns[1], conns[0]) + errors <- s.proxy(conns[1], conns[0]) wg.Done() }() @@ -169,14 +173,15 @@ func (s *session) GetServerInvitationMessage(from syncthingprotocol.DeviceID) pr } } -func proxy(c1, c2 net.Conn) error { +func (s *session) proxy(c1, c2 net.Conn) error { if debug { log.Println("Proxy", c1.RemoteAddr(), "->", c2.RemoteAddr()) } - buf := make([]byte, 1024) + + buf := make([]byte, 65536) for { c1.SetReadDeadline(time.Now().Add(networkTimeout)) - n, err := c1.Read(buf[0:]) + n, err := c1.Read(buf) if err != nil { return err } @@ -185,6 +190,10 @@ func proxy(c1, c2 net.Conn) error { log.Printf("%d bytes from %s to %s", n, c1.RemoteAddr(), c2.RemoteAddr()) } + if s.rateLimit != nil { + s.rateLimit(int64(n)) + } + c2.SetWriteDeadline(time.Now().Add(networkTimeout)) _, err = c2.Write(buf[:n]) if err != nil { @@ -196,3 +205,45 @@ func proxy(c1, c2 net.Conn) error { func (s *session) String() string { return fmt.Sprintf("<%s/%s>", hex.EncodeToString(s.clientkey)[:5], hex.EncodeToString(s.serverkey)[:5]) } + +func makeRateLimitFunc(sessionRateLimit, globalRateLimit *ratelimit.Bucket) func(int64) { + // This may be a case of super duper premature optimization... We build an + // optimized function to do the rate limiting here based on what we need + // to do and then use it in the loop. + + if sessionRateLimit == nil && globalRateLimit == nil { + // No limiting needed. We could equally well return a func(int64){} and + // not do a nil check were we use it, but I think the nil check there + // makes it clear that there will be no limiting if none is + // configured... + return nil + } + + if sessionRateLimit == nil { + // We only have a global limiter + return func(bytes int64) { + globalRateLimit.Wait(bytes) + } + } + + if globalRateLimit == nil { + // We only have a session limiter + return func(bytes int64) { + sessionRateLimit.Wait(bytes) + } + } + + // We have both. Queue the bytes on both the global and session specific + // rate limiters. Wait for both in parallell, so that the actual send + // happens when both conditions are satisfied. In practice this just means + // wait the longer of the two times. + return func(bytes int64) { + t0 := sessionRateLimit.Take(bytes) + t1 := globalRateLimit.Take(bytes) + if t0 > t1 { + time.Sleep(t0) + } else { + time.Sleep(t1) + } + } +} From f9bd59f031caa6f246a91c5a919e2e7313962f60 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Mon, 20 Jul 2015 11:38:00 +0200 Subject: [PATCH 62/94] Style and minor fixes, main package --- protocol_listener.go | 47 +++++++++++++++++++++++++++++++------------- session_listener.go | 29 +++++++++++++++++---------- 2 files changed, 52 insertions(+), 24 deletions(-) diff --git a/protocol_listener.go b/protocol_listener.go index 8825af827..a7243ff69 100644 --- a/protocol_listener.go +++ b/protocol_listener.go @@ -27,7 +27,6 @@ func protocolListener(addr string, config *tls.Config) { for { conn, err := listener.Accept() - setTCPOptions(conn) if err != nil { if debug { log.Println(err) @@ -35,6 +34,8 @@ func protocolListener(addr string, config *tls.Config) { continue } + setTCPOptions(conn) + if debug { log.Println("Protocol listener accepted connection from", conn.RemoteAddr()) } @@ -74,16 +75,12 @@ func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config) { errors := make(chan error, 1) outbox := make(chan interface{}) - go func(conn net.Conn, message chan<- interface{}, errors chan<- error) { - for { - msg, err := protocol.ReadMessage(conn) - if err != nil { - errors <- err - return - } - messages <- msg - } - }(conn, messages, errors) + // Read messages from the connection and send them on the messages + // channel. When there is an error, send it on the error channel and + // return. Applies also when the connection gets closed, so the pattern + // below is to close the connection on error, then wait for the error + // signal from messageReader to exit. + go messageReader(conn, messages, errors) pingTicker := time.NewTicker(pingInterval) timeoutTicker := time.NewTimer(networkTimeout) @@ -96,6 +93,7 @@ func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config) { if debug { log.Printf("Message %T from %s", message, id) } + switch msg := message.(type) { case protocol.JoinRelayRequest: outboxesMut.RLock() @@ -116,6 +114,7 @@ func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config) { joined = true protocol.WriteMessage(conn, protocol.ResponseSuccess) + case protocol.ConnectRequest: requestedPeer := syncthingprotocol.DeviceIDFromBytes(msg.ID) outboxesMut.RLock() @@ -151,7 +150,10 @@ func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config) { log.Println("Sent invitation from", id, "to", requestedPeer) } conn.Close() + case protocol.Pong: + // Nothing + default: if debug { log.Printf("Unknown message %s: %T", id, message) @@ -159,21 +161,25 @@ func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config) { protocol.WriteMessage(conn, protocol.ResponseUnexpectedMessage) conn.Close() } + case err := <-errors: if debug { log.Printf("Closing connection %s: %s", id, err) } - // Potentially closing a second time. close(outbox) + + // Potentially closing a second time. conn.Close() - // Only delete the outbox if the client join, as it migth be a - // lookup request coming from the same client. + + // Only delete the outbox if the client is joined, as it might be + // a lookup request coming from the same client. if joined { outboxesMut.Lock() delete(outboxes, id) outboxesMut.Unlock() } return + case <-pingTicker.C: if !joined { if debug { @@ -189,6 +195,7 @@ func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config) { } conn.Close() } + case <-timeoutTicker.C: // We should receive a error from the reader loop, which will cause // us to quit this loop. @@ -196,6 +203,7 @@ func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config) { log.Printf("%s timed out", id) } conn.Close() + case msg := <-outbox: if debug { log.Printf("Sending message %T to %s", msg, id) @@ -209,3 +217,14 @@ func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config) { } } } + +func messageReader(conn net.Conn, messages chan<- interface{}, errors chan<- error) { + for { + msg, err := protocol.ReadMessage(conn) + if err != nil { + errors <- err + return + } + messages <- msg + } +} diff --git a/session_listener.go b/session_listener.go index 6159ceef5..2f6bae9ab 100644 --- a/session_listener.go +++ b/session_listener.go @@ -18,7 +18,6 @@ func sessionListener(addr string) { for { conn, err := listener.Accept() - setTCPOptions(conn) if err != nil { if debug { log.Println(err) @@ -26,6 +25,8 @@ func sessionListener(addr string) { continue } + setTCPOptions(conn) + if debug { log.Println("Session listener accepted connection from", conn.RemoteAddr()) } @@ -35,10 +36,17 @@ func sessionListener(addr string) { } func sessionConnectionHandler(conn net.Conn) { - conn.SetDeadline(time.Now().Add(messageTimeout)) + defer conn.Close() + + if err := conn.SetDeadline(time.Now().Add(messageTimeout)); err != nil { + if debug { + log.Println("Weird error setting deadline:", err, "on", conn.RemoteAddr()) + } + return + } + message, err := protocol.ReadMessage(conn) if err != nil { - conn.Close() return } @@ -51,7 +59,6 @@ func sessionConnectionHandler(conn net.Conn) { if ses == nil { protocol.WriteMessage(conn, protocol.ResponseNotFound) - conn.Close() return } @@ -60,24 +67,26 @@ func sessionConnectionHandler(conn net.Conn) { log.Println("Failed to add", conn.RemoteAddr(), "to session", ses) } protocol.WriteMessage(conn, protocol.ResponseAlreadyConnected) - conn.Close() return } - err := protocol.WriteMessage(conn, protocol.ResponseSuccess) - if err != nil { + if err := protocol.WriteMessage(conn, protocol.ResponseSuccess); err != nil { if debug { log.Println("Failed to send session join response to ", conn.RemoteAddr(), "for", ses) } - conn.Close() return } - conn.SetDeadline(time.Time{}) + + if err := conn.SetDeadline(time.Time{}); err != nil { + if debug { + log.Println("Weird error setting deadline:", err, "on", conn.RemoteAddr()) + } + return + } default: if debug { log.Println("Unexpected message from", conn.RemoteAddr(), message) } protocol.WriteMessage(conn, protocol.ResponseUnexpectedMessage) - conn.Close() } } From 049d92b52581015a759593e64314fcbfa2462ad2 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Mon, 20 Jul 2015 11:56:10 +0200 Subject: [PATCH 63/94] Style and minor fixes, client package --- client/client.go | 282 ++++++++++++++++++++++++----------------------- 1 file changed, 143 insertions(+), 139 deletions(-) diff --git a/client/client.go b/client/client.go index d05944aca..7169e6a8b 100644 --- a/client/client.go +++ b/client/client.go @@ -14,27 +14,6 @@ import ( "github.com/syncthing/relaysrv/protocol" ) -func NewProtocolClient(uri *url.URL, certs []tls.Certificate, invitations chan protocol.SessionInvitation) ProtocolClient { - closeInvitationsOnFinish := false - if invitations == nil { - closeInvitationsOnFinish = true - invitations = make(chan protocol.SessionInvitation) - } - return ProtocolClient{ - URI: uri, - Invitations: invitations, - - closeInvitationsOnFinish: closeInvitationsOnFinish, - - config: configForCerts(certs), - - timeout: time.Minute * 2, - - stop: make(chan struct{}), - stopped: make(chan struct{}), - } -} - type ProtocolClient struct { URI *url.URL Invitations chan protocol.SessionInvitation @@ -51,6 +30,129 @@ type ProtocolClient struct { conn *tls.Conn } +func NewProtocolClient(uri *url.URL, certs []tls.Certificate, invitations chan protocol.SessionInvitation) *ProtocolClient { + closeInvitationsOnFinish := false + if invitations == nil { + closeInvitationsOnFinish = true + invitations = make(chan protocol.SessionInvitation) + } + + return &ProtocolClient{ + URI: uri, + Invitations: invitations, + + closeInvitationsOnFinish: closeInvitationsOnFinish, + + config: configForCerts(certs), + + timeout: time.Minute * 2, + + stop: make(chan struct{}), + stopped: make(chan struct{}), + } +} + +func (c *ProtocolClient) Serve() { + c.stop = make(chan struct{}) + c.stopped = make(chan struct{}) + defer close(c.stopped) + + if err := c.connect(); err != nil { + l.Infoln("Relay connect:", err) + return + } + + if debug { + l.Debugln(c, "connected", c.conn.RemoteAddr()) + } + + if err := c.join(); err != nil { + c.conn.Close() + l.Infoln("Relay join:", err) + return + } + + if err := c.conn.SetDeadline(time.Time{}); err != nil { + l.Infoln("Relay set deadline:", err) + return + } + + if debug { + l.Debugln(c, "joined", c.conn.RemoteAddr(), "via", c.conn.LocalAddr()) + } + + defer c.cleanup() + + messages := make(chan interface{}) + errors := make(chan error, 1) + + go messageReader(c.conn, messages, errors) + + timeout := time.NewTimer(c.timeout) + + for { + select { + case message := <-messages: + timeout.Reset(c.timeout) + if debug { + log.Printf("%s received message %T", c, message) + } + + switch msg := message.(type) { + case protocol.Ping: + if err := protocol.WriteMessage(c.conn, protocol.Pong{}); err != nil { + l.Infoln("Relay write:", err) + return + + } + if debug { + l.Debugln(c, "sent pong") + } + + case protocol.SessionInvitation: + ip := net.IP(msg.Address) + if len(ip) == 0 || ip.IsUnspecified() { + msg.Address = c.conn.RemoteAddr().(*net.TCPAddr).IP[:] + } + c.Invitations <- msg + + default: + l.Infoln("Relay: protocol error: unexpected message %v", msg) + return + } + + case <-c.stop: + if debug { + l.Debugln(c, "stopping") + } + return + + case err := <-errors: + l.Infoln("Relay received:", err) + return + + case <-timeout.C: + if debug { + l.Debugln(c, "timed out") + } + return + } + } +} + +func (c *ProtocolClient) Stop() { + if c.stop == nil { + return + } + + close(c.stop) + <-c.stopped +} + +func (c *ProtocolClient) String() string { + return fmt.Sprintf("ProtocolClient@%p", c) +} + func (c *ProtocolClient) connect() error { if c.URI.Scheme != "relay" { return fmt.Errorf("Unsupported relay schema:", c.URI.Scheme) @@ -61,9 +163,13 @@ func (c *ProtocolClient) connect() error { return err } - conn.SetDeadline(time.Now().Add(10 * time.Second)) + if err := conn.SetDeadline(time.Now().Add(10 * time.Second)); err != nil { + conn.Close() + return err + } if err := performHandshakeAndValidation(conn, c.URI); err != nil { + conn.Close() return err } @@ -71,101 +177,6 @@ func (c *ProtocolClient) connect() error { return nil } -func (c *ProtocolClient) Serve() { - if err := c.connect(); err != nil { - panic(err) - } - - if debug { - l.Debugln(c, "connected", c.conn.RemoteAddr()) - } - - if err := c.join(); err != nil { - c.conn.Close() - panic(err) - } - - c.conn.SetDeadline(time.Time{}) - - if debug { - l.Debugln(c, "joined", c.conn.RemoteAddr(), "via", c.conn.LocalAddr()) - } - - c.stop = make(chan struct{}) - c.stopped = make(chan struct{}) - - defer c.cleanup() - - messages := make(chan interface{}) - errors := make(chan error, 1) - - go func(conn net.Conn, message chan<- interface{}, errors chan<- error) { - for { - msg, err := protocol.ReadMessage(conn) - if err != nil { - errors <- err - return - } - messages <- msg - } - }(c.conn, messages, errors) - - timeout := time.NewTimer(c.timeout) - for { - select { - case message := <-messages: - timeout.Reset(c.timeout) - if debug { - log.Printf("%s received message %T", c, message) - } - switch msg := message.(type) { - case protocol.Ping: - if err := protocol.WriteMessage(c.conn, protocol.Pong{}); err != nil { - panic(err) - } - if debug { - l.Debugln(c, "sent pong") - } - case protocol.SessionInvitation: - ip := net.IP(msg.Address) - if len(ip) == 0 || ip.IsUnspecified() { - msg.Address = c.conn.RemoteAddr().(*net.TCPAddr).IP[:] - } - c.Invitations <- msg - default: - panic(fmt.Errorf("protocol error: unexpected message %v", msg)) - } - case <-c.stop: - if debug { - l.Debugln(c, "stopping") - } - break - case err := <-errors: - panic(err) - case <-timeout.C: - if debug { - l.Debugln(c, "timed out") - } - return - } - } - - c.stopped <- struct{}{} -} - -func (c *ProtocolClient) Stop() { - if c.stop == nil { - return - } - - c.stop <- struct{}{} - <-c.stopped -} - -func (c *ProtocolClient) String() string { - return fmt.Sprintf("ProtocolClient@%p", c) -} - func (c *ProtocolClient) cleanup() { if c.closeInvitationsOnFinish { close(c.Invitations) @@ -176,24 +187,11 @@ func (c *ProtocolClient) cleanup() { l.Debugln(c, "cleaning up") } - if c.stop != nil { - close(c.stop) - c.stop = nil - } - - if c.stopped != nil { - close(c.stopped) - c.stopped = nil - } - - if c.conn != nil { - c.conn.Close() - } + c.conn.Close() } func (c *ProtocolClient) join() error { - err := protocol.WriteMessage(c.conn, protocol.JoinRelayRequest{}) - if err != nil { + if err := protocol.WriteMessage(c.conn, protocol.JoinRelayRequest{}); err != nil { return err } @@ -207,6 +205,7 @@ func (c *ProtocolClient) join() error { if msg.Code != 0 { return fmt.Errorf("Incorrect response code %d: %s", msg.Code, msg.Message) } + default: return fmt.Errorf("protocol error: expecting response got %v", msg) } @@ -215,15 +214,12 @@ func (c *ProtocolClient) join() error { } func performHandshakeAndValidation(conn *tls.Conn, uri *url.URL) error { - err := conn.Handshake() - if err != nil { - conn.Close() + if err := conn.Handshake(); err != nil { return err } cs := conn.ConnectionState() if !cs.NegotiatedProtocolIsMutual || cs.NegotiatedProtocol != protocol.ProtocolName { - conn.Close() return fmt.Errorf("protocol negotiation error") } @@ -232,22 +228,30 @@ func performHandshakeAndValidation(conn *tls.Conn, uri *url.URL) error { if relayIDs != "" { relayID, err := syncthingprotocol.DeviceIDFromString(relayIDs) if err != nil { - conn.Close() return fmt.Errorf("relay address contains invalid verification id: %s", err) } certs := cs.PeerCertificates if cl := len(certs); cl != 1 { - conn.Close() return fmt.Errorf("unexpected certificate count: %d", cl) } remoteID := syncthingprotocol.NewDeviceID(certs[0].Raw) if remoteID != relayID { - conn.Close() return fmt.Errorf("relay id does not match. Expected %v got %v", relayID, remoteID) } } return nil } + +func messageReader(conn net.Conn, messages chan<- interface{}, errors chan<- error) { + for { + msg, err := protocol.ReadMessage(conn) + if err != nil { + errors <- err + return + } + messages <- msg + } +} From 516d88b0725588516d1ff75afe4dc5677b844342 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Tue, 21 Jul 2015 15:12:19 +0200 Subject: [PATCH 64/94] Base for better conflict resolution --- conflict_test.go | 23 +++++++++++++++++++++++ message.go | 25 +++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 conflict_test.go diff --git a/conflict_test.go b/conflict_test.go new file mode 100644 index 000000000..ef5c44d7e --- /dev/null +++ b/conflict_test.go @@ -0,0 +1,23 @@ +// Copyright (C) 2015 The Protocol Authors. + +package protocol + +import "testing" + +func TestWinsConflict(t *testing.T) { + testcases := [][2]FileInfo{ + // The first should always win over the second + {{Modified: 42}, {Modified: 41}}, + {{Modified: 41}, {Modified: 42, Flags: FlagDeleted}}, + {{Modified: 41, Version: Vector{{42, 2}, {43, 1}}}, {Modified: 41, Version: Vector{{42, 1}, {43, 2}}}}, + } + + for _, tc := range testcases { + if !tc[0].WinsConflict(tc[1]) { + t.Errorf("%v should win over %v", tc[0], tc[1]) + } + if tc[1].WinsConflict(tc[0]) { + t.Errorf("%v should not win over %v", tc[1], tc[0]) + } + } +} diff --git a/message.go b/message.go index c2d898944..49df7d4fa 100644 --- a/message.go +++ b/message.go @@ -58,6 +58,31 @@ func (f FileInfo) HasPermissionBits() bool { return f.Flags&FlagNoPermBits == 0 } +// WinsConflict returns true if "f" is the one to choose when it is in +// conflict with "other". +func (f FileInfo) WinsConflict(other FileInfo) bool { + // If a modification is in conflict with a delete, we pick the + // modification. + if !f.IsDeleted() && other.IsDeleted() { + return true + } + if f.IsDeleted() && !other.IsDeleted() { + return false + } + + // The one with the newer modification time wins. + if f.Modified > other.Modified { + return true + } + if f.Modified < other.Modified { + return false + } + + // The modification times were equal. Use the device ID in the version + // vector as tie breaker. + return f.Version.Compare(other.Version) == ConcurrentGreater +} + type BlockInfo struct { Offset int64 // noencode (cache only) Size int32 From 78ef42daa1943b4bd012e6ec7906b49755574967 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Wed, 22 Jul 2015 22:34:05 +0100 Subject: [PATCH 65/94] Add ability to lookup relay status --- client/client.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/client/client.go b/client/client.go index 7169e6a8b..48e97b400 100644 --- a/client/client.go +++ b/client/client.go @@ -12,6 +12,7 @@ import ( syncthingprotocol "github.com/syncthing/protocol" "github.com/syncthing/relaysrv/protocol" + "github.com/syncthing/syncthing/internal/sync" ) type ProtocolClient struct { @@ -28,6 +29,9 @@ type ProtocolClient struct { stopped chan struct{} conn *tls.Conn + + mut sync.RWMutex + connected bool } func NewProtocolClient(uri *url.URL, certs []tls.Certificate, invitations chan protocol.SessionInvitation) *ProtocolClient { @@ -49,6 +53,9 @@ func NewProtocolClient(uri *url.URL, certs []tls.Certificate, invitations chan p stop: make(chan struct{}), stopped: make(chan struct{}), + + mut: sync.NewRWMutex(), + connected: false, } } @@ -82,6 +89,9 @@ func (c *ProtocolClient) Serve() { } defer c.cleanup() + c.mut.Lock() + c.connected = true + c.mut.Unlock() messages := make(chan interface{}) errors := make(chan error, 1) @@ -149,6 +159,13 @@ func (c *ProtocolClient) Stop() { <-c.stopped } +func (c *ProtocolClient) StatusOK() bool { + c.mut.RLock() + con := c.connected + c.mut.RUnlock() + return con +} + func (c *ProtocolClient) String() string { return fmt.Sprintf("ProtocolClient@%p", c) } @@ -187,6 +204,10 @@ func (c *ProtocolClient) cleanup() { l.Debugln(c, "cleaning up") } + c.mut.Lock() + c.connected = false + c.mut.Unlock() + c.conn.Close() } From eb29989dff2810d7317229f7742e8e58d116dc73 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Thu, 23 Jul 2015 20:53:16 +0100 Subject: [PATCH 66/94] Connection errors are debug errors --- client/client.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/client/client.go b/client/client.go index 48e97b400..a1924e3ea 100644 --- a/client/client.go +++ b/client/client.go @@ -65,7 +65,9 @@ func (c *ProtocolClient) Serve() { defer close(c.stopped) if err := c.connect(); err != nil { - l.Infoln("Relay connect:", err) + if debug { + l.Debugln("Relay connect:", err) + } return } From ebcdea63c07327a342f65415bbadc497462b8f1f Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Wed, 29 Jul 2015 21:23:43 +0100 Subject: [PATCH 67/94] Use sync.Pool for response buffers --- common_test.go | 5 ++- nativemodel_darwin.go | 4 +- nativemodel_unix.go | 4 +- nativemodel_windows.go | 4 +- protocol.go | 91 ++++++++++++++++++++++++++++-------------- 5 files changed, 71 insertions(+), 37 deletions(-) diff --git a/common_test.go b/common_test.go index f46b6a8da..706a3b877 100644 --- a/common_test.go +++ b/common_test.go @@ -31,7 +31,7 @@ func (t *TestModel) Index(deviceID DeviceID, folder string, files []FileInfo, fl func (t *TestModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { } -func (t *TestModel) Request(deviceID DeviceID, folder, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) { +func (t *TestModel) Request(deviceID DeviceID, folder, name string, offset int64, size int, hash []byte, flags uint32, options []Option, buf []byte) error { t.folder = folder t.name = name t.offset = offset @@ -39,7 +39,8 @@ func (t *TestModel) Request(deviceID DeviceID, folder, name string, offset int64 t.hash = hash t.flags = flags t.options = options - return t.data, nil + copy(buf, t.data) + return nil } func (t *TestModel) Close(deviceID DeviceID, err error) { diff --git a/nativemodel_darwin.go b/nativemodel_darwin.go index 502a71f23..eb755a6e4 100644 --- a/nativemodel_darwin.go +++ b/nativemodel_darwin.go @@ -26,9 +26,9 @@ func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileI m.next.IndexUpdate(deviceID, folder, files, flags, options) } -func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) { +func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, flags uint32, options []Option, buf []byte) error { name = norm.NFD.String(name) - return m.next.Request(deviceID, folder, name, offset, size, hash, flags, options) + return m.next.Request(deviceID, folder, name, offset, hash, flags, options, buf) } func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) { diff --git a/nativemodel_unix.go b/nativemodel_unix.go index 21585e308..0611865e1 100644 --- a/nativemodel_unix.go +++ b/nativemodel_unix.go @@ -18,8 +18,8 @@ func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileI m.next.IndexUpdate(deviceID, folder, files, flags, options) } -func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) { - return m.next.Request(deviceID, folder, name, offset, size, hash, flags, options) +func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, flags uint32, options []Option, buf []byte) error { + return m.next.Request(deviceID, folder, name, offset, hash, flags, options, buf) } func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) { diff --git a/nativemodel_windows.go b/nativemodel_windows.go index f1a24898c..36a1d2749 100644 --- a/nativemodel_windows.go +++ b/nativemodel_windows.go @@ -34,9 +34,9 @@ func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileI m.next.IndexUpdate(deviceID, folder, files, flags, options) } -func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) { +func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, flags uint32, options []Option, buf []byte) error { name = filepath.FromSlash(name) - return m.next.Request(deviceID, folder, name, offset, size, hash, flags, options) + return m.next.Request(deviceID, folder, name, offset, hash, flags, options, buf) } func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) { diff --git a/protocol.go b/protocol.go index d0e23055d..8b41c0138 100644 --- a/protocol.go +++ b/protocol.go @@ -81,7 +81,7 @@ type Model interface { // An index update was received from the peer device IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) // A request was made by the peer device - Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) + Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, flags uint32, options []Option, buf []byte) error // A cluster configuration message was received ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) // The peer device closed the connection @@ -112,11 +112,11 @@ type rawConnection struct { idxMut sync.Mutex // ensures serialization of Index calls - nextID chan int - outbox chan hdrMsg - closed chan struct{} - once sync.Once - + nextID chan int + outbox chan hdrMsg + closed chan struct{} + once sync.Once + pool sync.Pool compression Compression rdbuf0 []byte // used & reused by readMessage @@ -129,8 +129,9 @@ type asyncResult struct { } type hdrMsg struct { - hdr header - msg encodable + hdr header + msg encodable + done chan struct{} } type encodable interface { @@ -151,14 +152,19 @@ func NewConnection(deviceID DeviceID, reader io.Reader, writer io.Writer, receiv cw := &countingWriter{Writer: writer} c := rawConnection{ - id: deviceID, - name: name, - receiver: nativeModel{receiver}, - cr: cr, - cw: cw, - outbox: make(chan hdrMsg), - nextID: make(chan int), - closed: make(chan struct{}), + id: deviceID, + name: name, + receiver: nativeModel{receiver}, + cr: cr, + cw: cw, + outbox: make(chan hdrMsg), + nextID: make(chan int), + closed: make(chan struct{}), + pool: sync.Pool{ + New: func() interface{} { + return make([]byte, BlockSize) + }, + }, compression: compress, } @@ -195,7 +201,7 @@ func (c *rawConnection) Index(folder string, idx []FileInfo, flags uint32, optio Files: idx, Flags: flags, Options: options, - }) + }, nil) c.idxMut.Unlock() return nil } @@ -213,7 +219,7 @@ func (c *rawConnection) IndexUpdate(folder string, idx []FileInfo, flags uint32, Files: idx, Flags: flags, Options: options, - }) + }, nil) c.idxMut.Unlock() return nil } @@ -243,7 +249,7 @@ func (c *rawConnection) Request(folder string, name string, offset int64, size i Hash: hash, Flags: flags, Options: options, - }) + }, nil) if !ok { return nil, ErrClosed } @@ -257,7 +263,7 @@ func (c *rawConnection) Request(folder string, name string, offset int64, size i // ClusterConfig send the cluster configuration message to the peer and returns any error func (c *rawConnection) ClusterConfig(config ClusterConfigMessage) { - c.send(-1, messageTypeClusterConfig, config) + c.send(-1, messageTypeClusterConfig, config, nil) } func (c *rawConnection) ping() bool { @@ -273,7 +279,7 @@ func (c *rawConnection) ping() bool { c.awaiting[id] = rc c.awaitingMut.Unlock() - ok := c.send(id, messageTypePing, nil) + ok := c.send(id, messageTypePing, nil, nil) if !ok { return false } @@ -342,7 +348,7 @@ func (c *rawConnection) readerLoop() (err error) { if state != stateReady { return fmt.Errorf("protocol error: ping message in state %d", state) } - c.send(hdr.msgID, messageTypePong, pongMessage{}) + c.send(hdr.msgID, messageTypePong, pongMessage{}, nil) case pongMessage: if state != stateReady { @@ -519,12 +525,36 @@ func filterIndexMessageFiles(fs []FileInfo) []FileInfo { } func (c *rawConnection) handleRequest(msgID int, req RequestMessage) { - data, err := c.receiver.Request(c.id, req.Folder, req.Name, int64(req.Offset), int(req.Size), req.Hash, req.Flags, req.Options) + size := int(req.Size) + usePool := size <= BlockSize - c.send(msgID, messageTypeResponse, ResponseMessage{ - Data: data, - Code: errorToCode(err), - }) + var buf []byte + var done chan struct{} + + if usePool { + buf = c.pool.Get().([]byte)[:size] + done = make(chan struct{}) + } else { + buf = make([]byte, size) + } + + err := c.receiver.Request(c.id, req.Folder, req.Name, int64(req.Offset), req.Hash, req.Flags, req.Options, buf) + if err != nil { + c.send(msgID, messageTypeResponse, ResponseMessage{ + Data: nil, + Code: errorToCode(err), + }, done) + } else { + c.send(msgID, messageTypeResponse, ResponseMessage{ + Data: buf, + Code: errorToCode(err), + }, done) + } + + if usePool { + <-done + c.pool.Put(buf) + } } func (c *rawConnection) handleResponse(msgID int, resp ResponseMessage) { @@ -547,7 +577,7 @@ func (c *rawConnection) handlePong(msgID int) { c.awaitingMut.Unlock() } -func (c *rawConnection) send(msgID int, msgType int, msg encodable) bool { +func (c *rawConnection) send(msgID int, msgType int, msg encodable, done chan struct{}) bool { if msgID < 0 { select { case id := <-c.nextID: @@ -564,7 +594,7 @@ func (c *rawConnection) send(msgID int, msgType int, msg encodable) bool { } select { - case c.outbox <- hdrMsg{hdr, msg}: + case c.outbox <- hdrMsg{hdr, msg, done}: return true case <-c.closed: return false @@ -583,6 +613,9 @@ func (c *rawConnection) writerLoop() { if hm.msg != nil { // Uncompressed message in uncBuf uncBuf, err = hm.msg.AppendXDR(uncBuf[:0]) + if hm.done != nil { + close(hm.done) + } if err != nil { c.close(err) return From a6936982794bf84da01bcea26d618a34708a7ea7 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Sun, 9 Aug 2015 10:00:28 +0200 Subject: [PATCH 68/94] Mend tests --- common_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common_test.go b/common_test.go index 706a3b877..8f1028078 100644 --- a/common_test.go +++ b/common_test.go @@ -31,11 +31,11 @@ func (t *TestModel) Index(deviceID DeviceID, folder string, files []FileInfo, fl func (t *TestModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { } -func (t *TestModel) Request(deviceID DeviceID, folder, name string, offset int64, size int, hash []byte, flags uint32, options []Option, buf []byte) error { +func (t *TestModel) Request(deviceID DeviceID, folder, name string, offset int64, hash []byte, flags uint32, options []Option, buf []byte) error { t.folder = folder t.name = name t.offset = offset - t.size = size + t.size = len(buf) t.hash = hash t.flags = flags t.options = options From c6f5075721db1a7fd1e770dac92a9ef9e6ef4959 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Tue, 18 Aug 2015 08:37:23 +0200 Subject: [PATCH 69/94] Enable testing with go-fuzz --- fuzz.go | 70 +++++++++++++++++++++++++++++++++++++++++ fuzz_test.go | 89 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 159 insertions(+) create mode 100644 fuzz.go create mode 100644 fuzz_test.go diff --git a/fuzz.go b/fuzz.go new file mode 100644 index 000000000..9b82abe7c --- /dev/null +++ b/fuzz.go @@ -0,0 +1,70 @@ +// Copyright (C) 2015 The Protocol Authors. + +// +build gofuzz + +package protocol + +import ( + "bytes" + "encoding/binary" + "encoding/hex" + "fmt" + "reflect" + "sync" +) + +func Fuzz(data []byte) int { + // Regenerate the length, or we'll most commonly exit quickly due to an + // unexpected eof which is unintestering. + if len(data) > 8 { + binary.BigEndian.PutUint32(data[4:], uint32(len(data))-8) + } + + // Setup a rawConnection we'll use to parse the message. + c := rawConnection{ + cr: &countingReader{Reader: bytes.NewReader(data)}, + closed: make(chan struct{}), + pool: sync.Pool{ + New: func() interface{} { + return make([]byte, BlockSize) + }, + }, + } + + // Attempt to parse the message. + hdr, msg, err := c.readMessage() + if err != nil { + return 0 + } + + // If parsing worked, attempt to encode it again. + newBs, err := msg.AppendXDR(nil) + if err != nil { + panic("not encodable") + } + + // Create an appriate header for the re-encoding. + newMsg := make([]byte, 8) + binary.BigEndian.PutUint32(newMsg, encodeHeader(hdr)) + binary.BigEndian.PutUint32(newMsg[4:], uint32(len(newBs))) + newMsg = append(newMsg, newBs...) + + // Use the rawConnection to parse the re-encoding. + c.cr = &countingReader{Reader: bytes.NewReader(newMsg)} + hdr2, msg2, err := c.readMessage() + if err != nil { + fmt.Println("Initial:\n" + hex.Dump(data)) + fmt.Println("New:\n" + hex.Dump(newMsg)) + panic("not parseable after re-encode: " + err.Error()) + } + + // Make sure the data is the same as it was before. + if hdr != hdr2 { + panic("headers differ") + } + if !reflect.DeepEqual(msg, msg2) { + panic("contents differ") + } + + return 1 +} diff --git a/fuzz_test.go b/fuzz_test.go new file mode 100644 index 000000000..65c2d9010 --- /dev/null +++ b/fuzz_test.go @@ -0,0 +1,89 @@ +// Copyright (C) 2015 The Protocol Authors. + +// +build gofuzz + +package protocol + +import ( + "encoding/binary" + "fmt" + "io/ioutil" + "os" + "strings" + "testing" + "testing/quick" +) + +// This can be used to generate a corpus of valid messages as a starting point +// for the fuzzer. +func TestGenerateCorpus(t *testing.T) { + t.Skip("Use to generate initial corpus only") + + n := 0 + check := func(idx IndexMessage) bool { + for i := range idx.Options { + if len(idx.Options[i].Key) > 64 { + idx.Options[i].Key = idx.Options[i].Key[:64] + } + } + hdr := header{ + version: 0, + msgID: 42, + msgType: messageTypeIndex, + compression: false, + } + + msgBs := idx.MustMarshalXDR() + + buf := make([]byte, 8) + binary.BigEndian.PutUint32(buf, encodeHeader(hdr)) + binary.BigEndian.PutUint32(buf[4:], uint32(len(msgBs))) + buf = append(buf, msgBs...) + + ioutil.WriteFile(fmt.Sprintf("testdata/corpus/test-%03d.xdr", n), buf, 0644) + n++ + return true + } + + if err := quick.Check(check, &quick.Config{MaxCount: 1000}); err != nil { + t.Fatal(err) + } +} + +// Tests any crashers found by the fuzzer, for closer investigation. +func TestCrashers(t *testing.T) { + testFiles(t, "testdata/crashers") +} + +// Tests the entire corpus, which should PASS before the fuzzer starts +// fuzzing. +func TestCorpus(t *testing.T) { + testFiles(t, "testdata/corpus") +} + +func testFiles(t *testing.T, dir string) { + fd, err := os.Open(dir) + if err != nil { + t.Fatal(err) + } + crashers, err := fd.Readdirnames(-1) + if err != nil { + t.Fatal(err) + } + for _, name := range crashers { + if strings.HasSuffix(name, ".output") { + continue + } + if strings.HasSuffix(name, ".quoted") { + continue + } + + t.Log(name) + crasher, err := ioutil.ReadFile(dir + "/" + name) + if err != nil { + t.Fatal(err) + } + + Fuzz(crasher) + } +} From f769df16e8b6ef2236183f8eab1b9ca22a94408c Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Tue, 18 Aug 2015 08:38:06 +0200 Subject: [PATCH 70/94] Reject unreasonably large messages We allocate a []byte to read the message into, so if the header says the messages is several gigabytes large we may run into trouble. In reality, a message should never be that large so we impose a limit. --- protocol.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/protocol.go b/protocol.go index 8b41c0138..420b20859 100644 --- a/protocol.go +++ b/protocol.go @@ -15,7 +15,11 @@ import ( ) const ( - BlockSize = 128 * 1024 + // Data block size (128 KiB) + BlockSize = 128 << 10 + + // We reject messages larger than this when encountered on the wire. (64 MiB) + MaxMessageLen = 64 << 20 ) const ( @@ -383,6 +387,11 @@ func (c *rawConnection) readMessage() (hdr header, msg encodable, err error) { l.Debugf("read header %v (msglen=%d)", hdr, msglen) } + if msglen > MaxMessageLen { + err = fmt.Errorf("message length %d exceeds maximum %d", msglen, MaxMessageLen) + return + } + if hdr.version != 0 { err = fmt.Errorf("unknown protocol version 0x%x", hdr.version) return From 9c8b907ff195e46ac4ba0688104405f896104ccf Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Tue, 18 Aug 2015 08:41:34 +0200 Subject: [PATCH 71/94] All slice types must have limits The XDR unmarshaller allocates a []T when it sees a slice type and reads the expected length, so we must always limit the length in order to avoid allocating too much memory when encountering corruption. --- message.go | 14 +++++++------- message_xdr.go | 40 ++++++++++++++++++++++++++++++++-------- vector_xdr.go | 5 +++++ 3 files changed, 44 insertions(+), 15 deletions(-) diff --git a/message.go b/message.go index 49df7d4fa..0cfeaa381 100644 --- a/message.go +++ b/message.go @@ -9,7 +9,7 @@ import "fmt" type IndexMessage struct { Folder string - Files []FileInfo + Files []FileInfo // max:1000000 Flags uint32 Options []Option // max:64 } @@ -20,7 +20,7 @@ type FileInfo struct { Modified int64 Version Vector LocalVersion int64 - Blocks []BlockInfo + Blocks []BlockInfo // max:1000000 } func (f FileInfo) String() string { @@ -109,9 +109,9 @@ type ResponseMessage struct { } type ClusterConfigMessage struct { - ClientName string // max:64 - ClientVersion string // max:64 - Folders []Folder + ClientName string // max:64 + ClientVersion string // max:64 + Folders []Folder // max:1000000 Options []Option // max:64 } @@ -125,8 +125,8 @@ func (o *ClusterConfigMessage) GetOption(key string) string { } type Folder struct { - ID string // max:64 - Devices []Device + ID string // max:64 + Devices []Device // max:1000000 Flags uint32 Options []Option // max:64 } diff --git a/message_xdr.go b/message_xdr.go index 68d01b696..876fbb77c 100644 --- a/message_xdr.go +++ b/message_xdr.go @@ -42,7 +42,7 @@ IndexMessage Structure: struct IndexMessage { string Folder<>; - FileInfo Files<>; + FileInfo Files<1000000>; unsigned int Flags; Option Options<64>; } @@ -75,6 +75,9 @@ func (o IndexMessage) AppendXDR(bs []byte) ([]byte, error) { func (o IndexMessage) EncodeXDRInto(xw *xdr.Writer) (int, error) { xw.WriteString(o.Folder) + if l := len(o.Files); l > 1000000 { + return xw.Tot(), xdr.ElementSizeExceeded("Files", l, 1000000) + } xw.WriteUint32(uint32(len(o.Files))) for i := range o.Files { _, err := o.Files[i].EncodeXDRInto(xw) @@ -111,7 +114,10 @@ func (o *IndexMessage) DecodeXDRFrom(xr *xdr.Reader) error { o.Folder = xr.ReadString() _FilesSize := int(xr.ReadUint32()) if _FilesSize < 0 { - return xdr.ElementSizeExceeded("Files", _FilesSize, 0) + return xdr.ElementSizeExceeded("Files", _FilesSize, 1000000) + } + if _FilesSize > 1000000 { + return xdr.ElementSizeExceeded("Files", _FilesSize, 1000000) } o.Files = make([]FileInfo, _FilesSize) for i := range o.Files { @@ -173,7 +179,7 @@ struct FileInfo { hyper Modified; Vector Version; hyper LocalVersion; - BlockInfo Blocks<>; + BlockInfo Blocks<1000000>; } */ @@ -214,6 +220,9 @@ func (o FileInfo) EncodeXDRInto(xw *xdr.Writer) (int, error) { return xw.Tot(), err } xw.WriteUint64(uint64(o.LocalVersion)) + if l := len(o.Blocks); l > 1000000 { + return xw.Tot(), xdr.ElementSizeExceeded("Blocks", l, 1000000) + } xw.WriteUint32(uint32(len(o.Blocks))) for i := range o.Blocks { _, err := o.Blocks[i].EncodeXDRInto(xw) @@ -243,7 +252,10 @@ func (o *FileInfo) DecodeXDRFrom(xr *xdr.Reader) error { o.LocalVersion = int64(xr.ReadUint64()) _BlocksSize := int(xr.ReadUint32()) if _BlocksSize < 0 { - return xdr.ElementSizeExceeded("Blocks", _BlocksSize, 0) + return xdr.ElementSizeExceeded("Blocks", _BlocksSize, 1000000) + } + if _BlocksSize > 1000000 { + return xdr.ElementSizeExceeded("Blocks", _BlocksSize, 1000000) } o.Blocks = make([]BlockInfo, _BlocksSize) for i := range o.Blocks { @@ -571,7 +583,7 @@ ClusterConfigMessage Structure: struct ClusterConfigMessage { string ClientName<64>; string ClientVersion<64>; - Folder Folders<>; + Folder Folders<1000000>; Option Options<64>; } @@ -610,6 +622,9 @@ func (o ClusterConfigMessage) EncodeXDRInto(xw *xdr.Writer) (int, error) { return xw.Tot(), xdr.ElementSizeExceeded("ClientVersion", l, 64) } xw.WriteString(o.ClientVersion) + if l := len(o.Folders); l > 1000000 { + return xw.Tot(), xdr.ElementSizeExceeded("Folders", l, 1000000) + } xw.WriteUint32(uint32(len(o.Folders))) for i := range o.Folders { _, err := o.Folders[i].EncodeXDRInto(xw) @@ -646,7 +661,10 @@ func (o *ClusterConfigMessage) DecodeXDRFrom(xr *xdr.Reader) error { o.ClientVersion = xr.ReadStringMax(64) _FoldersSize := int(xr.ReadUint32()) if _FoldersSize < 0 { - return xdr.ElementSizeExceeded("Folders", _FoldersSize, 0) + return xdr.ElementSizeExceeded("Folders", _FoldersSize, 1000000) + } + if _FoldersSize > 1000000 { + return xdr.ElementSizeExceeded("Folders", _FoldersSize, 1000000) } o.Folders = make([]Folder, _FoldersSize) for i := range o.Folders { @@ -697,7 +715,7 @@ Folder Structure: struct Folder { string ID<64>; - Device Devices<>; + Device Devices<1000000>; unsigned int Flags; Option Options<64>; } @@ -733,6 +751,9 @@ func (o Folder) EncodeXDRInto(xw *xdr.Writer) (int, error) { return xw.Tot(), xdr.ElementSizeExceeded("ID", l, 64) } xw.WriteString(o.ID) + if l := len(o.Devices); l > 1000000 { + return xw.Tot(), xdr.ElementSizeExceeded("Devices", l, 1000000) + } xw.WriteUint32(uint32(len(o.Devices))) for i := range o.Devices { _, err := o.Devices[i].EncodeXDRInto(xw) @@ -769,7 +790,10 @@ func (o *Folder) DecodeXDRFrom(xr *xdr.Reader) error { o.ID = xr.ReadStringMax(64) _DevicesSize := int(xr.ReadUint32()) if _DevicesSize < 0 { - return xdr.ElementSizeExceeded("Devices", _DevicesSize, 0) + return xdr.ElementSizeExceeded("Devices", _DevicesSize, 1000000) + } + if _DevicesSize > 1000000 { + return xdr.ElementSizeExceeded("Devices", _DevicesSize, 1000000) } o.Devices = make([]Device, _DevicesSize) for i := range o.Devices { diff --git a/vector_xdr.go b/vector_xdr.go index a4b6b132b..01efa7e4e 100644 --- a/vector_xdr.go +++ b/vector_xdr.go @@ -2,6 +2,8 @@ package protocol +import "github.com/calmh/xdr" + // This stuff is hacked up manually because genxdr doesn't support 'type // Vector []Counter' declarations and it was tricky when I tried to add it... @@ -28,6 +30,9 @@ func (v Vector) EncodeXDRInto(w xdrWriter) (int, error) { // DecodeXDRFrom decodes the XDR objects from the given reader into itself. func (v *Vector) DecodeXDRFrom(r xdrReader) error { l := int(r.ReadUint32()) + if l > 1e6 { + return xdr.ElementSizeExceeded("number of counters", l, 1e6) + } n := make(Vector, l) for i := range n { n[i].ID = r.ReadUint64() From 6a36ec63d75f6ef354d8689027d23d3f16d08f8d Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Tue, 18 Aug 2015 08:42:39 +0200 Subject: [PATCH 72/94] Empty messages with the compression bit set should be accepted --- protocol.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/protocol.go b/protocol.go index 420b20859..8e73afea5 100644 --- a/protocol.go +++ b/protocol.go @@ -412,7 +412,7 @@ func (c *rawConnection) readMessage() (hdr header, msg encodable, err error) { } msgBuf := c.rdbuf0 - if hdr.compression { + if hdr.compression && msglen > 0 { c.rdbuf1 = c.rdbuf1[:cap(c.rdbuf1)] c.rdbuf1, err = lz4.Decode(c.rdbuf1, c.rdbuf0) if err != nil { From 7c6a31017968e7c1a69148db1ca3dea71eba8236 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Wed, 19 Aug 2015 20:49:34 +0100 Subject: [PATCH 73/94] Fix after package move --- client/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/client.go b/client/client.go index a1924e3ea..94e4eedd2 100644 --- a/client/client.go +++ b/client/client.go @@ -12,7 +12,7 @@ import ( syncthingprotocol "github.com/syncthing/protocol" "github.com/syncthing/relaysrv/protocol" - "github.com/syncthing/syncthing/internal/sync" + "github.com/syncthing/syncthing/lib/sync" ) type ProtocolClient struct { From f0c0c5483fe50b5ec060aac32db7da848368f160 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Thu, 20 Aug 2015 12:33:11 +0200 Subject: [PATCH 74/94] Cleaner build --- build.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/build.sh b/build.sh index 5f605e1b9..42c2681dd 100755 --- a/build.sh +++ b/build.sh @@ -11,15 +11,17 @@ build() { export GOOS="$1" export GOARCH="$2" target="relaysrv-$GOOS-$GOARCH" - go build -v + go build -i -v -ldflags -w mkdir "$target" if [ -f relaysrv ] ; then mv relaysrv "$target" - tar zcvf "$target.tar.gz" "$target" + tar zcvf "$target.tar.gz" "$target" + rm -r "$target" fi if [ -f relaysrv.exe ] ; then - mv relaysrv.exe "$target" + mv relaysrv.exe "$target" zip -r "$target.zip" "$target" + rm -r "$target" fi } From d7949aa58ee92df583793e43ce751ffcc0c4f283 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Thu, 20 Aug 2015 12:33:52 +0200 Subject: [PATCH 75/94] I contribute stuff --- CONTRIBUTORS | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS b/CONTRIBUTORS index e69de29bb..4b00258cc 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -0,0 +1 @@ +Jakob Borg From f76a66fc555150cf752040193fa2d154cbc61b61 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Thu, 20 Aug 2015 12:59:44 +0200 Subject: [PATCH 76/94] Very basic status service --- main.go | 7 +++++++ session.go | 12 ++++++++++-- status.go | 39 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 status.go diff --git a/main.go b/main.go index b429d94a3..2defed871 100644 --- a/main.go +++ b/main.go @@ -32,6 +32,8 @@ var ( globalLimitBps int sessionLimiter *ratelimit.Bucket globalLimiter *ratelimit.Bucket + + statusAddr string ) func main() { @@ -47,6 +49,7 @@ func main() { flag.IntVar(&sessionLimitBps, "per-session-rate", sessionLimitBps, "Per session rate limit, in bytes/s") flag.IntVar(&globalLimitBps, "global-rate", globalLimitBps, "Global rate limit, in bytes/s") flag.BoolVar(&debug, "debug", false, "Enable debug output") + flag.StringVar(&statusAddr, "status-srv", ":22070", "Listen address for status service (blank to disable)") flag.Parse() @@ -99,5 +102,9 @@ func main() { go sessionListener(listenSession) + if statusAddr != "" { + go statusService(statusAddr) + } + protocolListener(listenProtocol, tlsCfg) } diff --git a/session.go b/session.go index c526ed5d3..a189ab403 100644 --- a/session.go +++ b/session.go @@ -9,6 +9,7 @@ import ( "log" "net" "sync" + "sync/atomic" "time" "github.com/juju/ratelimit" @@ -18,8 +19,10 @@ import ( ) var ( - sessionMut = sync.Mutex{} - sessions = make(map[string]*session, 0) + sessionMut = sync.Mutex{} + sessions = make(map[string]*session, 0) + numProxies int64 + bytesProxied int64 ) type session struct { @@ -178,6 +181,9 @@ func (s *session) proxy(c1, c2 net.Conn) error { log.Println("Proxy", c1.RemoteAddr(), "->", c2.RemoteAddr()) } + atomic.AddInt64(&numProxies, 1) + defer atomic.AddInt64(&numProxies, -1) + buf := make([]byte, 65536) for { c1.SetReadDeadline(time.Now().Add(networkTimeout)) @@ -186,6 +192,8 @@ func (s *session) proxy(c1, c2 net.Conn) error { return err } + atomic.AddInt64(&bytesProxied, int64(n)) + if debug { log.Printf("%d bytes from %s to %s", n, c1.RemoteAddr(), c2.RemoteAddr()) } diff --git a/status.go b/status.go new file mode 100644 index 000000000..d0391e17d --- /dev/null +++ b/status.go @@ -0,0 +1,39 @@ +package main + +import ( + "encoding/json" + "log" + "net/http" + "runtime" + "sync/atomic" +) + +func statusService(addr string) { + http.HandleFunc("/status", getStatus) + if err := http.ListenAndServe(addr, nil); err != nil { + log.Fatal(err) + } +} + +func getStatus(w http.ResponseWriter, r *http.Request) { + status := make(map[string]interface{}) + + sessionMut.Lock() + status["numSessions"] = len(sessions) + sessionMut.Unlock() + status["numProxies"] = atomic.LoadInt64(&numProxies) + status["bytesProxied"] = atomic.LoadInt64(&bytesProxied) + status["goVersion"] = runtime.Version() + status["goOS"] = runtime.GOOS + status["goAarch"] = runtime.GOARCH + status["goMaxProcs"] = runtime.GOMAXPROCS(-1) + + bs, err := json.MarshalIndent(status, "", " ") + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + w.Write(bs) +} From 37cbe68204c4bf16c6d7db374c5908580028e279 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Thu, 20 Aug 2015 13:58:07 +0200 Subject: [PATCH 77/94] Fix broken connection close --- session_listener.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/session_listener.go b/session_listener.go index 2f6bae9ab..82d2ec732 100644 --- a/session_listener.go +++ b/session_listener.go @@ -36,8 +36,6 @@ func sessionListener(addr string) { } func sessionConnectionHandler(conn net.Conn) { - defer conn.Close() - if err := conn.SetDeadline(time.Now().Add(messageTimeout)); err != nil { if debug { log.Println("Weird error setting deadline:", err, "on", conn.RemoteAddr()) @@ -59,6 +57,7 @@ func sessionConnectionHandler(conn net.Conn) { if ses == nil { protocol.WriteMessage(conn, protocol.ResponseNotFound) + conn.Close() return } @@ -67,6 +66,7 @@ func sessionConnectionHandler(conn net.Conn) { log.Println("Failed to add", conn.RemoteAddr(), "to session", ses) } protocol.WriteMessage(conn, protocol.ResponseAlreadyConnected) + conn.Close() return } @@ -81,12 +81,15 @@ func sessionConnectionHandler(conn net.Conn) { if debug { log.Println("Weird error setting deadline:", err, "on", conn.RemoteAddr()) } + conn.Close() return } + default: if debug { log.Println("Unexpected message from", conn.RemoteAddr(), message) } protocol.WriteMessage(conn, protocol.ResponseUnexpectedMessage) + conn.Close() } } From 7fe1fdd8c751df165ea825bc8d3e895f118bb236 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Thu, 20 Aug 2015 14:02:52 +0200 Subject: [PATCH 78/94] Improve status reporter --- .gitignore | 1 + protocol_listener.go | 11 ++++++--- session.go | 13 ++++++----- status.go | 53 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 69 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index b7006615a..775c43cc2 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,4 @@ _testmain.go *.prof *.tar.gz *.zip +relaysrv diff --git a/protocol_listener.go b/protocol_listener.go index a7243ff69..0ff0154d9 100644 --- a/protocol_listener.go +++ b/protocol_listener.go @@ -7,6 +7,7 @@ import ( "log" "net" "sync" + "sync/atomic" "time" syncthingprotocol "github.com/syncthing/protocol" @@ -15,8 +16,9 @@ import ( ) var ( - outboxesMut = sync.RWMutex{} - outboxes = make(map[syncthingprotocol.DeviceID]chan interface{}) + outboxesMut = sync.RWMutex{} + outboxes = make(map[syncthingprotocol.DeviceID]chan interface{}) + numConnections int64 ) func protocolListener(addr string, config *tls.Config) { @@ -122,7 +124,7 @@ func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config) { outboxesMut.RUnlock() if !ok { if debug { - log.Println(id, "is looking", requestedPeer, "which does not exist") + log.Println(id, "is looking for", requestedPeer, "which does not exist") } protocol.WriteMessage(conn, protocol.ResponseNotFound) conn.Close() @@ -219,6 +221,9 @@ func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config) { } func messageReader(conn net.Conn, messages chan<- interface{}, errors chan<- error) { + atomic.AddInt64(&numConnections, 1) + defer atomic.AddInt64(&numConnections, -1) + for { msg, err := protocol.ReadMessage(conn) if err != nil { diff --git a/session.go b/session.go index a189ab403..c94cdc2a0 100644 --- a/session.go +++ b/session.go @@ -110,30 +110,31 @@ func (s *session) Serve() { close(s.conns) if debug { - log.Println("Session", s, "starting between", conns[0].RemoteAddr(), conns[1].RemoteAddr()) + log.Println("Session", s, "starting between", conns[0].RemoteAddr(), "and", conns[1].RemoteAddr()) } wg := sync.WaitGroup{} wg.Add(2) - errors := make(chan error, 2) - + var err0 error go func() { - errors <- s.proxy(conns[0], conns[1]) + err0 = s.proxy(conns[0], conns[1]) wg.Done() }() + var err1 error go func() { - errors <- s.proxy(conns[1], conns[0]) + err1 = s.proxy(conns[1], conns[0]) wg.Done() }() wg.Wait() if debug { - log.Println("Session", s, "ended, outcomes:", <-errors, <-errors) + log.Println("Session", s, "ended, outcomes:", err0, "and", err1) } goto done + case <-timedout: if debug { log.Println("Session", s, "timed out") diff --git a/status.go b/status.go index d0391e17d..53cea572c 100644 --- a/status.go +++ b/status.go @@ -6,9 +6,14 @@ import ( "net/http" "runtime" "sync/atomic" + "time" ) +var rc *rateCalculator + func statusService(addr string) { + rc = newRateCalculator(360, 10*time.Second, &bytesProxied) + http.HandleFunc("/status", getStatus) if err := http.ListenAndServe(addr, nil); err != nil { log.Fatal(err) @@ -21,12 +26,21 @@ func getStatus(w http.ResponseWriter, r *http.Request) { sessionMut.Lock() status["numSessions"] = len(sessions) sessionMut.Unlock() + status["numConnections"] = atomic.LoadInt64(&numConnections) status["numProxies"] = atomic.LoadInt64(&numProxies) status["bytesProxied"] = atomic.LoadInt64(&bytesProxied) status["goVersion"] = runtime.Version() status["goOS"] = runtime.GOOS status["goAarch"] = runtime.GOARCH status["goMaxProcs"] = runtime.GOMAXPROCS(-1) + status["kbps10s1m5m15m30m60m"] = []int64{ + rc.rate(10/10) * 8 / 1000, + rc.rate(60/10) * 8 / 1000, + rc.rate(5*60/10) * 8 / 1000, + rc.rate(15*60/10) * 8 / 1000, + rc.rate(30*60/10) * 8 / 1000, + rc.rate(60*60/10) * 8 / 1000, + } bs, err := json.MarshalIndent(status, "", " ") if err != nil { @@ -37,3 +51,42 @@ func getStatus(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") w.Write(bs) } + +type rateCalculator struct { + rates []int64 + prev int64 + counter *int64 +} + +func newRateCalculator(keepIntervals int, interval time.Duration, counter *int64) *rateCalculator { + r := &rateCalculator{ + rates: make([]int64, keepIntervals), + counter: counter, + } + + go r.updateRates(interval) + + return r +} + +func (r *rateCalculator) updateRates(interval time.Duration) { + for { + now := time.Now() + next := now.Truncate(interval).Add(interval) + time.Sleep(next.Sub(now)) + + cur := atomic.LoadInt64(r.counter) + rate := int64(float64(cur-r.prev) / interval.Seconds()) + copy(r.rates[1:], r.rates) + r.rates[0] = rate + r.prev = cur + } +} + +func (r *rateCalculator) rate(periods int) int64 { + var tot int64 + for i := 0; i < periods; i++ { + tot += r.rates[i] + } + return tot / int64(periods) +} From 68c5dcd83d9be8f28ae59e951a87cdcf01c6f5cb Mon Sep 17 00:00:00 2001 From: AudriusButkevicius Date: Wed, 26 Aug 2015 22:33:03 +0100 Subject: [PATCH 79/94] Add CachedSize field --- message.go | 1 + 1 file changed, 1 insertion(+) diff --git a/message.go b/message.go index 0cfeaa381..2a37136b5 100644 --- a/message.go +++ b/message.go @@ -20,6 +20,7 @@ type FileInfo struct { Modified int64 Version Vector LocalVersion int64 + CachedSize int64 // noencode (cache only) Blocks []BlockInfo // max:1000000 } From 84365882de255d2204d0eeda8dee288082a27f98 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Fri, 28 Aug 2015 09:01:21 +0200 Subject: [PATCH 80/94] Fix tests --- protocol_test.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/protocol_test.go b/protocol_test.go index 051672411..2c64a9b32 100644 --- a/protocol_test.go +++ b/protocol_test.go @@ -278,11 +278,12 @@ func TestMarshalIndexMessage(t *testing.T) { } f := func(m1 IndexMessage) bool { - for _, f := range m1.Files { - for i := range f.Blocks { - f.Blocks[i].Offset = 0 - if len(f.Blocks[i].Hash) == 0 { - f.Blocks[i].Hash = nil + for i, f := range m1.Files { + m1.Files[i].CachedSize = 0 + for j := range f.Blocks { + f.Blocks[j].Offset = 0 + if len(f.Blocks[j].Hash) == 0 { + f.Blocks[j].Hash = nil } } } From c0554c9fbfcca55f701082a8245382d5d74440b5 Mon Sep 17 00:00:00 2001 From: AudriusButkevicius Date: Wed, 2 Sep 2015 21:35:52 +0100 Subject: [PATCH 81/94] Use a single socket for relaying --- protocol_listener.go => listener.go | 77 +++++++++++++++++++++-- main.go | 15 ++--- protocol/packets.go | 3 +- session_listener.go | 95 ----------------------------- 4 files changed, 79 insertions(+), 111 deletions(-) rename protocol_listener.go => listener.go (73%) delete mode 100644 session_listener.go diff --git a/protocol_listener.go b/listener.go similarity index 73% rename from protocol_listener.go rename to listener.go index 0ff0154d9..a63bc4c8c 100644 --- a/protocol_listener.go +++ b/listener.go @@ -11,6 +11,7 @@ import ( "time" syncthingprotocol "github.com/syncthing/protocol" + "github.com/syncthing/syncthing/lib/tlsutil" "github.com/syncthing/relaysrv/protocol" ) @@ -21,14 +22,16 @@ var ( numConnections int64 ) -func protocolListener(addr string, config *tls.Config) { - listener, err := net.Listen("tcp", addr) +func listener(addr string, config *tls.Config) { + tcpListener, err := net.Listen("tcp", addr) if err != nil { log.Fatalln(err) } + listener := tlsutil.DowngradingListener{tcpListener, nil} + for { - conn, err := listener.Accept() + conn, isTLS, err := listener.AcceptNoWrap() if err != nil { if debug { log.Println(err) @@ -39,10 +42,15 @@ func protocolListener(addr string, config *tls.Config) { setTCPOptions(conn) if debug { - log.Println("Protocol listener accepted connection from", conn.RemoteAddr()) + log.Println("Listener accepted connection from", conn.RemoteAddr(), "tls", isTLS) + } + + if isTLS { + go protocolConnectionHandler(conn, config) + } else { + go sessionConnectionHandler(conn) } - go protocolConnectionHandler(conn, config) } } @@ -220,6 +228,65 @@ func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config) { } } +func sessionConnectionHandler(conn net.Conn) { + if err := conn.SetDeadline(time.Now().Add(messageTimeout)); err != nil { + if debug { + log.Println("Weird error setting deadline:", err, "on", conn.RemoteAddr()) + } + return + } + + message, err := protocol.ReadMessage(conn) + if err != nil { + return + } + + switch msg := message.(type) { + case protocol.JoinSessionRequest: + ses := findSession(string(msg.Key)) + if debug { + log.Println(conn.RemoteAddr(), "session lookup", ses) + } + + if ses == nil { + protocol.WriteMessage(conn, protocol.ResponseNotFound) + conn.Close() + return + } + + if !ses.AddConnection(conn) { + if debug { + log.Println("Failed to add", conn.RemoteAddr(), "to session", ses) + } + protocol.WriteMessage(conn, protocol.ResponseAlreadyConnected) + conn.Close() + return + } + + if err := protocol.WriteMessage(conn, protocol.ResponseSuccess); err != nil { + if debug { + log.Println("Failed to send session join response to ", conn.RemoteAddr(), "for", ses) + } + return + } + + if err := conn.SetDeadline(time.Time{}); err != nil { + if debug { + log.Println("Weird error setting deadline:", err, "on", conn.RemoteAddr()) + } + conn.Close() + return + } + + default: + if debug { + log.Println("Unexpected message from", conn.RemoteAddr(), message) + } + protocol.WriteMessage(conn, protocol.ResponseUnexpectedMessage) + conn.Close() + } +} + func messageReader(conn net.Conn, messages chan<- interface{}, errors chan<- error) { atomic.AddInt64(&numConnections, 1) defer atomic.AddInt64(&numConnections, -1) diff --git a/main.go b/main.go index 2defed871..c050c355d 100644 --- a/main.go +++ b/main.go @@ -17,9 +17,8 @@ import ( ) var ( - listenProtocol string - listenSession string - debug bool + listen string + debug bool sessionAddress []byte sessionPort uint16 @@ -39,9 +38,7 @@ var ( func main() { var dir, extAddress string - flag.StringVar(&listenProtocol, "protocol-listen", ":22067", "Protocol listen address") - flag.StringVar(&listenSession, "session-listen", ":22068", "Session listen address") - flag.StringVar(&extAddress, "external-address", "", "External address to advertise, defaults no IP and session-listen port, causing clients to use the remote IP from the protocol connection") + flag.StringVar(&listen, "listen", ":22067", "Protocol listen address") flag.StringVar(&dir, "keys", ".", "Directory where cert.pem and key.pem is stored") flag.DurationVar(&networkTimeout, "network-timeout", 2*time.Minute, "Timeout for network operations") flag.DurationVar(&pingInterval, "ping-interval", time.Minute, "How often pings are sent") @@ -54,7 +51,7 @@ func main() { flag.Parse() if extAddress == "" { - extAddress = listenSession + extAddress = listen } addr, err := net.ResolveTCPAddr("tcp", extAddress) @@ -100,11 +97,9 @@ func main() { globalLimiter = ratelimit.NewBucketWithRate(float64(globalLimitBps), int64(2*globalLimitBps)) } - go sessionListener(listenSession) - if statusAddr != "" { go statusService(statusAddr) } - protocolListener(listenProtocol, tlsCfg) + listener(listen, tlsCfg) } diff --git a/protocol/packets.go b/protocol/packets.go index 7ff020115..1b21eba24 100644 --- a/protocol/packets.go +++ b/protocol/packets.go @@ -7,8 +7,9 @@ package protocol import ( "fmt" - syncthingprotocol "github.com/syncthing/protocol" "net" + + syncthingprotocol "github.com/syncthing/protocol" ) const ( diff --git a/session_listener.go b/session_listener.go deleted file mode 100644 index 82d2ec732..000000000 --- a/session_listener.go +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file). - -package main - -import ( - "log" - "net" - "time" - - "github.com/syncthing/relaysrv/protocol" -) - -func sessionListener(addr string) { - listener, err := net.Listen("tcp", addr) - if err != nil { - log.Fatalln(err) - } - - for { - conn, err := listener.Accept() - if err != nil { - if debug { - log.Println(err) - } - continue - } - - setTCPOptions(conn) - - if debug { - log.Println("Session listener accepted connection from", conn.RemoteAddr()) - } - - go sessionConnectionHandler(conn) - } -} - -func sessionConnectionHandler(conn net.Conn) { - if err := conn.SetDeadline(time.Now().Add(messageTimeout)); err != nil { - if debug { - log.Println("Weird error setting deadline:", err, "on", conn.RemoteAddr()) - } - return - } - - message, err := protocol.ReadMessage(conn) - if err != nil { - return - } - - switch msg := message.(type) { - case protocol.JoinSessionRequest: - ses := findSession(string(msg.Key)) - if debug { - log.Println(conn.RemoteAddr(), "session lookup", ses) - } - - if ses == nil { - protocol.WriteMessage(conn, protocol.ResponseNotFound) - conn.Close() - return - } - - if !ses.AddConnection(conn) { - if debug { - log.Println("Failed to add", conn.RemoteAddr(), "to session", ses) - } - protocol.WriteMessage(conn, protocol.ResponseAlreadyConnected) - conn.Close() - return - } - - if err := protocol.WriteMessage(conn, protocol.ResponseSuccess); err != nil { - if debug { - log.Println("Failed to send session join response to ", conn.RemoteAddr(), "for", ses) - } - return - } - - if err := conn.SetDeadline(time.Time{}); err != nil { - if debug { - log.Println("Weird error setting deadline:", err, "on", conn.RemoteAddr()) - } - conn.Close() - return - } - - default: - if debug { - log.Println("Unexpected message from", conn.RemoteAddr(), message) - } - protocol.WriteMessage(conn, protocol.ResponseUnexpectedMessage) - conn.Close() - } -} From 541d05df1b732c26438d1e9b8758487c4a51db96 Mon Sep 17 00:00:00 2001 From: AudriusButkevicius Date: Wed, 2 Sep 2015 22:02:17 +0100 Subject: [PATCH 82/94] Use new method name --- listener.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/listener.go b/listener.go index a63bc4c8c..2091b1dd6 100644 --- a/listener.go +++ b/listener.go @@ -31,7 +31,7 @@ func listener(addr string, config *tls.Config) { listener := tlsutil.DowngradingListener{tcpListener, nil} for { - conn, isTLS, err := listener.AcceptNoWrap() + conn, isTLS, err := listener.AcceptNoWrapTLS() if err != nil { if debug { log.Println(err) From 11b2815b8858d3fadc36c9dcf5ae9b64b7d3d3ce Mon Sep 17 00:00:00 2001 From: AudriusButkevicius Date: Sun, 6 Sep 2015 18:35:38 +0100 Subject: [PATCH 83/94] Add a test method, fix nil pointer panic --- client/methods.go | 22 +++++++++++++++++++++- testutil/main.go | 9 ++++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/client/methods.go b/client/methods.go index ef6145e9c..cfc810f5e 100644 --- a/client/methods.go +++ b/client/methods.go @@ -8,6 +8,7 @@ import ( "net" "net/url" "strconv" + "strings" "time" syncthingprotocol "github.com/syncthing/protocol" @@ -20,10 +21,10 @@ func GetInvitationFromRelay(uri *url.URL, id syncthingprotocol.DeviceID, certs [ } conn, err := tls.Dial("tcp", uri.Host, configForCerts(certs)) - conn.SetDeadline(time.Now().Add(10 * time.Second)) if err != nil { return protocol.SessionInvitation{}, err } + conn.SetDeadline(time.Now().Add(10 * time.Second)) if err := performHandshakeAndValidation(conn, uri); err != nil { return protocol.SessionInvitation{}, err @@ -97,6 +98,25 @@ func JoinSession(invitation protocol.SessionInvitation) (net.Conn, error) { } } +func TestRelay(uri *url.URL, certs []tls.Certificate) bool { + id := syncthingprotocol.NewDeviceID(certs[0].Certificate[0]) + c := NewProtocolClient(uri, certs, nil) + go c.Serve() + defer c.Stop() + + for i := 0; i < 5; i++ { + _, err := GetInvitationFromRelay(uri, id, certs) + if err == nil { + return true + } + if !strings.Contains(err.Error(), "Incorrect response code") { + return false + } + time.Sleep(time.Second) + } + return false +} + func configForCerts(certs []tls.Certificate) *tls.Config { return &tls.Config{ Certificates: certs, diff --git a/testutil/main.go b/testutil/main.go index 69dbb00a1..ffeb9942d 100644 --- a/testutil/main.go +++ b/testutil/main.go @@ -23,10 +23,11 @@ func main() { log.SetFlags(log.LstdFlags | log.Lshortfile) var connect, relay, dir string - var join bool + var join, test bool flag.StringVar(&connect, "connect", "", "Device ID to which to connect to") flag.BoolVar(&join, "join", false, "Join relay") + flag.BoolVar(&test, "test", false, "Generic relay test") flag.StringVar(&relay, "relay", "relay://127.0.0.1:22067", "Relay address") flag.StringVar(&dir, "keys", ".", "Directory where cert.pem and key.pem is stored") @@ -99,6 +100,12 @@ func main() { log.Println("Joined", conn.RemoteAddr(), conn.LocalAddr()) connectToStdio(stdin, conn) log.Println("Finished", conn.RemoteAddr(), conn.LocalAddr()) + } else if test { + if client.TestRelay(uri, []tls.Certificate{cert}) { + log.Println("OK") + } else { + log.Println("FAIL") + } } else { log.Fatal("Requires either join or connect") } From e3ca797dadcbec00b8e5bd7d98cb2f2b2bb43a7a Mon Sep 17 00:00:00 2001 From: AudriusButkevicius Date: Sun, 6 Sep 2015 20:25:53 +0100 Subject: [PATCH 84/94] Receive the invite, otherwise stop blocks, add extra arguments --- client/methods.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/client/methods.go b/client/methods.go index cfc810f5e..67a9a71c1 100644 --- a/client/methods.go +++ b/client/methods.go @@ -98,13 +98,17 @@ func JoinSession(invitation protocol.SessionInvitation) (net.Conn, error) { } } -func TestRelay(uri *url.URL, certs []tls.Certificate) bool { +func TestRelay(uri *url.URL, certs []tls.Certificate, sleep time.Duration, times int) bool { id := syncthingprotocol.NewDeviceID(certs[0].Certificate[0]) - c := NewProtocolClient(uri, certs, nil) + invs := make(chan protocol.SessionInvitation, 1) + c := NewProtocolClient(uri, certs, invs) go c.Serve() - defer c.Stop() + defer func() { + close(invs) + c.Stop() + }() - for i := 0; i < 5; i++ { + for i := 0; i < times; i++ { _, err := GetInvitationFromRelay(uri, id, certs) if err == nil { return true @@ -112,7 +116,7 @@ func TestRelay(uri *url.URL, certs []tls.Certificate) bool { if !strings.Contains(err.Error(), "Incorrect response code") { return false } - time.Sleep(time.Second) + time.Sleep(sleep) } return false } From eab5fd5bdd35f27e4134bec87f6518f823565fe8 Mon Sep 17 00:00:00 2001 From: AudriusButkevicius Date: Mon, 7 Sep 2015 09:21:23 +0100 Subject: [PATCH 85/94] Join relay pool by default --- main.go | 28 ++++++++++++++++++++++++- pool.go | 63 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 pool.go diff --git a/main.go b/main.go index c050c355d..4e7e9a137 100644 --- a/main.go +++ b/main.go @@ -5,9 +5,12 @@ package main import ( "crypto/tls" "flag" + "fmt" "log" "net" + "net/url" "path/filepath" + "strings" "time" "github.com/juju/ratelimit" @@ -32,7 +35,9 @@ var ( sessionLimiter *ratelimit.Bucket globalLimiter *ratelimit.Bucket - statusAddr string + statusAddr string + poolAddrs string + defaultPoolAddrs string = "https://relays.syncthing.net" ) func main() { @@ -47,6 +52,7 @@ func main() { flag.IntVar(&globalLimitBps, "global-rate", globalLimitBps, "Global rate limit, in bytes/s") flag.BoolVar(&debug, "debug", false, "Enable debug output") flag.StringVar(&statusAddr, "status-srv", ":22070", "Listen address for status service (blank to disable)") + flag.StringVar(&poolAddrs, "pools", defaultPoolAddrs, "Comma separated list of relau pool addresses to join") flag.Parse() @@ -101,5 +107,25 @@ func main() { go statusService(statusAddr) } + uri, err := url.Parse(fmt.Sprintf("relay://%s/?id=%s", extAddress, id)) + if err != nil { + log.Fatalln("Failed to construct URI", err) + } + + if poolAddrs == defaultPoolAddrs { + log.Println("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") + log.Println("!! Joining default relay pools, this relay will be available for public use. !!") + log.Println(`!! Use the -pools="" command line option to make the relay private. !!`) + log.Println("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") + } + + pools := strings.Split(poolAddrs, ",") + for _, pool := range pools { + pool = strings.TrimSpace(pool) + if len(pool) > 0 { + go poolHandler(pool, uri) + } + } + listener(listen, tlsCfg) } diff --git a/pool.go b/pool.go new file mode 100644 index 000000000..398b0c0a8 --- /dev/null +++ b/pool.go @@ -0,0 +1,63 @@ +// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file). + +package main + +import ( + "bytes" + "encoding/json" + "io/ioutil" + "log" + "net/http" + "net/url" + "time" +) + +func poolHandler(pool string, uri *url.URL) { + for { + var b bytes.Buffer + json.NewEncoder(&b).Encode(struct { + URL string `json:"url"` + }{ + uri.String(), + }) + + resp, err := http.Post(pool, "application/json", &b) + if err != nil { + if debug { + log.Println("Error joining pool", pool, err) + } + } else if resp.StatusCode == 500 { + if debug { + bs, err := ioutil.ReadAll(resp.Body) + if err != nil { + log.Println("Failed to read response body for", pool, err) + } else { + log.Println("Response for", pool, string(bs)) + } + resp.Body.Close() + } + } else if resp.StatusCode == 429 { + if debug { + log.Println(pool, "under load, will retry in a minute") + } + time.Sleep(time.Minute) + continue + } else if resp.StatusCode == 200 { + var x struct { + EvictionIn time.Duration `json:"evictionIn"` + } + err := json.NewDecoder(resp.Body).Decode(&x) + if err == nil { + rejoin := x.EvictionIn - (x.EvictionIn / 5) + if debug { + log.Println("Joined", pool, "rejoining in", rejoin) + } + time.Sleep(rejoin) + continue + } else if debug { + log.Println("Failed to deserialize respnse", err) + } + } + time.Sleep(time.Hour) + } +} From d180bc794b56b660a98b412ff193a7fa15a570ef Mon Sep 17 00:00:00 2001 From: AudriusButkevicius Date: Mon, 7 Sep 2015 18:12:18 +0100 Subject: [PATCH 86/94] Handle 403 --- pool.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pool.go b/pool.go index 398b0c0a8..0f327f690 100644 --- a/pool.go +++ b/pool.go @@ -42,6 +42,11 @@ func poolHandler(pool string, uri *url.URL) { } time.Sleep(time.Minute) continue + } else if resp.StatusCode == 403 { + if debug { + log.Println(pool, "failed to join due to IP address not matching external address") + } + return } else if resp.StatusCode == 200 { var x struct { EvictionIn time.Duration `json:"evictionIn"` From 7e0106da0cf3eddba2a5ed338fd67906753d3194 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Fri, 11 Sep 2015 20:01:33 +0100 Subject: [PATCH 87/94] Tweaks 1. Advertise relay server paramters so that clients could make a decision wether or not to connect 2. Generate certificate if it's not there. --- main.go | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/main.go b/main.go index 4e7e9a137..1da3c455a 100644 --- a/main.go +++ b/main.go @@ -15,20 +15,21 @@ import ( "github.com/juju/ratelimit" "github.com/syncthing/relaysrv/protocol" + "github.com/syncthing/syncthing/lib/tlsutil" syncthingprotocol "github.com/syncthing/protocol" ) var ( listen string - debug bool + debug bool = false sessionAddress []byte sessionPort uint16 - networkTimeout time.Duration - pingInterval time.Duration - messageTimeout time.Duration + networkTimeout time.Duration = 2 * time.Minute + pingInterval time.Duration = time.Minute + messageTimeout time.Duration = time.Minute sessionLimitBps int globalLimitBps int @@ -45,14 +46,14 @@ func main() { flag.StringVar(&listen, "listen", ":22067", "Protocol listen address") flag.StringVar(&dir, "keys", ".", "Directory where cert.pem and key.pem is stored") - flag.DurationVar(&networkTimeout, "network-timeout", 2*time.Minute, "Timeout for network operations") - flag.DurationVar(&pingInterval, "ping-interval", time.Minute, "How often pings are sent") - flag.DurationVar(&messageTimeout, "message-timeout", time.Minute, "Maximum amount of time we wait for relevant messages to arrive") + flag.DurationVar(&networkTimeout, "network-timeout", networkTimeout, "Timeout for network operations between the client and the relay.\n\tIf no data is received between the client and the relay in this period of time, the connection is terminated.\n\tFurthermore, if no data is sent between either clients being relayed within this period of time, the session is also terminated.") + flag.DurationVar(&pingInterval, "ping-interval", pingInterval, "How often pings are sent") + flag.DurationVar(&messageTimeout, "message-timeout", messageTimeout, "Maximum amount of time we wait for relevant messages to arrive") flag.IntVar(&sessionLimitBps, "per-session-rate", sessionLimitBps, "Per session rate limit, in bytes/s") flag.IntVar(&globalLimitBps, "global-rate", globalLimitBps, "Global rate limit, in bytes/s") - flag.BoolVar(&debug, "debug", false, "Enable debug output") + flag.BoolVar(&debug, "debug", debug, "Enable debug output") flag.StringVar(&statusAddr, "status-srv", ":22070", "Listen address for status service (blank to disable)") - flag.StringVar(&poolAddrs, "pools", defaultPoolAddrs, "Comma separated list of relau pool addresses to join") + flag.StringVar(&poolAddrs, "pools", defaultPoolAddrs, "Comma separated list of relay pool addresses to join") flag.Parse() @@ -71,7 +72,11 @@ func main() { certFile, keyFile := filepath.Join(dir, "cert.pem"), filepath.Join(dir, "key.pem") cert, err := tls.LoadX509KeyPair(certFile, keyFile) if err != nil { - log.Fatalln("Failed to load X509 key pair:", err) + log.Println("Failed to load keypair. Generating one, this might take a while...") + cert, err = tlsutil.NewCertificate(certFile, keyFile, "relaysrv", 3072) + if err != nil { + log.Fatalln("Failed to generate X509 key pair:", err) + } } tlsCfg := &tls.Config{ @@ -107,11 +112,15 @@ func main() { go statusService(statusAddr) } - uri, err := url.Parse(fmt.Sprintf("relay://%s/?id=%s", extAddress, id)) + uri, err := url.Parse(fmt.Sprintf("relay://%s/?id=%s&pingInterval=%s&networkTimeout=%s&sessionLimitBps=%d&globalLimitBps=%d&statusAddr=%s", extAddress, id, pingInterval, networkTimeout, sessionLimitBps, globalLimitBps, statusAddr)) if err != nil { log.Fatalln("Failed to construct URI", err) } + if debug { + log.Println("URI:", uri.String()) + } + if poolAddrs == defaultPoolAddrs { log.Println("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") log.Println("!! Joining default relay pools, this relay will be available for public use. !!") From f9f12131ae2fa70edc866413fee5ca4951d278b1 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Fri, 11 Sep 2015 22:29:50 +0100 Subject: [PATCH 88/94] Drop all sessions when we realize a node has gone away --- listener.go | 24 ++++++---- main.go | 2 + session.go | 131 +++++++++++++++++++++++++++++++++++++--------------- status.go | 4 +- 4 files changed, 114 insertions(+), 47 deletions(-) diff --git a/listener.go b/listener.go index 2091b1dd6..9dde76273 100644 --- a/listener.go +++ b/listener.go @@ -4,6 +4,7 @@ package main import ( "crypto/tls" + "encoding/hex" "log" "net" "sync" @@ -34,7 +35,7 @@ func listener(addr string, config *tls.Config) { conn, isTLS, err := listener.AcceptNoWrapTLS() if err != nil { if debug { - log.Println(err) + log.Println("Listener failed to accept connection from", conn.RemoteAddr(), ". Possibly a TCP Ping.") } continue } @@ -138,13 +139,13 @@ func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config) { conn.Close() continue } - - ses := newSession(sessionLimiter, globalLimiter) + // requestedPeer is the server, id is the client + ses := newSession(requestedPeer, id, sessionLimiter, globalLimiter) go ses.Serve() - clientInvitation := ses.GetClientInvitationMessage(requestedPeer) - serverInvitation := ses.GetServerInvitationMessage(id) + clientInvitation := ses.GetClientInvitationMessage() + serverInvitation := ses.GetServerInvitationMessage() if err := protocol.WriteMessage(conn, clientInvitation); err != nil { if debug { @@ -181,12 +182,19 @@ func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config) { // Potentially closing a second time. conn.Close() - // Only delete the outbox if the client is joined, as it might be - // a lookup request coming from the same client. if joined { + // Only delete the outbox if the client is joined, as it might be + // a lookup request coming from the same client. outboxesMut.Lock() delete(outboxes, id) outboxesMut.Unlock() + // Also, kill all sessions related to this node, as it probably + // went offline. This is for the other end to realize the client + // is no longer there faster. This also helps resolve + // 'already connected' errors when one of the sides is + // restarting, and connecting to the other peer before the other + // peer even realised that the node has gone away. + dropSessions(id) } return @@ -245,7 +253,7 @@ func sessionConnectionHandler(conn net.Conn) { case protocol.JoinSessionRequest: ses := findSession(string(msg.Key)) if debug { - log.Println(conn.RemoteAddr(), "session lookup", ses) + log.Println(conn.RemoteAddr(), "session lookup", ses, hex.EncodeToString(msg.Key)[:5]) } if ses == nil { diff --git a/main.go b/main.go index 1da3c455a..614f82c7f 100644 --- a/main.go +++ b/main.go @@ -42,6 +42,8 @@ var ( ) func main() { + log.SetFlags(log.Lshortfile | log.LstdFlags) + var dir, extAddress string flag.StringVar(&listen, "listen", ":22067", "Protocol listen address") diff --git a/session.go b/session.go index c94cdc2a0..bbd29d1f5 100644 --- a/session.go +++ b/session.go @@ -19,22 +19,14 @@ import ( ) var ( - sessionMut = sync.Mutex{} - sessions = make(map[string]*session, 0) - numProxies int64 - bytesProxied int64 + sessionMut = sync.RWMutex{} + activeSessions = make([]*session, 0) + pendingSessions = make(map[string]*session, 0) + numProxies int64 + bytesProxied int64 ) -type session struct { - serverkey []byte - clientkey []byte - - rateLimit func(bytes int64) - - conns chan net.Conn -} - -func newSession(sessionRateLimit, globalRateLimit *ratelimit.Bucket) *session { +func newSession(serverid, clientid syncthingprotocol.DeviceID, sessionRateLimit, globalRateLimit *ratelimit.Bucket) *session { serverkey := make([]byte, 32) _, err := rand.Read(serverkey) if err != nil { @@ -49,9 +41,12 @@ func newSession(sessionRateLimit, globalRateLimit *ratelimit.Bucket) *session { ses := &session{ serverkey: serverkey, + serverid: serverid, clientkey: clientkey, + clientid: clientid, rateLimit: makeRateLimitFunc(sessionRateLimit, globalRateLimit), - conns: make(chan net.Conn), + connsChan: make(chan net.Conn), + conns: make([]net.Conn, 0, 2), } if debug { @@ -59,8 +54,8 @@ func newSession(sessionRateLimit, globalRateLimit *ratelimit.Bucket) *session { } sessionMut.Lock() - sessions[string(ses.serverkey)] = ses - sessions[string(ses.clientkey)] = ses + pendingSessions[string(ses.serverkey)] = ses + pendingSessions[string(ses.clientkey)] = ses sessionMut.Unlock() return ses @@ -69,13 +64,41 @@ func newSession(sessionRateLimit, globalRateLimit *ratelimit.Bucket) *session { func findSession(key string) *session { sessionMut.Lock() defer sessionMut.Unlock() - lob, ok := sessions[key] + ses, ok := pendingSessions[key] if !ok { return nil } - delete(sessions, key) - return lob + delete(pendingSessions, key) + return ses +} + +func dropSessions(id syncthingprotocol.DeviceID) { + sessionMut.RLock() + for _, session := range activeSessions { + if session.HasParticipant(id) { + if debug { + log.Println("Dropping session", session, "involving", id) + } + session.CloseConns() + } + } + sessionMut.RUnlock() +} + +type session struct { + mut sync.Mutex + + serverkey []byte + serverid syncthingprotocol.DeviceID + + clientkey []byte + clientid syncthingprotocol.DeviceID + + rateLimit func(bytes int64) + + connsChan chan net.Conn + conns []net.Conn } func (s *session) AddConnection(conn net.Conn) bool { @@ -84,7 +107,7 @@ func (s *session) AddConnection(conn net.Conn) bool { } select { - case s.conns <- conn: + case s.connsChan <- conn: return true default: } @@ -98,19 +121,21 @@ func (s *session) Serve() { log.Println("Session", s, "serving") } - conns := make([]net.Conn, 0, 2) for { select { - case conn := <-s.conns: - conns = append(conns, conn) - if len(conns) < 2 { + case conn := <-s.connsChan: + s.mut.Lock() + s.conns = append(s.conns, conn) + s.mut.Unlock() + // We're the only ones mutating% s.conns, hence we are free to read it. + if len(s.conns) < 2 { continue } - close(s.conns) + close(s.connsChan) if debug { - log.Println("Session", s, "starting between", conns[0].RemoteAddr(), "and", conns[1].RemoteAddr()) + log.Println("Session", s, "starting between", s.conns[0].RemoteAddr(), "and", s.conns[1].RemoteAddr()) } wg := sync.WaitGroup{} @@ -118,16 +143,20 @@ func (s *session) Serve() { var err0 error go func() { - err0 = s.proxy(conns[0], conns[1]) + err0 = s.proxy(s.conns[0], s.conns[1]) wg.Done() }() var err1 error go func() { - err1 = s.proxy(conns[1], conns[0]) + err1 = s.proxy(s.conns[1], s.conns[0]) wg.Done() }() + sessionMut.Lock() + activeSessions = append(activeSessions, s) + sessionMut.Unlock() + wg.Wait() if debug { @@ -143,23 +172,37 @@ func (s *session) Serve() { } } done: + // We can end up here in 3 cases: + // 1. Timeout joining, in which case there are potentially entries in pendingSessions + // 2. General session end/timeout, in which case there are entries in activeSessions + // 3. Protocol handler calls dropSession as one of it's clients disconnects. + sessionMut.Lock() - delete(sessions, string(s.serverkey)) - delete(sessions, string(s.clientkey)) + delete(pendingSessions, string(s.serverkey)) + delete(pendingSessions, string(s.clientkey)) + + for i, session := range activeSessions { + if session == s { + l := len(activeSessions) - 1 + activeSessions[i] = activeSessions[l] + activeSessions[l] = nil + activeSessions = activeSessions[:l] + } + } sessionMut.Unlock() - for _, conn := range conns { - conn.Close() - } + // If we are here because of case 2 or 3, we are potentially closing some or + // all connections a second time. + s.CloseConns() if debug { log.Println("Session", s, "stopping") } } -func (s *session) GetClientInvitationMessage(from syncthingprotocol.DeviceID) protocol.SessionInvitation { +func (s *session) GetClientInvitationMessage() protocol.SessionInvitation { return protocol.SessionInvitation{ - From: from[:], + From: s.serverid[:], Key: []byte(s.clientkey), Address: sessionAddress, Port: sessionPort, @@ -167,9 +210,9 @@ func (s *session) GetClientInvitationMessage(from syncthingprotocol.DeviceID) pr } } -func (s *session) GetServerInvitationMessage(from syncthingprotocol.DeviceID) protocol.SessionInvitation { +func (s *session) GetServerInvitationMessage() protocol.SessionInvitation { return protocol.SessionInvitation{ - From: from[:], + From: s.clientid[:], Key: []byte(s.serverkey), Address: sessionAddress, Port: sessionPort, @@ -177,6 +220,18 @@ func (s *session) GetServerInvitationMessage(from syncthingprotocol.DeviceID) pr } } +func (s *session) HasParticipant(id syncthingprotocol.DeviceID) bool { + return s.clientid == id || s.serverid == id +} + +func (s *session) CloseConns() { + s.mut.Lock() + for _, conn := range s.conns { + conn.Close() + } + s.mut.Unlock() +} + func (s *session) proxy(c1, c2 net.Conn) error { if debug { log.Println("Proxy", c1.RemoteAddr(), "->", c2.RemoteAddr()) diff --git a/status.go b/status.go index 53cea572c..b18cf3ea7 100644 --- a/status.go +++ b/status.go @@ -24,7 +24,9 @@ func getStatus(w http.ResponseWriter, r *http.Request) { status := make(map[string]interface{}) sessionMut.Lock() - status["numSessions"] = len(sessions) + // This can potentially be double the number of pending sessions, as each session has two keys, one for each side. + status["numPendingSessionKeys"] = len(pendingSessions) + status["numActiveSessions"] = len(activeSessions) sessionMut.Unlock() status["numConnections"] = atomic.LoadInt64(&numConnections) status["numProxies"] = atomic.LoadInt64(&numProxies) From fccb9c0bf4e1c4b7f9b3ab98ab23d31f0acbc4d7 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Mon, 14 Sep 2015 13:44:47 +0200 Subject: [PATCH 89/94] Server should respond to ping --- listener.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/listener.go b/listener.go index 2091b1dd6..88fe8fd0f 100644 --- a/listener.go +++ b/listener.go @@ -161,6 +161,15 @@ func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config) { } conn.Close() + case protocol.Ping: + if err := protocol.WriteMessage(conn, protocol.Pong{}); err != nil { + if debug { + log.Println("Error writing pong:", err) + } + conn.Close() + continue + } + case protocol.Pong: // Nothing From 22783d8f6c07c34e686e824819b0945528510291 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Mon, 14 Sep 2015 13:55:00 +0200 Subject: [PATCH 90/94] Connected clients should know their own latency --- client/client.go | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/client/client.go b/client/client.go index 94e4eedd2..89b16e000 100644 --- a/client/client.go +++ b/client/client.go @@ -32,6 +32,7 @@ type ProtocolClient struct { mut sync.RWMutex connected bool + latency time.Duration } func NewProtocolClient(uri *url.URL, certs []tls.Certificate, invitations chan protocol.SessionInvitation) *ProtocolClient { @@ -168,6 +169,13 @@ func (c *ProtocolClient) StatusOK() bool { return con } +func (c *ProtocolClient) Latency() time.Duration { + c.mut.RLock() + lat := c.latency + c.mut.RUnlock() + return lat +} + func (c *ProtocolClient) String() string { return fmt.Sprintf("ProtocolClient@%p", c) } @@ -177,11 +185,21 @@ func (c *ProtocolClient) connect() error { return fmt.Errorf("Unsupported relay schema:", c.URI.Scheme) } - conn, err := tls.Dial("tcp", c.URI.Host, c.config) + t0 := time.Now() + tcpConn, err := net.Dial("tcp", c.URI.Host) if err != nil { return err } + c.mut.Lock() + c.latency = time.Since(t0) + c.mut.Unlock() + + conn := tls.Client(tcpConn, c.config) + if err = conn.Handshake(); err != nil { + return err + } + if err := conn.SetDeadline(time.Now().Add(10 * time.Second)); err != nil { conn.Close() return err From 05c79ac8c274ad3c207ce15f1faf25d6d0c3a3ec Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Mon, 21 Sep 2015 08:51:42 +0200 Subject: [PATCH 91/94] Simplify and improve the ping mechanism This should resolve the spurious ping timeouts we've had on low powered boxes. Those errors are the result of us requiring a timely Pong response to our Pings. However this is unnecessarily strict - as long as we've received *anything* recently, we know the other peer is alive. So the new mechanism removes the Pong message entirely and separates the ping check into two routines: - One that makes sure to send ping periodically, if nothing else has been sent. This guarantees a message sent every 45-90 seconds. - One that checks how long it was since we last received a message. If it's longer than 300 seconds, we trigger an ErrTimeout. So we're guaranteed to detect a connection failure in 300 + 300/2 seconds (due to how often the check runs) and we may detect it much sooner if we get an actual error on the ping write (a connection reset or so). This is more sluggish than before but I think that's an OK price to pay for making it actually work out of the box. This removes the configurability of it, as the timeout on one side is dependent on the send interval on the other side. Do we still need it configurable? --- protocol.go | 106 ++++++++++++++++++++++------------------------- protocol_test.go | 89 --------------------------------------- 2 files changed, 50 insertions(+), 145 deletions(-) diff --git a/protocol.go b/protocol.go index 8e73afea5..4c1364eaf 100644 --- a/protocol.go +++ b/protocol.go @@ -28,7 +28,6 @@ const ( messageTypeRequest = 2 messageTypeResponse = 3 messageTypePing = 4 - messageTypePong = 5 messageTypeIndexUpdate = 6 messageTypeClose = 7 ) @@ -71,13 +70,12 @@ const ( ) var ( - ErrClusterHash = fmt.Errorf("configuration error: mismatched cluster hash") - ErrClosed = errors.New("connection closed") + ErrClosed = errors.New("connection closed") + ErrTimeout = errors.New("read timeout") ) // Specific variants of empty messages... type pingMessage struct{ EmptyMessage } -type pongMessage struct{ EmptyMessage } type Model interface { // An index was received from the peer device @@ -146,9 +144,11 @@ type isEofer interface { IsEOF() bool } -var ( - PingTimeout = 30 * time.Second - PingIdleTime = 60 * time.Second +const ( + // We make sure to send a message at least this often, by triggering pings. + PingSendInterval = 90 * time.Second + // If we haven't received a message from the other side for this long, close the connection. + ReceiveTimeout = 300 * time.Second ) func NewConnection(deviceID DeviceID, reader io.Reader, writer io.Writer, receiver Model, name string, compress Compression) Connection { @@ -180,7 +180,8 @@ func NewConnection(deviceID DeviceID, reader io.Reader, writer io.Writer, receiv func (c *rawConnection) Start() { go c.readerLoop() go c.writerLoop() - go c.pingerLoop() + go c.pingSender() + go c.pingReceiver() go c.idGenerator() } @@ -278,18 +279,7 @@ func (c *rawConnection) ping() bool { return false } - rc := make(chan asyncResult, 1) - c.awaitingMut.Lock() - c.awaiting[id] = rc - c.awaitingMut.Unlock() - - ok := c.send(id, messageTypePing, nil, nil) - if !ok { - return false - } - - res, ok := <-rc - return ok && res.err == nil + return c.send(id, messageTypePing, nil, nil) } func (c *rawConnection) readerLoop() (err error) { @@ -352,13 +342,7 @@ func (c *rawConnection) readerLoop() (err error) { if state != stateReady { return fmt.Errorf("protocol error: ping message in state %d", state) } - c.send(hdr.msgID, messageTypePong, pongMessage{}, nil) - - case pongMessage: - if state != stateReady { - return fmt.Errorf("protocol error: pong message in state %d", state) - } - c.handlePong(hdr.msgID) + // Nothing case CloseMessage: return errors.New(msg.Reason) @@ -467,9 +451,6 @@ func (c *rawConnection) readMessage() (hdr header, msg encodable, err error) { case messageTypePing: msg = pingMessage{} - case messageTypePong: - msg = pongMessage{} - case messageTypeClusterConfig: var cc ClusterConfigMessage err = cc.UnmarshalXDR(msgBuf) @@ -729,42 +710,55 @@ func (c *rawConnection) idGenerator() { } } -func (c *rawConnection) pingerLoop() { - var rc = make(chan bool, 1) - ticker := time.Tick(PingIdleTime / 2) +// The pingSender makes sure that we've sent a message within the last +// PingSendInterval. If we already have something sent in the last +// PingSendInterval/2, we do nothing. Otherwise we send a ping message. This +// results in an effecting ping interval of somewhere between +// PingSendInterval/2 and PingSendInterval. +func (c *rawConnection) pingSender() { + ticker := time.Tick(PingSendInterval / 2) + for { select { case <-ticker: - if d := time.Since(c.cr.Last()); d < PingIdleTime { - if debug { - l.Debugln(c.id, "ping skipped after rd", d) - } - continue - } - if d := time.Since(c.cw.Last()); d < PingIdleTime { + d := time.Since(c.cw.Last()) + if d < PingSendInterval/2 { if debug { l.Debugln(c.id, "ping skipped after wr", d) } continue } - go func() { + + if debug { + l.Debugln(c.id, "ping -> after", d) + } + c.ping() + + case <-c.closed: + return + } + } +} + +// The pingReciever checks that we've received a message (any message will do, +// but we expect pings in the absence of other messages) within the last +// ReceiveTimeout. If not, we close the connection with an ErrTimeout. +func (c *rawConnection) pingReceiver() { + ticker := time.Tick(ReceiveTimeout / 2) + + for { + select { + case <-ticker: + d := time.Since(c.cr.Last()) + if d > ReceiveTimeout { if debug { - l.Debugln(c.id, "ping ->") + l.Debugln(c.id, "ping timeout", d) } - rc <- c.ping() - }() - select { - case ok := <-rc: - if debug { - l.Debugln(c.id, "<- pong") - } - if !ok { - c.close(fmt.Errorf("ping failure")) - } - case <-time.After(PingTimeout): - c.close(fmt.Errorf("ping timeout")) - case <-c.closed: - return + c.close(ErrTimeout) + } + + if debug { + l.Debugln(c.id, "last read within", d) } case <-c.closed: diff --git a/protocol_test.go b/protocol_test.go index 2c64a9b32..8a4708843 100644 --- a/protocol_test.go +++ b/protocol_test.go @@ -6,7 +6,6 @@ import ( "bytes" "encoding/hex" "encoding/json" - "errors" "fmt" "io" "io/ioutil" @@ -82,94 +81,6 @@ func TestPing(t *testing.T) { } } -func TestPingErr(t *testing.T) { - e := errors.New("something broke") - - for i := 0; i < 32; i++ { - for j := 0; j < 32; j++ { - m0 := newTestModel() - m1 := newTestModel() - - ar, aw := io.Pipe() - br, bw := io.Pipe() - eaw := &ErrPipe{PipeWriter: *aw, max: i, err: e} - ebw := &ErrPipe{PipeWriter: *bw, max: j, err: e} - - c0 := NewConnection(c0ID, ar, ebw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) - c0.Start() - c1 := NewConnection(c1ID, br, eaw, m1, "name", CompressAlways) - c1.Start() - c0.ClusterConfig(ClusterConfigMessage{}) - c1.ClusterConfig(ClusterConfigMessage{}) - - res := c0.ping() - if (i < 8 || j < 8) && res { - // This should have resulted in failure, as there is no way an empty ClusterConfig plus a Ping message fits in eight bytes. - t.Errorf("Unexpected ping success; i=%d, j=%d", i, j) - } else if (i >= 28 && j >= 28) && !res { - // This should have worked though, as 28 bytes is plenty for both. - t.Errorf("Unexpected ping fail; i=%d, j=%d", i, j) - } - } - } -} - -// func TestRequestResponseErr(t *testing.T) { -// e := errors.New("something broke") - -// var pass bool -// for i := 0; i < 48; i++ { -// for j := 0; j < 38; j++ { -// m0 := newTestModel() -// m0.data = []byte("response data") -// m1 := newTestModel() - -// ar, aw := io.Pipe() -// br, bw := io.Pipe() -// eaw := &ErrPipe{PipeWriter: *aw, max: i, err: e} -// ebw := &ErrPipe{PipeWriter: *bw, max: j, err: e} - -// NewConnection(c0ID, ar, ebw, m0, nil) -// c1 := NewConnection(c1ID, br, eaw, m1, nil).(wireFormatConnection).next.(*rawConnection) - -// d, err := c1.Request("default", "tn", 1234, 5678) -// if err == e || err == ErrClosed { -// t.Logf("Error at %d+%d bytes", i, j) -// if !m1.isClosed() { -// t.Fatal("c1 not closed") -// } -// if !m0.isClosed() { -// t.Fatal("c0 not closed") -// } -// continue -// } -// if err != nil { -// t.Fatal(err) -// } -// if string(d) != "response data" { -// t.Fatalf("Incorrect response data %q", string(d)) -// } -// if m0.folder != "default" { -// t.Fatalf("Incorrect folder %q", m0.folder) -// } -// if m0.name != "tn" { -// t.Fatalf("Incorrect name %q", m0.name) -// } -// if m0.offset != 1234 { -// t.Fatalf("Incorrect offset %d", m0.offset) -// } -// if m0.size != 5678 { -// t.Fatalf("Incorrect size %d", m0.size) -// } -// t.Logf("Pass at %d+%d bytes", i, j) -// pass = true -// } -// } -// if !pass { -// t.Fatal("Never passed") -// } -// } - func TestVersionErr(t *testing.T) { m0 := newTestModel() m1 := newTestModel() From f177924629396526442335c4087365065a50badf Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Tue, 22 Sep 2015 19:37:03 +0200 Subject: [PATCH 92/94] Rejiggle lib/relaysrv/* -> lib/relay/* --- lib/{relaysrv => relay}/client/client.go | 0 lib/{relaysrv => relay}/client/debug.go | 0 lib/{relaysrv => relay}/client/methods.go | 0 lib/{relaysrv => relay}/protocol/packets.go | 0 .../protocol/packets_xdr.go | 0 lib/{relaysrv => relay}/protocol/protocol.go | 0 lib/relaysrv/.gitignore | 27 -- lib/relaysrv/CONTRIBUTORS | 1 - lib/relaysrv/LICENSE | 22 -- lib/relaysrv/README.md | 6 - lib/relaysrv/build.sh | 39 --- lib/relaysrv/listener.go | 319 ------------------ lib/relaysrv/main.go | 142 -------- lib/relaysrv/pool.go | 68 ---- lib/relaysrv/session.go | 313 ----------------- lib/relaysrv/status.go | 94 ------ lib/relaysrv/testutil/main.go | 149 -------- lib/relaysrv/utils.go | 28 -- 18 files changed, 1208 deletions(-) rename lib/{relaysrv => relay}/client/client.go (100%) rename lib/{relaysrv => relay}/client/debug.go (100%) rename lib/{relaysrv => relay}/client/methods.go (100%) rename lib/{relaysrv => relay}/protocol/packets.go (100%) rename lib/{relaysrv => relay}/protocol/packets_xdr.go (100%) rename lib/{relaysrv => relay}/protocol/protocol.go (100%) delete mode 100644 lib/relaysrv/.gitignore delete mode 100644 lib/relaysrv/CONTRIBUTORS delete mode 100644 lib/relaysrv/LICENSE delete mode 100644 lib/relaysrv/README.md delete mode 100755 lib/relaysrv/build.sh delete mode 100644 lib/relaysrv/listener.go delete mode 100644 lib/relaysrv/main.go delete mode 100644 lib/relaysrv/pool.go delete mode 100644 lib/relaysrv/session.go delete mode 100644 lib/relaysrv/status.go delete mode 100644 lib/relaysrv/testutil/main.go delete mode 100644 lib/relaysrv/utils.go diff --git a/lib/relaysrv/client/client.go b/lib/relay/client/client.go similarity index 100% rename from lib/relaysrv/client/client.go rename to lib/relay/client/client.go diff --git a/lib/relaysrv/client/debug.go b/lib/relay/client/debug.go similarity index 100% rename from lib/relaysrv/client/debug.go rename to lib/relay/client/debug.go diff --git a/lib/relaysrv/client/methods.go b/lib/relay/client/methods.go similarity index 100% rename from lib/relaysrv/client/methods.go rename to lib/relay/client/methods.go diff --git a/lib/relaysrv/protocol/packets.go b/lib/relay/protocol/packets.go similarity index 100% rename from lib/relaysrv/protocol/packets.go rename to lib/relay/protocol/packets.go diff --git a/lib/relaysrv/protocol/packets_xdr.go b/lib/relay/protocol/packets_xdr.go similarity index 100% rename from lib/relaysrv/protocol/packets_xdr.go rename to lib/relay/protocol/packets_xdr.go diff --git a/lib/relaysrv/protocol/protocol.go b/lib/relay/protocol/protocol.go similarity index 100% rename from lib/relaysrv/protocol/protocol.go rename to lib/relay/protocol/protocol.go diff --git a/lib/relaysrv/.gitignore b/lib/relaysrv/.gitignore deleted file mode 100644 index 775c43cc2..000000000 --- a/lib/relaysrv/.gitignore +++ /dev/null @@ -1,27 +0,0 @@ -# Compiled Object files, Static and Dynamic libs (Shared Objects) -*.o -*.a -*.so - -# Folders -_obj -_test - -# Architecture specific extensions/prefixes -*.[568vq] -[568vq].out - -*.cgo1.go -*.cgo2.c -_cgo_defun.c -_cgo_gotypes.go -_cgo_export.* - -_testmain.go - -*.exe -*.test -*.prof -*.tar.gz -*.zip -relaysrv diff --git a/lib/relaysrv/CONTRIBUTORS b/lib/relaysrv/CONTRIBUTORS deleted file mode 100644 index 4b00258cc..000000000 --- a/lib/relaysrv/CONTRIBUTORS +++ /dev/null @@ -1 +0,0 @@ -Jakob Borg diff --git a/lib/relaysrv/LICENSE b/lib/relaysrv/LICENSE deleted file mode 100644 index 581a17054..000000000 --- a/lib/relaysrv/LICENSE +++ /dev/null @@ -1,22 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2015 The Syncthing Project - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - diff --git a/lib/relaysrv/README.md b/lib/relaysrv/README.md deleted file mode 100644 index e88929280..000000000 --- a/lib/relaysrv/README.md +++ /dev/null @@ -1,6 +0,0 @@ -relaysrv -======== - -This is the relay server for the `syncthing` project. - -`go get github.com/syncthing/relaysrv` diff --git a/lib/relaysrv/build.sh b/lib/relaysrv/build.sh deleted file mode 100755 index 42c2681dd..000000000 --- a/lib/relaysrv/build.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -set -euo pipefail -set nullglob - -echo Get dependencies -go get -d - -rm -rf relaysrv-*-* - -build() { - export GOOS="$1" - export GOARCH="$2" - target="relaysrv-$GOOS-$GOARCH" - go build -i -v -ldflags -w - mkdir "$target" - if [ -f relaysrv ] ; then - mv relaysrv "$target" - tar zcvf "$target.tar.gz" "$target" - rm -r "$target" - fi - if [ -f relaysrv.exe ] ; then - mv relaysrv.exe "$target" - zip -r "$target.zip" "$target" - rm -r "$target" - fi -} - -for goos in linux darwin windows freebsd openbsd netbsd solaris ; do - build "$goos" amd64 -done -for goos in linux windows freebsd openbsd netbsd ; do - build "$goos" 386 -done -build linux arm - -# Hack used because we run as root under Docker -if [[ ${CHOWN_USER:-} != "" ]] ; then - chown -R $CHOWN_USER . -fi diff --git a/lib/relaysrv/listener.go b/lib/relaysrv/listener.go deleted file mode 100644 index 3f584535e..000000000 --- a/lib/relaysrv/listener.go +++ /dev/null @@ -1,319 +0,0 @@ -// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file). - -package main - -import ( - "crypto/tls" - "encoding/hex" - "log" - "net" - "sync" - "sync/atomic" - "time" - - syncthingprotocol "github.com/syncthing/protocol" - "github.com/syncthing/syncthing/lib/tlsutil" - - "github.com/syncthing/relaysrv/protocol" -) - -var ( - outboxesMut = sync.RWMutex{} - outboxes = make(map[syncthingprotocol.DeviceID]chan interface{}) - numConnections int64 -) - -func listener(addr string, config *tls.Config) { - tcpListener, err := net.Listen("tcp", addr) - if err != nil { - log.Fatalln(err) - } - - listener := tlsutil.DowngradingListener{tcpListener, nil} - - for { - conn, isTLS, err := listener.AcceptNoWrapTLS() - if err != nil { - if debug { - log.Println("Listener failed to accept connection from", conn.RemoteAddr(), ". Possibly a TCP Ping.") - } - continue - } - - setTCPOptions(conn) - - if debug { - log.Println("Listener accepted connection from", conn.RemoteAddr(), "tls", isTLS) - } - - if isTLS { - go protocolConnectionHandler(conn, config) - } else { - go sessionConnectionHandler(conn) - } - - } -} - -func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config) { - conn := tls.Server(tcpConn, config) - err := conn.Handshake() - if err != nil { - if debug { - log.Println("Protocol connection TLS handshake:", conn.RemoteAddr(), err) - } - conn.Close() - return - } - - state := conn.ConnectionState() - if (!state.NegotiatedProtocolIsMutual || state.NegotiatedProtocol != protocol.ProtocolName) && debug { - log.Println("Protocol negotiation error") - } - - certs := state.PeerCertificates - if len(certs) != 1 { - if debug { - log.Println("Certificate list error") - } - conn.Close() - return - } - - id := syncthingprotocol.NewDeviceID(certs[0].Raw) - - messages := make(chan interface{}) - errors := make(chan error, 1) - outbox := make(chan interface{}) - - // Read messages from the connection and send them on the messages - // channel. When there is an error, send it on the error channel and - // return. Applies also when the connection gets closed, so the pattern - // below is to close the connection on error, then wait for the error - // signal from messageReader to exit. - go messageReader(conn, messages, errors) - - pingTicker := time.NewTicker(pingInterval) - timeoutTicker := time.NewTimer(networkTimeout) - joined := false - - for { - select { - case message := <-messages: - timeoutTicker.Reset(networkTimeout) - if debug { - log.Printf("Message %T from %s", message, id) - } - - switch msg := message.(type) { - case protocol.JoinRelayRequest: - outboxesMut.RLock() - _, ok := outboxes[id] - outboxesMut.RUnlock() - if ok { - protocol.WriteMessage(conn, protocol.ResponseAlreadyConnected) - if debug { - log.Println("Already have a peer with the same ID", id, conn.RemoteAddr()) - } - conn.Close() - continue - } - - outboxesMut.Lock() - outboxes[id] = outbox - outboxesMut.Unlock() - joined = true - - protocol.WriteMessage(conn, protocol.ResponseSuccess) - - case protocol.ConnectRequest: - requestedPeer := syncthingprotocol.DeviceIDFromBytes(msg.ID) - outboxesMut.RLock() - peerOutbox, ok := outboxes[requestedPeer] - outboxesMut.RUnlock() - if !ok { - if debug { - log.Println(id, "is looking for", requestedPeer, "which does not exist") - } - protocol.WriteMessage(conn, protocol.ResponseNotFound) - conn.Close() - continue - } - // requestedPeer is the server, id is the client - ses := newSession(requestedPeer, id, sessionLimiter, globalLimiter) - - go ses.Serve() - - clientInvitation := ses.GetClientInvitationMessage() - serverInvitation := ses.GetServerInvitationMessage() - - if err := protocol.WriteMessage(conn, clientInvitation); err != nil { - if debug { - log.Printf("Error sending invitation from %s to client: %s", id, err) - } - conn.Close() - continue - } - - peerOutbox <- serverInvitation - - if debug { - log.Println("Sent invitation from", id, "to", requestedPeer) - } - conn.Close() - - case protocol.Ping: - if err := protocol.WriteMessage(conn, protocol.Pong{}); err != nil { - if debug { - log.Println("Error writing pong:", err) - } - conn.Close() - continue - } - - case protocol.Pong: - // Nothing - - default: - if debug { - log.Printf("Unknown message %s: %T", id, message) - } - protocol.WriteMessage(conn, protocol.ResponseUnexpectedMessage) - conn.Close() - } - - case err := <-errors: - if debug { - log.Printf("Closing connection %s: %s", id, err) - } - close(outbox) - - // Potentially closing a second time. - conn.Close() - - if joined { - // Only delete the outbox if the client is joined, as it might be - // a lookup request coming from the same client. - outboxesMut.Lock() - delete(outboxes, id) - outboxesMut.Unlock() - // Also, kill all sessions related to this node, as it probably - // went offline. This is for the other end to realize the client - // is no longer there faster. This also helps resolve - // 'already connected' errors when one of the sides is - // restarting, and connecting to the other peer before the other - // peer even realised that the node has gone away. - dropSessions(id) - } - return - - case <-pingTicker.C: - if !joined { - if debug { - log.Println(id, "didn't join within", pingInterval) - } - conn.Close() - continue - } - - if err := protocol.WriteMessage(conn, protocol.Ping{}); err != nil { - if debug { - log.Println(id, err) - } - conn.Close() - } - - case <-timeoutTicker.C: - // We should receive a error from the reader loop, which will cause - // us to quit this loop. - if debug { - log.Printf("%s timed out", id) - } - conn.Close() - - case msg := <-outbox: - if debug { - log.Printf("Sending message %T to %s", msg, id) - } - if err := protocol.WriteMessage(conn, msg); err != nil { - if debug { - log.Println(id, err) - } - conn.Close() - } - } - } -} - -func sessionConnectionHandler(conn net.Conn) { - if err := conn.SetDeadline(time.Now().Add(messageTimeout)); err != nil { - if debug { - log.Println("Weird error setting deadline:", err, "on", conn.RemoteAddr()) - } - return - } - - message, err := protocol.ReadMessage(conn) - if err != nil { - return - } - - switch msg := message.(type) { - case protocol.JoinSessionRequest: - ses := findSession(string(msg.Key)) - if debug { - log.Println(conn.RemoteAddr(), "session lookup", ses, hex.EncodeToString(msg.Key)[:5]) - } - - if ses == nil { - protocol.WriteMessage(conn, protocol.ResponseNotFound) - conn.Close() - return - } - - if !ses.AddConnection(conn) { - if debug { - log.Println("Failed to add", conn.RemoteAddr(), "to session", ses) - } - protocol.WriteMessage(conn, protocol.ResponseAlreadyConnected) - conn.Close() - return - } - - if err := protocol.WriteMessage(conn, protocol.ResponseSuccess); err != nil { - if debug { - log.Println("Failed to send session join response to ", conn.RemoteAddr(), "for", ses) - } - return - } - - if err := conn.SetDeadline(time.Time{}); err != nil { - if debug { - log.Println("Weird error setting deadline:", err, "on", conn.RemoteAddr()) - } - conn.Close() - return - } - - default: - if debug { - log.Println("Unexpected message from", conn.RemoteAddr(), message) - } - protocol.WriteMessage(conn, protocol.ResponseUnexpectedMessage) - conn.Close() - } -} - -func messageReader(conn net.Conn, messages chan<- interface{}, errors chan<- error) { - atomic.AddInt64(&numConnections, 1) - defer atomic.AddInt64(&numConnections, -1) - - for { - msg, err := protocol.ReadMessage(conn) - if err != nil { - errors <- err - return - } - messages <- msg - } -} diff --git a/lib/relaysrv/main.go b/lib/relaysrv/main.go deleted file mode 100644 index 614f82c7f..000000000 --- a/lib/relaysrv/main.go +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file). - -package main - -import ( - "crypto/tls" - "flag" - "fmt" - "log" - "net" - "net/url" - "path/filepath" - "strings" - "time" - - "github.com/juju/ratelimit" - "github.com/syncthing/relaysrv/protocol" - "github.com/syncthing/syncthing/lib/tlsutil" - - syncthingprotocol "github.com/syncthing/protocol" -) - -var ( - listen string - debug bool = false - - sessionAddress []byte - sessionPort uint16 - - networkTimeout time.Duration = 2 * time.Minute - pingInterval time.Duration = time.Minute - messageTimeout time.Duration = time.Minute - - sessionLimitBps int - globalLimitBps int - sessionLimiter *ratelimit.Bucket - globalLimiter *ratelimit.Bucket - - statusAddr string - poolAddrs string - defaultPoolAddrs string = "https://relays.syncthing.net" -) - -func main() { - log.SetFlags(log.Lshortfile | log.LstdFlags) - - var dir, extAddress string - - flag.StringVar(&listen, "listen", ":22067", "Protocol listen address") - flag.StringVar(&dir, "keys", ".", "Directory where cert.pem and key.pem is stored") - flag.DurationVar(&networkTimeout, "network-timeout", networkTimeout, "Timeout for network operations between the client and the relay.\n\tIf no data is received between the client and the relay in this period of time, the connection is terminated.\n\tFurthermore, if no data is sent between either clients being relayed within this period of time, the session is also terminated.") - flag.DurationVar(&pingInterval, "ping-interval", pingInterval, "How often pings are sent") - flag.DurationVar(&messageTimeout, "message-timeout", messageTimeout, "Maximum amount of time we wait for relevant messages to arrive") - flag.IntVar(&sessionLimitBps, "per-session-rate", sessionLimitBps, "Per session rate limit, in bytes/s") - flag.IntVar(&globalLimitBps, "global-rate", globalLimitBps, "Global rate limit, in bytes/s") - flag.BoolVar(&debug, "debug", debug, "Enable debug output") - flag.StringVar(&statusAddr, "status-srv", ":22070", "Listen address for status service (blank to disable)") - flag.StringVar(&poolAddrs, "pools", defaultPoolAddrs, "Comma separated list of relay pool addresses to join") - - flag.Parse() - - if extAddress == "" { - extAddress = listen - } - - addr, err := net.ResolveTCPAddr("tcp", extAddress) - if err != nil { - log.Fatal(err) - } - - sessionAddress = addr.IP[:] - sessionPort = uint16(addr.Port) - - certFile, keyFile := filepath.Join(dir, "cert.pem"), filepath.Join(dir, "key.pem") - cert, err := tls.LoadX509KeyPair(certFile, keyFile) - if err != nil { - log.Println("Failed to load keypair. Generating one, this might take a while...") - cert, err = tlsutil.NewCertificate(certFile, keyFile, "relaysrv", 3072) - if err != nil { - log.Fatalln("Failed to generate X509 key pair:", err) - } - } - - tlsCfg := &tls.Config{ - Certificates: []tls.Certificate{cert}, - NextProtos: []string{protocol.ProtocolName}, - ClientAuth: tls.RequestClientCert, - SessionTicketsDisabled: true, - InsecureSkipVerify: true, - MinVersion: tls.VersionTLS12, - CipherSuites: []uint16{ - tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, - tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, - tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, - tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, - tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, - tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, - }, - } - - id := syncthingprotocol.NewDeviceID(cert.Certificate[0]) - if debug { - log.Println("ID:", id) - } - - if sessionLimitBps > 0 { - sessionLimiter = ratelimit.NewBucketWithRate(float64(sessionLimitBps), int64(2*sessionLimitBps)) - } - if globalLimitBps > 0 { - globalLimiter = ratelimit.NewBucketWithRate(float64(globalLimitBps), int64(2*globalLimitBps)) - } - - if statusAddr != "" { - go statusService(statusAddr) - } - - uri, err := url.Parse(fmt.Sprintf("relay://%s/?id=%s&pingInterval=%s&networkTimeout=%s&sessionLimitBps=%d&globalLimitBps=%d&statusAddr=%s", extAddress, id, pingInterval, networkTimeout, sessionLimitBps, globalLimitBps, statusAddr)) - if err != nil { - log.Fatalln("Failed to construct URI", err) - } - - if debug { - log.Println("URI:", uri.String()) - } - - if poolAddrs == defaultPoolAddrs { - log.Println("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") - log.Println("!! Joining default relay pools, this relay will be available for public use. !!") - log.Println(`!! Use the -pools="" command line option to make the relay private. !!`) - log.Println("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") - } - - pools := strings.Split(poolAddrs, ",") - for _, pool := range pools { - pool = strings.TrimSpace(pool) - if len(pool) > 0 { - go poolHandler(pool, uri) - } - } - - listener(listen, tlsCfg) -} diff --git a/lib/relaysrv/pool.go b/lib/relaysrv/pool.go deleted file mode 100644 index 0f327f690..000000000 --- a/lib/relaysrv/pool.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file). - -package main - -import ( - "bytes" - "encoding/json" - "io/ioutil" - "log" - "net/http" - "net/url" - "time" -) - -func poolHandler(pool string, uri *url.URL) { - for { - var b bytes.Buffer - json.NewEncoder(&b).Encode(struct { - URL string `json:"url"` - }{ - uri.String(), - }) - - resp, err := http.Post(pool, "application/json", &b) - if err != nil { - if debug { - log.Println("Error joining pool", pool, err) - } - } else if resp.StatusCode == 500 { - if debug { - bs, err := ioutil.ReadAll(resp.Body) - if err != nil { - log.Println("Failed to read response body for", pool, err) - } else { - log.Println("Response for", pool, string(bs)) - } - resp.Body.Close() - } - } else if resp.StatusCode == 429 { - if debug { - log.Println(pool, "under load, will retry in a minute") - } - time.Sleep(time.Minute) - continue - } else if resp.StatusCode == 403 { - if debug { - log.Println(pool, "failed to join due to IP address not matching external address") - } - return - } else if resp.StatusCode == 200 { - var x struct { - EvictionIn time.Duration `json:"evictionIn"` - } - err := json.NewDecoder(resp.Body).Decode(&x) - if err == nil { - rejoin := x.EvictionIn - (x.EvictionIn / 5) - if debug { - log.Println("Joined", pool, "rejoining in", rejoin) - } - time.Sleep(rejoin) - continue - } else if debug { - log.Println("Failed to deserialize respnse", err) - } - } - time.Sleep(time.Hour) - } -} diff --git a/lib/relaysrv/session.go b/lib/relaysrv/session.go deleted file mode 100644 index bbd29d1f5..000000000 --- a/lib/relaysrv/session.go +++ /dev/null @@ -1,313 +0,0 @@ -// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file). - -package main - -import ( - "crypto/rand" - "encoding/hex" - "fmt" - "log" - "net" - "sync" - "sync/atomic" - "time" - - "github.com/juju/ratelimit" - "github.com/syncthing/relaysrv/protocol" - - syncthingprotocol "github.com/syncthing/protocol" -) - -var ( - sessionMut = sync.RWMutex{} - activeSessions = make([]*session, 0) - pendingSessions = make(map[string]*session, 0) - numProxies int64 - bytesProxied int64 -) - -func newSession(serverid, clientid syncthingprotocol.DeviceID, sessionRateLimit, globalRateLimit *ratelimit.Bucket) *session { - serverkey := make([]byte, 32) - _, err := rand.Read(serverkey) - if err != nil { - return nil - } - - clientkey := make([]byte, 32) - _, err = rand.Read(clientkey) - if err != nil { - return nil - } - - ses := &session{ - serverkey: serverkey, - serverid: serverid, - clientkey: clientkey, - clientid: clientid, - rateLimit: makeRateLimitFunc(sessionRateLimit, globalRateLimit), - connsChan: make(chan net.Conn), - conns: make([]net.Conn, 0, 2), - } - - if debug { - log.Println("New session", ses) - } - - sessionMut.Lock() - pendingSessions[string(ses.serverkey)] = ses - pendingSessions[string(ses.clientkey)] = ses - sessionMut.Unlock() - - return ses -} - -func findSession(key string) *session { - sessionMut.Lock() - defer sessionMut.Unlock() - ses, ok := pendingSessions[key] - if !ok { - return nil - - } - delete(pendingSessions, key) - return ses -} - -func dropSessions(id syncthingprotocol.DeviceID) { - sessionMut.RLock() - for _, session := range activeSessions { - if session.HasParticipant(id) { - if debug { - log.Println("Dropping session", session, "involving", id) - } - session.CloseConns() - } - } - sessionMut.RUnlock() -} - -type session struct { - mut sync.Mutex - - serverkey []byte - serverid syncthingprotocol.DeviceID - - clientkey []byte - clientid syncthingprotocol.DeviceID - - rateLimit func(bytes int64) - - connsChan chan net.Conn - conns []net.Conn -} - -func (s *session) AddConnection(conn net.Conn) bool { - if debug { - log.Println("New connection for", s, "from", conn.RemoteAddr()) - } - - select { - case s.connsChan <- conn: - return true - default: - } - return false -} - -func (s *session) Serve() { - timedout := time.After(messageTimeout) - - if debug { - log.Println("Session", s, "serving") - } - - for { - select { - case conn := <-s.connsChan: - s.mut.Lock() - s.conns = append(s.conns, conn) - s.mut.Unlock() - // We're the only ones mutating% s.conns, hence we are free to read it. - if len(s.conns) < 2 { - continue - } - - close(s.connsChan) - - if debug { - log.Println("Session", s, "starting between", s.conns[0].RemoteAddr(), "and", s.conns[1].RemoteAddr()) - } - - wg := sync.WaitGroup{} - wg.Add(2) - - var err0 error - go func() { - err0 = s.proxy(s.conns[0], s.conns[1]) - wg.Done() - }() - - var err1 error - go func() { - err1 = s.proxy(s.conns[1], s.conns[0]) - wg.Done() - }() - - sessionMut.Lock() - activeSessions = append(activeSessions, s) - sessionMut.Unlock() - - wg.Wait() - - if debug { - log.Println("Session", s, "ended, outcomes:", err0, "and", err1) - } - goto done - - case <-timedout: - if debug { - log.Println("Session", s, "timed out") - } - goto done - } - } -done: - // We can end up here in 3 cases: - // 1. Timeout joining, in which case there are potentially entries in pendingSessions - // 2. General session end/timeout, in which case there are entries in activeSessions - // 3. Protocol handler calls dropSession as one of it's clients disconnects. - - sessionMut.Lock() - delete(pendingSessions, string(s.serverkey)) - delete(pendingSessions, string(s.clientkey)) - - for i, session := range activeSessions { - if session == s { - l := len(activeSessions) - 1 - activeSessions[i] = activeSessions[l] - activeSessions[l] = nil - activeSessions = activeSessions[:l] - } - } - sessionMut.Unlock() - - // If we are here because of case 2 or 3, we are potentially closing some or - // all connections a second time. - s.CloseConns() - - if debug { - log.Println("Session", s, "stopping") - } -} - -func (s *session) GetClientInvitationMessage() protocol.SessionInvitation { - return protocol.SessionInvitation{ - From: s.serverid[:], - Key: []byte(s.clientkey), - Address: sessionAddress, - Port: sessionPort, - ServerSocket: false, - } -} - -func (s *session) GetServerInvitationMessage() protocol.SessionInvitation { - return protocol.SessionInvitation{ - From: s.clientid[:], - Key: []byte(s.serverkey), - Address: sessionAddress, - Port: sessionPort, - ServerSocket: true, - } -} - -func (s *session) HasParticipant(id syncthingprotocol.DeviceID) bool { - return s.clientid == id || s.serverid == id -} - -func (s *session) CloseConns() { - s.mut.Lock() - for _, conn := range s.conns { - conn.Close() - } - s.mut.Unlock() -} - -func (s *session) proxy(c1, c2 net.Conn) error { - if debug { - log.Println("Proxy", c1.RemoteAddr(), "->", c2.RemoteAddr()) - } - - atomic.AddInt64(&numProxies, 1) - defer atomic.AddInt64(&numProxies, -1) - - buf := make([]byte, 65536) - for { - c1.SetReadDeadline(time.Now().Add(networkTimeout)) - n, err := c1.Read(buf) - if err != nil { - return err - } - - atomic.AddInt64(&bytesProxied, int64(n)) - - if debug { - log.Printf("%d bytes from %s to %s", n, c1.RemoteAddr(), c2.RemoteAddr()) - } - - if s.rateLimit != nil { - s.rateLimit(int64(n)) - } - - c2.SetWriteDeadline(time.Now().Add(networkTimeout)) - _, err = c2.Write(buf[:n]) - if err != nil { - return err - } - } -} - -func (s *session) String() string { - return fmt.Sprintf("<%s/%s>", hex.EncodeToString(s.clientkey)[:5], hex.EncodeToString(s.serverkey)[:5]) -} - -func makeRateLimitFunc(sessionRateLimit, globalRateLimit *ratelimit.Bucket) func(int64) { - // This may be a case of super duper premature optimization... We build an - // optimized function to do the rate limiting here based on what we need - // to do and then use it in the loop. - - if sessionRateLimit == nil && globalRateLimit == nil { - // No limiting needed. We could equally well return a func(int64){} and - // not do a nil check were we use it, but I think the nil check there - // makes it clear that there will be no limiting if none is - // configured... - return nil - } - - if sessionRateLimit == nil { - // We only have a global limiter - return func(bytes int64) { - globalRateLimit.Wait(bytes) - } - } - - if globalRateLimit == nil { - // We only have a session limiter - return func(bytes int64) { - sessionRateLimit.Wait(bytes) - } - } - - // We have both. Queue the bytes on both the global and session specific - // rate limiters. Wait for both in parallell, so that the actual send - // happens when both conditions are satisfied. In practice this just means - // wait the longer of the two times. - return func(bytes int64) { - t0 := sessionRateLimit.Take(bytes) - t1 := globalRateLimit.Take(bytes) - if t0 > t1 { - time.Sleep(t0) - } else { - time.Sleep(t1) - } - } -} diff --git a/lib/relaysrv/status.go b/lib/relaysrv/status.go deleted file mode 100644 index b18cf3ea7..000000000 --- a/lib/relaysrv/status.go +++ /dev/null @@ -1,94 +0,0 @@ -package main - -import ( - "encoding/json" - "log" - "net/http" - "runtime" - "sync/atomic" - "time" -) - -var rc *rateCalculator - -func statusService(addr string) { - rc = newRateCalculator(360, 10*time.Second, &bytesProxied) - - http.HandleFunc("/status", getStatus) - if err := http.ListenAndServe(addr, nil); err != nil { - log.Fatal(err) - } -} - -func getStatus(w http.ResponseWriter, r *http.Request) { - status := make(map[string]interface{}) - - sessionMut.Lock() - // This can potentially be double the number of pending sessions, as each session has two keys, one for each side. - status["numPendingSessionKeys"] = len(pendingSessions) - status["numActiveSessions"] = len(activeSessions) - sessionMut.Unlock() - status["numConnections"] = atomic.LoadInt64(&numConnections) - status["numProxies"] = atomic.LoadInt64(&numProxies) - status["bytesProxied"] = atomic.LoadInt64(&bytesProxied) - status["goVersion"] = runtime.Version() - status["goOS"] = runtime.GOOS - status["goAarch"] = runtime.GOARCH - status["goMaxProcs"] = runtime.GOMAXPROCS(-1) - status["kbps10s1m5m15m30m60m"] = []int64{ - rc.rate(10/10) * 8 / 1000, - rc.rate(60/10) * 8 / 1000, - rc.rate(5*60/10) * 8 / 1000, - rc.rate(15*60/10) * 8 / 1000, - rc.rate(30*60/10) * 8 / 1000, - rc.rate(60*60/10) * 8 / 1000, - } - - bs, err := json.MarshalIndent(status, "", " ") - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - w.Header().Set("Content-Type", "application/json") - w.Write(bs) -} - -type rateCalculator struct { - rates []int64 - prev int64 - counter *int64 -} - -func newRateCalculator(keepIntervals int, interval time.Duration, counter *int64) *rateCalculator { - r := &rateCalculator{ - rates: make([]int64, keepIntervals), - counter: counter, - } - - go r.updateRates(interval) - - return r -} - -func (r *rateCalculator) updateRates(interval time.Duration) { - for { - now := time.Now() - next := now.Truncate(interval).Add(interval) - time.Sleep(next.Sub(now)) - - cur := atomic.LoadInt64(r.counter) - rate := int64(float64(cur-r.prev) / interval.Seconds()) - copy(r.rates[1:], r.rates) - r.rates[0] = rate - r.prev = cur - } -} - -func (r *rateCalculator) rate(periods int) int64 { - var tot int64 - for i := 0; i < periods; i++ { - tot += r.rates[i] - } - return tot / int64(periods) -} diff --git a/lib/relaysrv/testutil/main.go b/lib/relaysrv/testutil/main.go deleted file mode 100644 index ffeb9942d..000000000 --- a/lib/relaysrv/testutil/main.go +++ /dev/null @@ -1,149 +0,0 @@ -// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file). - -package main - -import ( - "bufio" - "crypto/tls" - "flag" - "log" - "net" - "net/url" - "os" - "path/filepath" - "time" - - syncthingprotocol "github.com/syncthing/protocol" - "github.com/syncthing/relaysrv/client" - "github.com/syncthing/relaysrv/protocol" -) - -func main() { - log.SetOutput(os.Stdout) - log.SetFlags(log.LstdFlags | log.Lshortfile) - - var connect, relay, dir string - var join, test bool - - flag.StringVar(&connect, "connect", "", "Device ID to which to connect to") - flag.BoolVar(&join, "join", false, "Join relay") - flag.BoolVar(&test, "test", false, "Generic relay test") - flag.StringVar(&relay, "relay", "relay://127.0.0.1:22067", "Relay address") - flag.StringVar(&dir, "keys", ".", "Directory where cert.pem and key.pem is stored") - - flag.Parse() - - certFile, keyFile := filepath.Join(dir, "cert.pem"), filepath.Join(dir, "key.pem") - cert, err := tls.LoadX509KeyPair(certFile, keyFile) - if err != nil { - log.Fatalln("Failed to load X509 key pair:", err) - } - - id := syncthingprotocol.NewDeviceID(cert.Certificate[0]) - log.Println("ID:", id) - - uri, err := url.Parse(relay) - if err != nil { - log.Fatal(err) - } - - stdin := make(chan string) - - go stdinReader(stdin) - - if join { - log.Println("Creating client") - relay := client.NewProtocolClient(uri, []tls.Certificate{cert}, nil) - log.Println("Created client") - - go relay.Serve() - - recv := make(chan protocol.SessionInvitation) - - go func() { - log.Println("Starting invitation receiver") - for invite := range relay.Invitations { - select { - case recv <- invite: - log.Println("Received invitation", invite) - default: - log.Println("Discarding invitation", invite) - } - } - }() - - for { - conn, err := client.JoinSession(<-recv) - if err != nil { - log.Fatalln("Failed to join", err) - } - log.Println("Joined", conn.RemoteAddr(), conn.LocalAddr()) - connectToStdio(stdin, conn) - log.Println("Finished", conn.RemoteAddr(), conn.LocalAddr()) - } - } else if connect != "" { - id, err := syncthingprotocol.DeviceIDFromString(connect) - if err != nil { - log.Fatal(err) - } - - invite, err := client.GetInvitationFromRelay(uri, id, []tls.Certificate{cert}) - if err != nil { - log.Fatal(err) - } - - log.Println("Received invitation", invite) - conn, err := client.JoinSession(invite) - if err != nil { - log.Fatalln("Failed to join", err) - } - log.Println("Joined", conn.RemoteAddr(), conn.LocalAddr()) - connectToStdio(stdin, conn) - log.Println("Finished", conn.RemoteAddr(), conn.LocalAddr()) - } else if test { - if client.TestRelay(uri, []tls.Certificate{cert}) { - log.Println("OK") - } else { - log.Println("FAIL") - } - } else { - log.Fatal("Requires either join or connect") - } -} - -func stdinReader(c chan<- string) { - scanner := bufio.NewScanner(os.Stdin) - for scanner.Scan() { - c <- scanner.Text() - c <- "\n" - } -} - -func connectToStdio(stdin <-chan string, conn net.Conn) { - go func() { - - }() - - buf := make([]byte, 1024) - for { - conn.SetReadDeadline(time.Now().Add(time.Millisecond)) - n, err := conn.Read(buf[0:]) - if err != nil { - nerr, ok := err.(net.Error) - if !ok || !nerr.Timeout() { - log.Println(err) - return - } - } - os.Stdout.Write(buf[:n]) - - select { - case msg := <-stdin: - _, err := conn.Write([]byte(msg)) - if err != nil { - return - } - default: - } - } -} diff --git a/lib/relaysrv/utils.go b/lib/relaysrv/utils.go deleted file mode 100644 index 7d1f6bfa4..000000000 --- a/lib/relaysrv/utils.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file). - -package main - -import ( - "errors" - "net" -) - -func setTCPOptions(conn net.Conn) error { - tcpConn, ok := conn.(*net.TCPConn) - if !ok { - return errors.New("Not a TCP connection") - } - if err := tcpConn.SetLinger(0); err != nil { - return err - } - if err := tcpConn.SetNoDelay(true); err != nil { - return err - } - if err := tcpConn.SetKeepAlivePeriod(networkTimeout); err != nil { - return err - } - if err := tcpConn.SetKeepAlive(true); err != nil { - return err - } - return nil -} From 4581c57478eb1b93d681b113eb2f609f30a9b312 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Tue, 22 Sep 2015 19:38:46 +0200 Subject: [PATCH 93/94] Fix import paths --- .../src/github.com/syncthing/relaysrv/client/client.go | 4 ++-- .../src/github.com/syncthing/relaysrv/client/methods.go | 4 ++-- .../src/github.com/syncthing/relaysrv/protocol/packets.go | 2 +- cmd/stfileinfo/main.go | 2 +- cmd/stfinddevice/main.go | 2 +- cmd/stindex/main.go | 2 +- cmd/syncthing/connections.go | 4 ++-- cmd/syncthing/gui.go | 2 +- cmd/syncthing/main.go | 2 +- cmd/syncthing/main_test.go | 2 +- cmd/syncthing/perfstats_unix.go | 2 +- cmd/syncthing/usage_report.go | 2 +- lib/config/config.go | 2 +- lib/config/config_test.go | 2 +- lib/config/wrapper.go | 2 +- lib/db/blockmap.go | 2 +- lib/db/blockmap_test.go | 2 +- lib/db/leveldb.go | 2 +- lib/db/set.go | 2 +- lib/db/set_test.go | 2 +- lib/db/truncated.go | 2 +- lib/discover/cache.go | 2 +- lib/discover/discover.go | 2 +- lib/discover/global.go | 2 +- lib/discover/global_test.go | 2 +- lib/discover/local.go | 2 +- lib/model/connection.go | 2 +- lib/model/deviceactivity.go | 2 +- lib/model/deviceactivity_test.go | 2 +- lib/model/model.go | 2 +- lib/model/model_test.go | 2 +- lib/model/rwfolder.go | 2 +- lib/model/rwfolder_test.go | 2 +- lib/model/sharedpullerstate.go | 2 +- lib/rc/rc.go | 2 +- lib/relay/client/client.go | 4 ++-- lib/relay/client/methods.go | 4 ++-- lib/relay/protocol/packets.go | 2 +- lib/relay/relay.go | 4 ++-- lib/scanner/blockqueue.go | 2 +- lib/scanner/blocks.go | 2 +- lib/scanner/blocks_test.go | 2 +- lib/scanner/walk.go | 2 +- lib/scanner/walk_test.go | 2 +- lib/symlinks/symlink_windows.go | 2 +- test/filetype_test.go | 2 +- test/http_test.go | 2 +- test/manypeers_test.go | 2 +- test/norestart_test.go | 2 +- test/override_test.go | 2 +- test/symlink_test.go | 2 +- test/sync_test.go | 2 +- 52 files changed, 58 insertions(+), 58 deletions(-) diff --git a/Godeps/_workspace/src/github.com/syncthing/relaysrv/client/client.go b/Godeps/_workspace/src/github.com/syncthing/relaysrv/client/client.go index 89b16e000..2e1b51e03 100644 --- a/Godeps/_workspace/src/github.com/syncthing/relaysrv/client/client.go +++ b/Godeps/_workspace/src/github.com/syncthing/relaysrv/client/client.go @@ -10,8 +10,8 @@ import ( "net/url" "time" - syncthingprotocol "github.com/syncthing/protocol" - "github.com/syncthing/relaysrv/protocol" + syncthingprotocol "github.com/syncthing/syncthing/lib/protocol" + "github.com/syncthing/syncthing/lib/relay/protocol" "github.com/syncthing/syncthing/lib/sync" ) diff --git a/Godeps/_workspace/src/github.com/syncthing/relaysrv/client/methods.go b/Godeps/_workspace/src/github.com/syncthing/relaysrv/client/methods.go index 67a9a71c1..ced788b8a 100644 --- a/Godeps/_workspace/src/github.com/syncthing/relaysrv/client/methods.go +++ b/Godeps/_workspace/src/github.com/syncthing/relaysrv/client/methods.go @@ -11,8 +11,8 @@ import ( "strings" "time" - syncthingprotocol "github.com/syncthing/protocol" - "github.com/syncthing/relaysrv/protocol" + syncthingprotocol "github.com/syncthing/syncthing/lib/protocol" + "github.com/syncthing/syncthing/lib/relay/protocol" ) func GetInvitationFromRelay(uri *url.URL, id syncthingprotocol.DeviceID, certs []tls.Certificate) (protocol.SessionInvitation, error) { diff --git a/Godeps/_workspace/src/github.com/syncthing/relaysrv/protocol/packets.go b/Godeps/_workspace/src/github.com/syncthing/relaysrv/protocol/packets.go index 1b21eba24..f11954caa 100644 --- a/Godeps/_workspace/src/github.com/syncthing/relaysrv/protocol/packets.go +++ b/Godeps/_workspace/src/github.com/syncthing/relaysrv/protocol/packets.go @@ -9,7 +9,7 @@ import ( "fmt" "net" - syncthingprotocol "github.com/syncthing/protocol" + syncthingprotocol "github.com/syncthing/syncthing/lib/protocol" ) const ( diff --git a/cmd/stfileinfo/main.go b/cmd/stfileinfo/main.go index e663dc76d..0feffbea5 100644 --- a/cmd/stfileinfo/main.go +++ b/cmd/stfileinfo/main.go @@ -12,7 +12,7 @@ import ( "os" "path/filepath" - "github.com/syncthing/protocol" + "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/scanner" ) diff --git a/cmd/stfinddevice/main.go b/cmd/stfinddevice/main.go index f01fe5fbd..939a3c0f3 100644 --- a/cmd/stfinddevice/main.go +++ b/cmd/stfinddevice/main.go @@ -15,9 +15,9 @@ import ( "os" "time" - "github.com/syncthing/protocol" "github.com/syncthing/syncthing/lib/config" "github.com/syncthing/syncthing/lib/discover" + "github.com/syncthing/syncthing/lib/protocol" ) var timeout = 5 * time.Second diff --git a/cmd/stindex/main.go b/cmd/stindex/main.go index 1b6a5605b..370aab6bb 100644 --- a/cmd/stindex/main.go +++ b/cmd/stindex/main.go @@ -13,8 +13,8 @@ import ( "log" "os" - "github.com/syncthing/protocol" "github.com/syncthing/syncthing/lib/db" + "github.com/syncthing/syncthing/lib/protocol" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/opt" ) diff --git a/cmd/syncthing/connections.go b/cmd/syncthing/connections.go index 1be9db1bb..bdddaec83 100644 --- a/cmd/syncthing/connections.go +++ b/cmd/syncthing/connections.go @@ -15,14 +15,14 @@ import ( "sync" "time" - "github.com/syncthing/protocol" - "github.com/syncthing/relaysrv/client" "github.com/syncthing/syncthing/lib/config" "github.com/syncthing/syncthing/lib/discover" "github.com/syncthing/syncthing/lib/events" "github.com/syncthing/syncthing/lib/model" "github.com/syncthing/syncthing/lib/osutil" + "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/relay" + "github.com/syncthing/syncthing/lib/relay/client" "github.com/thejerf/suture" ) diff --git a/cmd/syncthing/gui.go b/cmd/syncthing/gui.go index ed3697fe5..6a1d4f2ef 100644 --- a/cmd/syncthing/gui.go +++ b/cmd/syncthing/gui.go @@ -25,7 +25,6 @@ import ( "time" "github.com/calmh/logger" - "github.com/syncthing/protocol" "github.com/syncthing/syncthing/lib/auto" "github.com/syncthing/syncthing/lib/config" "github.com/syncthing/syncthing/lib/db" @@ -33,6 +32,7 @@ import ( "github.com/syncthing/syncthing/lib/events" "github.com/syncthing/syncthing/lib/model" "github.com/syncthing/syncthing/lib/osutil" + "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/relay" "github.com/syncthing/syncthing/lib/sync" "github.com/syncthing/syncthing/lib/tlsutil" diff --git a/cmd/syncthing/main.go b/cmd/syncthing/main.go index dc96196a8..5c7f3c6f2 100644 --- a/cmd/syncthing/main.go +++ b/cmd/syncthing/main.go @@ -27,13 +27,13 @@ import ( "github.com/calmh/logger" "github.com/juju/ratelimit" - "github.com/syncthing/protocol" "github.com/syncthing/syncthing/lib/config" "github.com/syncthing/syncthing/lib/db" "github.com/syncthing/syncthing/lib/discover" "github.com/syncthing/syncthing/lib/events" "github.com/syncthing/syncthing/lib/model" "github.com/syncthing/syncthing/lib/osutil" + "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/relay" "github.com/syncthing/syncthing/lib/symlinks" "github.com/syncthing/syncthing/lib/tlsutil" diff --git a/cmd/syncthing/main_test.go b/cmd/syncthing/main_test.go index b5b71b950..7730b7dfa 100644 --- a/cmd/syncthing/main_test.go +++ b/cmd/syncthing/main_test.go @@ -10,10 +10,10 @@ import ( "os" "testing" - "github.com/syncthing/protocol" "github.com/syncthing/syncthing/lib/config" "github.com/syncthing/syncthing/lib/db" "github.com/syncthing/syncthing/lib/model" + "github.com/syncthing/syncthing/lib/protocol" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/storage" diff --git a/cmd/syncthing/perfstats_unix.go b/cmd/syncthing/perfstats_unix.go index 0ed00bd50..2b9af3637 100644 --- a/cmd/syncthing/perfstats_unix.go +++ b/cmd/syncthing/perfstats_unix.go @@ -15,7 +15,7 @@ import ( "syscall" "time" - "github.com/syncthing/protocol" + "github.com/syncthing/syncthing/lib/protocol" ) func init() { diff --git a/cmd/syncthing/usage_report.go b/cmd/syncthing/usage_report.go index c2ee687c3..0d0505ee5 100644 --- a/cmd/syncthing/usage_report.go +++ b/cmd/syncthing/usage_report.go @@ -19,9 +19,9 @@ import ( "sort" "time" - "github.com/syncthing/protocol" "github.com/syncthing/syncthing/lib/config" "github.com/syncthing/syncthing/lib/model" + "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/upgrade" "github.com/thejerf/suture" ) diff --git a/lib/config/config.go b/lib/config/config.go index 5cd22a446..bd19c32db 100644 --- a/lib/config/config.go +++ b/lib/config/config.go @@ -20,8 +20,8 @@ import ( "strconv" "strings" - "github.com/syncthing/protocol" "github.com/syncthing/syncthing/lib/osutil" + "github.com/syncthing/syncthing/lib/protocol" "golang.org/x/crypto/bcrypt" ) diff --git a/lib/config/config_test.go b/lib/config/config_test.go index b6ff45341..626452f89 100644 --- a/lib/config/config_test.go +++ b/lib/config/config_test.go @@ -17,7 +17,7 @@ import ( "strings" "testing" - "github.com/syncthing/protocol" + "github.com/syncthing/syncthing/lib/protocol" ) var device1, device2, device3, device4 protocol.DeviceID diff --git a/lib/config/wrapper.go b/lib/config/wrapper.go index 394cf82ef..2460fbdae 100644 --- a/lib/config/wrapper.go +++ b/lib/config/wrapper.go @@ -9,9 +9,9 @@ package config import ( "os" - "github.com/syncthing/protocol" "github.com/syncthing/syncthing/lib/events" "github.com/syncthing/syncthing/lib/osutil" + "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/sync" ) diff --git a/lib/db/blockmap.go b/lib/db/blockmap.go index 3caa9ff5b..41354d348 100644 --- a/lib/db/blockmap.go +++ b/lib/db/blockmap.go @@ -17,8 +17,8 @@ import ( "encoding/binary" "fmt" - "github.com/syncthing/protocol" "github.com/syncthing/syncthing/lib/osutil" + "github.com/syncthing/syncthing/lib/protocol" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/util" diff --git a/lib/db/blockmap_test.go b/lib/db/blockmap_test.go index 3a1587038..110383d23 100644 --- a/lib/db/blockmap_test.go +++ b/lib/db/blockmap_test.go @@ -9,7 +9,7 @@ package db import ( "testing" - "github.com/syncthing/protocol" + "github.com/syncthing/syncthing/lib/protocol" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/storage" diff --git a/lib/db/leveldb.go b/lib/db/leveldb.go index ec64bda2c..78f0944e9 100644 --- a/lib/db/leveldb.go +++ b/lib/db/leveldb.go @@ -15,7 +15,7 @@ import ( "runtime" "sort" - "github.com/syncthing/protocol" + "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/sync" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/iterator" diff --git a/lib/db/set.go b/lib/db/set.go index 8924c958b..990cebd7c 100644 --- a/lib/db/set.go +++ b/lib/db/set.go @@ -13,8 +13,8 @@ package db import ( - "github.com/syncthing/protocol" "github.com/syncthing/syncthing/lib/osutil" + "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/sync" "github.com/syndtr/goleveldb/leveldb" ) diff --git a/lib/db/set_test.go b/lib/db/set_test.go index c4318f5c7..abe355c48 100644 --- a/lib/db/set_test.go +++ b/lib/db/set_test.go @@ -13,8 +13,8 @@ import ( "sort" "testing" - "github.com/syncthing/protocol" "github.com/syncthing/syncthing/lib/db" + "github.com/syncthing/syncthing/lib/protocol" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/storage" ) diff --git a/lib/db/truncated.go b/lib/db/truncated.go index 3fa7fcca5..c15e529b2 100644 --- a/lib/db/truncated.go +++ b/lib/db/truncated.go @@ -6,7 +6,7 @@ package db -import "github.com/syncthing/protocol" +import "github.com/syncthing/syncthing/lib/protocol" type FileInfoTruncated struct { protocol.FileInfo diff --git a/lib/discover/cache.go b/lib/discover/cache.go index 2fb99f9f1..ce2f99ce8 100644 --- a/lib/discover/cache.go +++ b/lib/discover/cache.go @@ -10,7 +10,7 @@ import ( stdsync "sync" "time" - "github.com/syncthing/protocol" + "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/sync" "github.com/thejerf/suture" ) diff --git a/lib/discover/discover.go b/lib/discover/discover.go index 8f59ec157..82bd2b7cb 100644 --- a/lib/discover/discover.go +++ b/lib/discover/discover.go @@ -9,7 +9,7 @@ package discover import ( "time" - "github.com/syncthing/protocol" + "github.com/syncthing/syncthing/lib/protocol" "github.com/thejerf/suture" ) diff --git a/lib/discover/global.go b/lib/discover/global.go index 5d5ab3496..a42b765ef 100644 --- a/lib/discover/global.go +++ b/lib/discover/global.go @@ -18,8 +18,8 @@ import ( stdsync "sync" "time" - "github.com/syncthing/protocol" "github.com/syncthing/syncthing/lib/events" + "github.com/syncthing/syncthing/lib/protocol" ) type globalClient struct { diff --git a/lib/discover/global_test.go b/lib/discover/global_test.go index b4fb24e5c..000824241 100644 --- a/lib/discover/global_test.go +++ b/lib/discover/global_test.go @@ -15,7 +15,7 @@ import ( "testing" "time" - "github.com/syncthing/protocol" + "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/tlsutil" ) diff --git a/lib/discover/local.go b/lib/discover/local.go index 71b164743..943c799fe 100644 --- a/lib/discover/local.go +++ b/lib/discover/local.go @@ -17,9 +17,9 @@ import ( "strconv" "time" - "github.com/syncthing/protocol" "github.com/syncthing/syncthing/lib/beacon" "github.com/syncthing/syncthing/lib/events" + "github.com/syncthing/syncthing/lib/protocol" "github.com/thejerf/suture" ) diff --git a/lib/model/connection.go b/lib/model/connection.go index 78e9f0f2c..fb7f930ec 100644 --- a/lib/model/connection.go +++ b/lib/model/connection.go @@ -10,7 +10,7 @@ import ( "crypto/tls" "net" - "github.com/syncthing/protocol" + "github.com/syncthing/syncthing/lib/protocol" ) type IntermediateConnection struct { diff --git a/lib/model/deviceactivity.go b/lib/model/deviceactivity.go index cab14bcee..f6c5ba665 100644 --- a/lib/model/deviceactivity.go +++ b/lib/model/deviceactivity.go @@ -7,7 +7,7 @@ package model import ( - "github.com/syncthing/protocol" + "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/sync" ) diff --git a/lib/model/deviceactivity_test.go b/lib/model/deviceactivity_test.go index 97c8f520f..465957711 100644 --- a/lib/model/deviceactivity_test.go +++ b/lib/model/deviceactivity_test.go @@ -9,7 +9,7 @@ package model import ( "testing" - "github.com/syncthing/protocol" + "github.com/syncthing/syncthing/lib/protocol" ) func TestDeviceActivity(t *testing.T) { diff --git a/lib/model/model.go b/lib/model/model.go index 505e9a39e..a0af20e20 100644 --- a/lib/model/model.go +++ b/lib/model/model.go @@ -22,12 +22,12 @@ import ( stdsync "sync" "time" - "github.com/syncthing/protocol" "github.com/syncthing/syncthing/lib/config" "github.com/syncthing/syncthing/lib/db" "github.com/syncthing/syncthing/lib/events" "github.com/syncthing/syncthing/lib/ignore" "github.com/syncthing/syncthing/lib/osutil" + "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/scanner" "github.com/syncthing/syncthing/lib/stats" "github.com/syncthing/syncthing/lib/symlinks" diff --git a/lib/model/model_test.go b/lib/model/model_test.go index 31df11f4d..6a831b76a 100644 --- a/lib/model/model_test.go +++ b/lib/model/model_test.go @@ -19,9 +19,9 @@ import ( "testing" "time" - "github.com/syncthing/protocol" "github.com/syncthing/syncthing/lib/config" "github.com/syncthing/syncthing/lib/db" + "github.com/syncthing/syncthing/lib/protocol" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/storage" ) diff --git a/lib/model/rwfolder.go b/lib/model/rwfolder.go index fa49ba273..ff5c716e7 100644 --- a/lib/model/rwfolder.go +++ b/lib/model/rwfolder.go @@ -16,12 +16,12 @@ import ( "sort" "time" - "github.com/syncthing/protocol" "github.com/syncthing/syncthing/lib/config" "github.com/syncthing/syncthing/lib/db" "github.com/syncthing/syncthing/lib/events" "github.com/syncthing/syncthing/lib/ignore" "github.com/syncthing/syncthing/lib/osutil" + "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/scanner" "github.com/syncthing/syncthing/lib/symlinks" "github.com/syncthing/syncthing/lib/sync" diff --git a/lib/model/rwfolder_test.go b/lib/model/rwfolder_test.go index df34fdf4a..2d5417e3d 100644 --- a/lib/model/rwfolder_test.go +++ b/lib/model/rwfolder_test.go @@ -12,7 +12,7 @@ import ( "testing" "time" - "github.com/syncthing/protocol" + "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/scanner" "github.com/syncthing/syncthing/lib/sync" diff --git a/lib/model/sharedpullerstate.go b/lib/model/sharedpullerstate.go index 4828dcf5b..5fdd72066 100644 --- a/lib/model/sharedpullerstate.go +++ b/lib/model/sharedpullerstate.go @@ -11,8 +11,8 @@ import ( "os" "path/filepath" - "github.com/syncthing/protocol" "github.com/syncthing/syncthing/lib/db" + "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/sync" ) diff --git a/lib/rc/rc.go b/lib/rc/rc.go index b5975436a..9dd28d0e8 100644 --- a/lib/rc/rc.go +++ b/lib/rc/rc.go @@ -25,8 +25,8 @@ import ( stdsync "sync" "time" - "github.com/syncthing/protocol" "github.com/syncthing/syncthing/lib/config" + "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/sync" ) diff --git a/lib/relay/client/client.go b/lib/relay/client/client.go index 89b16e000..2e1b51e03 100644 --- a/lib/relay/client/client.go +++ b/lib/relay/client/client.go @@ -10,8 +10,8 @@ import ( "net/url" "time" - syncthingprotocol "github.com/syncthing/protocol" - "github.com/syncthing/relaysrv/protocol" + syncthingprotocol "github.com/syncthing/syncthing/lib/protocol" + "github.com/syncthing/syncthing/lib/relay/protocol" "github.com/syncthing/syncthing/lib/sync" ) diff --git a/lib/relay/client/methods.go b/lib/relay/client/methods.go index 67a9a71c1..ced788b8a 100644 --- a/lib/relay/client/methods.go +++ b/lib/relay/client/methods.go @@ -11,8 +11,8 @@ import ( "strings" "time" - syncthingprotocol "github.com/syncthing/protocol" - "github.com/syncthing/relaysrv/protocol" + syncthingprotocol "github.com/syncthing/syncthing/lib/protocol" + "github.com/syncthing/syncthing/lib/relay/protocol" ) func GetInvitationFromRelay(uri *url.URL, id syncthingprotocol.DeviceID, certs []tls.Certificate) (protocol.SessionInvitation, error) { diff --git a/lib/relay/protocol/packets.go b/lib/relay/protocol/packets.go index 1b21eba24..f11954caa 100644 --- a/lib/relay/protocol/packets.go +++ b/lib/relay/protocol/packets.go @@ -9,7 +9,7 @@ import ( "fmt" "net" - syncthingprotocol "github.com/syncthing/protocol" + syncthingprotocol "github.com/syncthing/syncthing/lib/protocol" ) const ( diff --git a/lib/relay/relay.go b/lib/relay/relay.go index 184190a03..514826e40 100644 --- a/lib/relay/relay.go +++ b/lib/relay/relay.go @@ -15,11 +15,11 @@ import ( "sort" "time" - "github.com/syncthing/relaysrv/client" - "github.com/syncthing/relaysrv/protocol" "github.com/syncthing/syncthing/lib/config" "github.com/syncthing/syncthing/lib/events" "github.com/syncthing/syncthing/lib/osutil" + "github.com/syncthing/syncthing/lib/relay/client" + "github.com/syncthing/syncthing/lib/relay/protocol" "github.com/syncthing/syncthing/lib/sync" "github.com/thejerf/suture" diff --git a/lib/scanner/blockqueue.go b/lib/scanner/blockqueue.go index 1ab720ce2..9661c4d34 100644 --- a/lib/scanner/blockqueue.go +++ b/lib/scanner/blockqueue.go @@ -10,7 +10,7 @@ import ( "os" "path/filepath" - "github.com/syncthing/protocol" + "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/sync" ) diff --git a/lib/scanner/blocks.go b/lib/scanner/blocks.go index 06178b113..c9051677b 100644 --- a/lib/scanner/blocks.go +++ b/lib/scanner/blocks.go @@ -13,7 +13,7 @@ import ( "io" "sync/atomic" - "github.com/syncthing/protocol" + "github.com/syncthing/syncthing/lib/protocol" ) var SHA256OfNothing = []uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55} diff --git a/lib/scanner/blocks_test.go b/lib/scanner/blocks_test.go index a52101b7d..9fe8981b2 100644 --- a/lib/scanner/blocks_test.go +++ b/lib/scanner/blocks_test.go @@ -11,7 +11,7 @@ import ( "fmt" "testing" - "github.com/syncthing/protocol" + "github.com/syncthing/syncthing/lib/protocol" ) var blocksTestData = []struct { diff --git a/lib/scanner/walk.go b/lib/scanner/walk.go index ef784d953..0cacc140d 100644 --- a/lib/scanner/walk.go +++ b/lib/scanner/walk.go @@ -16,10 +16,10 @@ import ( "time" "unicode/utf8" - "github.com/syncthing/protocol" "github.com/syncthing/syncthing/lib/db" "github.com/syncthing/syncthing/lib/events" "github.com/syncthing/syncthing/lib/osutil" + "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/symlinks" "golang.org/x/text/unicode/norm" ) diff --git a/lib/scanner/walk_test.go b/lib/scanner/walk_test.go index 64c353094..45cc0e31b 100644 --- a/lib/scanner/walk_test.go +++ b/lib/scanner/walk_test.go @@ -17,9 +17,9 @@ import ( "sort" "testing" - "github.com/syncthing/protocol" "github.com/syncthing/syncthing/lib/ignore" "github.com/syncthing/syncthing/lib/osutil" + "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/symlinks" "golang.org/x/text/unicode/norm" ) diff --git a/lib/symlinks/symlink_windows.go b/lib/symlinks/symlink_windows.go index 0a6a534ab..c40861502 100644 --- a/lib/symlinks/symlink_windows.go +++ b/lib/symlinks/symlink_windows.go @@ -12,8 +12,8 @@ import ( "os" "path/filepath" - "github.com/syncthing/protocol" "github.com/syncthing/syncthing/lib/osutil" + "github.com/syncthing/syncthing/lib/protocol" "syscall" "unicode/utf16" diff --git a/test/filetype_test.go b/test/filetype_test.go index 9a558f46b..fcd61ed60 100644 --- a/test/filetype_test.go +++ b/test/filetype_test.go @@ -13,8 +13,8 @@ import ( "os" "testing" - "github.com/syncthing/protocol" "github.com/syncthing/syncthing/lib/config" + "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/rc" ) diff --git a/test/http_test.go b/test/http_test.go index 19574dba6..095f9dabd 100644 --- a/test/http_test.go +++ b/test/http_test.go @@ -16,7 +16,7 @@ import ( "strings" "testing" - "github.com/syncthing/protocol" + "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/rc" ) diff --git a/test/manypeers_test.go b/test/manypeers_test.go index 55f622301..9ae822f18 100644 --- a/test/manypeers_test.go +++ b/test/manypeers_test.go @@ -14,9 +14,9 @@ import ( "log" "testing" - "github.com/syncthing/protocol" "github.com/syncthing/syncthing/lib/config" "github.com/syncthing/syncthing/lib/osutil" + "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/rc" ) diff --git a/test/norestart_test.go b/test/norestart_test.go index 656942af2..ae21864fc 100644 --- a/test/norestart_test.go +++ b/test/norestart_test.go @@ -13,9 +13,9 @@ import ( "os" "testing" - "github.com/syncthing/protocol" "github.com/syncthing/syncthing/lib/config" "github.com/syncthing/syncthing/lib/osutil" + "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/rc" ) diff --git a/test/override_test.go b/test/override_test.go index 285b1e150..4076b8fb3 100644 --- a/test/override_test.go +++ b/test/override_test.go @@ -16,9 +16,9 @@ import ( "testing" "time" - "github.com/syncthing/protocol" "github.com/syncthing/syncthing/lib/config" "github.com/syncthing/syncthing/lib/osutil" + "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/rc" ) diff --git a/test/symlink_test.go b/test/symlink_test.go index b23636423..635f8d8e9 100644 --- a/test/symlink_test.go +++ b/test/symlink_test.go @@ -15,8 +15,8 @@ import ( "path/filepath" "testing" - "github.com/syncthing/protocol" "github.com/syncthing/syncthing/lib/config" + "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/rc" "github.com/syncthing/syncthing/lib/symlinks" ) diff --git a/test/sync_test.go b/test/sync_test.go index 00de4a97c..c6e6da590 100644 --- a/test/sync_test.go +++ b/test/sync_test.go @@ -17,8 +17,8 @@ import ( "testing" "time" - "github.com/syncthing/protocol" "github.com/syncthing/syncthing/lib/config" + "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/rc" ) From 24c499d2822ae891c95406066456872e8d6c8164 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Tue, 22 Sep 2015 19:39:07 +0200 Subject: [PATCH 94/94] Clean up deps --- Godeps/Godeps.json | 18 +- .../src/github.com/syncthing/protocol/AUTHORS | 4 - .../syncthing/protocol/CONTRIBUTING.md | 76 -- .../src/github.com/syncthing/protocol/LICENSE | 19 - .../github.com/syncthing/protocol/README.md | 13 - .../syncthing/protocol/common_test.go | 81 -- .../syncthing/protocol/compression.go | 53 - .../syncthing/protocol/compression_test.go | 49 - .../syncthing/protocol/conflict_test.go | 23 - .../github.com/syncthing/protocol/counting.go | 62 - .../github.com/syncthing/protocol/debug.go | 15 - .../github.com/syncthing/protocol/deviceid.go | 163 --- .../syncthing/protocol/deviceid_test.go | 76 -- .../src/github.com/syncthing/protocol/doc.go | 4 - .../github.com/syncthing/protocol/errors.go | 51 - .../src/github.com/syncthing/protocol/fuzz.go | 70 - .../syncthing/protocol/fuzz_test.go | 89 -- .../github.com/syncthing/protocol/header.go | 43 - .../github.com/syncthing/protocol/message.go | 152 --- .../syncthing/protocol/message_xdr.go | 1136 ----------------- .../syncthing/protocol/nativemodel_darwin.go | 40 - .../syncthing/protocol/nativemodel_unix.go | 31 - .../syncthing/protocol/nativemodel_windows.go | 63 - .../github.com/syncthing/protocol/protocol.go | 782 ------------ .../syncthing/protocol/protocol_test.go | 316 ----- .../github.com/syncthing/protocol/vector.go | 115 -- .../syncthing/protocol/vector_compare.go | 89 -- .../syncthing/protocol/vector_compare_test.go | 249 ---- .../syncthing/protocol/vector_test.go | 134 -- .../syncthing/protocol/vector_xdr.go | 43 - .../syncthing/protocol/wireformat.go | 60 - .../syncthing/relaysrv/client/client.go | 298 ----- .../syncthing/relaysrv/client/debug.go | 15 - .../syncthing/relaysrv/client/methods.go | 141 -- .../syncthing/relaysrv/protocol/packets.go | 66 - .../relaysrv/protocol/packets_xdr.go | 567 -------- .../syncthing/relaysrv/protocol/protocol.go | 114 -- 37 files changed, 2 insertions(+), 5318 deletions(-) delete mode 100644 Godeps/_workspace/src/github.com/syncthing/protocol/AUTHORS delete mode 100644 Godeps/_workspace/src/github.com/syncthing/protocol/CONTRIBUTING.md delete mode 100644 Godeps/_workspace/src/github.com/syncthing/protocol/LICENSE delete mode 100644 Godeps/_workspace/src/github.com/syncthing/protocol/README.md delete mode 100644 Godeps/_workspace/src/github.com/syncthing/protocol/common_test.go delete mode 100644 Godeps/_workspace/src/github.com/syncthing/protocol/compression.go delete mode 100644 Godeps/_workspace/src/github.com/syncthing/protocol/compression_test.go delete mode 100644 Godeps/_workspace/src/github.com/syncthing/protocol/conflict_test.go delete mode 100644 Godeps/_workspace/src/github.com/syncthing/protocol/counting.go delete mode 100644 Godeps/_workspace/src/github.com/syncthing/protocol/debug.go delete mode 100644 Godeps/_workspace/src/github.com/syncthing/protocol/deviceid.go delete mode 100644 Godeps/_workspace/src/github.com/syncthing/protocol/deviceid_test.go delete mode 100644 Godeps/_workspace/src/github.com/syncthing/protocol/doc.go delete mode 100644 Godeps/_workspace/src/github.com/syncthing/protocol/errors.go delete mode 100644 Godeps/_workspace/src/github.com/syncthing/protocol/fuzz.go delete mode 100644 Godeps/_workspace/src/github.com/syncthing/protocol/fuzz_test.go delete mode 100644 Godeps/_workspace/src/github.com/syncthing/protocol/header.go delete mode 100644 Godeps/_workspace/src/github.com/syncthing/protocol/message.go delete mode 100644 Godeps/_workspace/src/github.com/syncthing/protocol/message_xdr.go delete mode 100644 Godeps/_workspace/src/github.com/syncthing/protocol/nativemodel_darwin.go delete mode 100644 Godeps/_workspace/src/github.com/syncthing/protocol/nativemodel_unix.go delete mode 100644 Godeps/_workspace/src/github.com/syncthing/protocol/nativemodel_windows.go delete mode 100644 Godeps/_workspace/src/github.com/syncthing/protocol/protocol.go delete mode 100644 Godeps/_workspace/src/github.com/syncthing/protocol/protocol_test.go delete mode 100644 Godeps/_workspace/src/github.com/syncthing/protocol/vector.go delete mode 100644 Godeps/_workspace/src/github.com/syncthing/protocol/vector_compare.go delete mode 100644 Godeps/_workspace/src/github.com/syncthing/protocol/vector_compare_test.go delete mode 100644 Godeps/_workspace/src/github.com/syncthing/protocol/vector_test.go delete mode 100644 Godeps/_workspace/src/github.com/syncthing/protocol/vector_xdr.go delete mode 100644 Godeps/_workspace/src/github.com/syncthing/protocol/wireformat.go delete mode 100644 Godeps/_workspace/src/github.com/syncthing/relaysrv/client/client.go delete mode 100644 Godeps/_workspace/src/github.com/syncthing/relaysrv/client/debug.go delete mode 100644 Godeps/_workspace/src/github.com/syncthing/relaysrv/client/methods.go delete mode 100644 Godeps/_workspace/src/github.com/syncthing/relaysrv/protocol/packets.go delete mode 100644 Godeps/_workspace/src/github.com/syncthing/relaysrv/protocol/packets_xdr.go delete mode 100644 Godeps/_workspace/src/github.com/syncthing/relaysrv/protocol/protocol.go diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 490cac137..1280d3217 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -37,20 +37,6 @@ "ImportPath": "github.com/kardianos/osext", "Rev": "6e7f843663477789fac7c02def0d0909e969b4e5" }, - { - "ImportPath": "github.com/syncthing/protocol", - "Rev": "f91191218b192ace841c878f161832d19c09145a" - }, - { - "ImportPath": "github.com/syncthing/relaysrv/client", - "Comment": "v0.12.0", - "Rev": "6e126fb97e2ff566d35f8d8824e86793d22b2147" - }, - { - "ImportPath": "github.com/syncthing/relaysrv/protocol", - "Comment": "v0.12.0", - "Rev": "6e126fb97e2ff566d35f8d8824e86793d22b2147" - }, { "ImportPath": "github.com/syndtr/goleveldb/leveldb", "Rev": "1a9d62f03ea92815b46fcaab357cfd4df264b1a0" @@ -82,11 +68,11 @@ }, { "ImportPath": "golang.org/x/net/internal/iana", - "Rev": "66f0418ca41253f8d1a024eb9754e9441a8e79b9" + "Rev": "db8e4de5b2d6653f66aea53094624468caad15d2" }, { "ImportPath": "golang.org/x/net/ipv6", - "Rev": "66f0418ca41253f8d1a024eb9754e9441a8e79b9" + "Rev": "db8e4de5b2d6653f66aea53094624468caad15d2" }, { "ImportPath": "golang.org/x/text/transform", diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/AUTHORS b/Godeps/_workspace/src/github.com/syncthing/protocol/AUTHORS deleted file mode 100644 index d84404ee2..000000000 --- a/Godeps/_workspace/src/github.com/syncthing/protocol/AUTHORS +++ /dev/null @@ -1,4 +0,0 @@ -# This is the official list of Protocol Authors for copyright purposes. - -Audrius Butkevicius -Jakob Borg diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/CONTRIBUTING.md b/Godeps/_workspace/src/github.com/syncthing/protocol/CONTRIBUTING.md deleted file mode 100644 index 67e6a9c70..000000000 --- a/Godeps/_workspace/src/github.com/syncthing/protocol/CONTRIBUTING.md +++ /dev/null @@ -1,76 +0,0 @@ -## Reporting Bugs - -Please file bugs in the [Github Issue -Tracker](https://github.com/syncthing/protocol/issues). - -## Contributing Code - -Every contribution is welcome. Following the points below will make this -a smoother process. - -Individuals making significant and valuable contributions are given -commit-access to the project. If you make a significant contribution and -are not considered for commit-access, please contact any of the -Syncthing core team members. - -All nontrivial contributions should go through the pull request -mechanism for internal review. Determining what is "nontrivial" is left -at the discretion of the contributor. - -### Authorship - -All code authors are listed in the AUTHORS file. Commits must be made -with the same name and email as listed in the AUTHORS file. To -accomplish this, ensure that your git configuration is set correctly -prior to making your first commit; - - $ git config --global user.name "Jane Doe" - $ git config --global user.email janedoe@example.com - -You must be reachable on the given email address. If you do not wish to -use your real name for whatever reason, using a nickname or pseudonym is -perfectly acceptable. - -## Coding Style - -- Follow the conventions laid out in [Effective Go](https://golang.org/doc/effective_go.html) - as much as makes sense. - -- All text files use Unix line endings. - -- Each commit should be `go fmt` clean. - -- The commit message subject should be a single short sentence - describing the change, starting with a capital letter. - -- Commits that resolve an existing issue must include the issue number - as `(fixes #123)` at the end of the commit message subject. - -- Imports are grouped per `goimports` standard; that is, standard - library first, then third party libraries after a blank line. - -- A contribution solving a single issue or introducing a single new - feature should probably be a single commit based on the current - `master` branch. You may be asked to "rebase" or "squash" your pull - request to make sure this is the case, especially if there have been - amendments during review. - -## Licensing - -All contributions are made under the same MIT license as the rest of the -project, except documentation, user interface text and translation -strings which are licensed under the Creative Commons Attribution 4.0 -International License. You retain the copyright to code you have -written. - -When accepting your first contribution, the maintainer of the project -will ensure that you are added to the AUTHORS file. You are welcome to -add yourself as a separate commit in your first pull request. - -## Tests - -Yes please! - -## License - -MIT diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/LICENSE b/Godeps/_workspace/src/github.com/syncthing/protocol/LICENSE deleted file mode 100644 index 6f6960a75..000000000 --- a/Godeps/_workspace/src/github.com/syncthing/protocol/LICENSE +++ /dev/null @@ -1,19 +0,0 @@ -Copyright (C) 2014-2015 The Protocol Authors - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - -- The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/README.md b/Godeps/_workspace/src/github.com/syncthing/protocol/README.md deleted file mode 100644 index bcba44b42..000000000 --- a/Godeps/_workspace/src/github.com/syncthing/protocol/README.md +++ /dev/null @@ -1,13 +0,0 @@ -The BEPv1 Protocol -================== - -[![Latest Build](http://img.shields.io/jenkins/s/http/build.syncthing.net/protocol.svg?style=flat-square)](http://build.syncthing.net/job/protocol/lastBuild/) -[![API Documentation](http://img.shields.io/badge/api-Godoc-blue.svg?style=flat-square)](http://godoc.org/github.com/syncthing/protocol) -[![MIT License](http://img.shields.io/badge/license-MIT-blue.svg?style=flat-square)](http://opensource.org/licenses/MIT) - -This is the protocol implementation used by Syncthing. - -License -======= - -MIT diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/common_test.go b/Godeps/_workspace/src/github.com/syncthing/protocol/common_test.go deleted file mode 100644 index 8f1028078..000000000 --- a/Godeps/_workspace/src/github.com/syncthing/protocol/common_test.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright (C) 2014 The Protocol Authors. - -package protocol - -import ( - "io" - "time" -) - -type TestModel struct { - data []byte - folder string - name string - offset int64 - size int - hash []byte - flags uint32 - options []Option - closedCh chan bool -} - -func newTestModel() *TestModel { - return &TestModel{ - closedCh: make(chan bool), - } -} - -func (t *TestModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { -} - -func (t *TestModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { -} - -func (t *TestModel) Request(deviceID DeviceID, folder, name string, offset int64, hash []byte, flags uint32, options []Option, buf []byte) error { - t.folder = folder - t.name = name - t.offset = offset - t.size = len(buf) - t.hash = hash - t.flags = flags - t.options = options - copy(buf, t.data) - return nil -} - -func (t *TestModel) Close(deviceID DeviceID, err error) { - close(t.closedCh) -} - -func (t *TestModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) { -} - -func (t *TestModel) isClosed() bool { - select { - case <-t.closedCh: - return true - case <-time.After(1 * time.Second): - return false // Timeout - } -} - -type ErrPipe struct { - io.PipeWriter - written int - max int - err error - closed bool -} - -func (e *ErrPipe) Write(data []byte) (int, error) { - if e.closed { - return 0, e.err - } - if e.written+len(data) > e.max { - n, _ := e.PipeWriter.Write(data[:e.max-e.written]) - e.PipeWriter.CloseWithError(e.err) - e.closed = true - return n, e.err - } - return e.PipeWriter.Write(data) -} diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/compression.go b/Godeps/_workspace/src/github.com/syncthing/protocol/compression.go deleted file mode 100644 index 9e17213b6..000000000 --- a/Godeps/_workspace/src/github.com/syncthing/protocol/compression.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright (C) 2015 The Protocol Authors. - -package protocol - -import "fmt" - -type Compression int - -const ( - CompressMetadata Compression = iota // zero value is the default, default should be "metadata" - CompressNever - CompressAlways - - compressionThreshold = 128 // don't bother compressing messages smaller than this many bytes -) - -var compressionMarshal = map[Compression]string{ - CompressNever: "never", - CompressMetadata: "metadata", - CompressAlways: "always", -} - -var compressionUnmarshal = map[string]Compression{ - // Legacy - "false": CompressNever, - "true": CompressMetadata, - - // Current - "never": CompressNever, - "metadata": CompressMetadata, - "always": CompressAlways, -} - -func (c Compression) String() string { - s, ok := compressionMarshal[c] - if !ok { - return fmt.Sprintf("unknown:%d", c) - } - return s -} - -func (c Compression) GoString() string { - return fmt.Sprintf("%q", c.String()) -} - -func (c Compression) MarshalText() ([]byte, error) { - return []byte(compressionMarshal[c]), nil -} - -func (c *Compression) UnmarshalText(bs []byte) error { - *c = compressionUnmarshal[string(bs)] - return nil -} diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/compression_test.go b/Godeps/_workspace/src/github.com/syncthing/protocol/compression_test.go deleted file mode 100644 index 90312344c..000000000 --- a/Godeps/_workspace/src/github.com/syncthing/protocol/compression_test.go +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright (C) 2015 The Protocol Authors. - -package protocol - -import "testing" - -func TestCompressionMarshal(t *testing.T) { - uTestcases := []struct { - s string - c Compression - }{ - {"true", CompressMetadata}, - {"false", CompressNever}, - {"never", CompressNever}, - {"metadata", CompressMetadata}, - {"always", CompressAlways}, - {"whatever", CompressMetadata}, - } - - mTestcases := []struct { - s string - c Compression - }{ - {"never", CompressNever}, - {"metadata", CompressMetadata}, - {"always", CompressAlways}, - } - - var c Compression - for _, tc := range uTestcases { - err := c.UnmarshalText([]byte(tc.s)) - if err != nil { - t.Error(err) - } - if c != tc.c { - t.Errorf("%s unmarshalled to %d, not %d", tc.s, c, tc.c) - } - } - - for _, tc := range mTestcases { - bs, err := tc.c.MarshalText() - if err != nil { - t.Error(err) - } - if s := string(bs); s != tc.s { - t.Errorf("%d marshalled to %q, not %q", tc.c, s, tc.s) - } - } -} diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/conflict_test.go b/Godeps/_workspace/src/github.com/syncthing/protocol/conflict_test.go deleted file mode 100644 index ef5c44d7e..000000000 --- a/Godeps/_workspace/src/github.com/syncthing/protocol/conflict_test.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (C) 2015 The Protocol Authors. - -package protocol - -import "testing" - -func TestWinsConflict(t *testing.T) { - testcases := [][2]FileInfo{ - // The first should always win over the second - {{Modified: 42}, {Modified: 41}}, - {{Modified: 41}, {Modified: 42, Flags: FlagDeleted}}, - {{Modified: 41, Version: Vector{{42, 2}, {43, 1}}}, {Modified: 41, Version: Vector{{42, 1}, {43, 2}}}}, - } - - for _, tc := range testcases { - if !tc[0].WinsConflict(tc[1]) { - t.Errorf("%v should win over %v", tc[0], tc[1]) - } - if tc[1].WinsConflict(tc[0]) { - t.Errorf("%v should not win over %v", tc[1], tc[0]) - } - } -} diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/counting.go b/Godeps/_workspace/src/github.com/syncthing/protocol/counting.go deleted file mode 100644 index d441ed311..000000000 --- a/Godeps/_workspace/src/github.com/syncthing/protocol/counting.go +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright (C) 2014 The Protocol Authors. - -package protocol - -import ( - "io" - "sync/atomic" - "time" -) - -type countingReader struct { - io.Reader - tot int64 // bytes - last int64 // unix nanos -} - -var ( - totalIncoming int64 - totalOutgoing int64 -) - -func (c *countingReader) Read(bs []byte) (int, error) { - n, err := c.Reader.Read(bs) - atomic.AddInt64(&c.tot, int64(n)) - atomic.AddInt64(&totalIncoming, int64(n)) - atomic.StoreInt64(&c.last, time.Now().UnixNano()) - return n, err -} - -func (c *countingReader) Tot() int64 { - return atomic.LoadInt64(&c.tot) -} - -func (c *countingReader) Last() time.Time { - return time.Unix(0, atomic.LoadInt64(&c.last)) -} - -type countingWriter struct { - io.Writer - tot int64 // bytes - last int64 // unix nanos -} - -func (c *countingWriter) Write(bs []byte) (int, error) { - n, err := c.Writer.Write(bs) - atomic.AddInt64(&c.tot, int64(n)) - atomic.AddInt64(&totalOutgoing, int64(n)) - atomic.StoreInt64(&c.last, time.Now().UnixNano()) - return n, err -} - -func (c *countingWriter) Tot() int64 { - return atomic.LoadInt64(&c.tot) -} - -func (c *countingWriter) Last() time.Time { - return time.Unix(0, atomic.LoadInt64(&c.last)) -} - -func TotalInOut() (int64, int64) { - return atomic.LoadInt64(&totalIncoming), atomic.LoadInt64(&totalOutgoing) -} diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/debug.go b/Godeps/_workspace/src/github.com/syncthing/protocol/debug.go deleted file mode 100644 index 435d7f5d2..000000000 --- a/Godeps/_workspace/src/github.com/syncthing/protocol/debug.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright (C) 2014 The Protocol Authors. - -package protocol - -import ( - "os" - "strings" - - "github.com/calmh/logger" -) - -var ( - debug = strings.Contains(os.Getenv("STTRACE"), "protocol") || os.Getenv("STTRACE") == "all" - l = logger.DefaultLogger -) diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/deviceid.go b/Godeps/_workspace/src/github.com/syncthing/protocol/deviceid.go deleted file mode 100644 index 2e0334a6a..000000000 --- a/Godeps/_workspace/src/github.com/syncthing/protocol/deviceid.go +++ /dev/null @@ -1,163 +0,0 @@ -// Copyright (C) 2014 The Protocol Authors. - -package protocol - -import ( - "bytes" - "crypto/sha256" - "encoding/base32" - "encoding/binary" - "errors" - "fmt" - "regexp" - "strings" - - "github.com/calmh/luhn" -) - -type DeviceID [32]byte - -var LocalDeviceID = DeviceID{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff} - -// NewDeviceID generates a new device ID from the raw bytes of a certificate -func NewDeviceID(rawCert []byte) DeviceID { - var n DeviceID - hf := sha256.New() - hf.Write(rawCert) - hf.Sum(n[:0]) - return n -} - -func DeviceIDFromString(s string) (DeviceID, error) { - var n DeviceID - err := n.UnmarshalText([]byte(s)) - return n, err -} - -func DeviceIDFromBytes(bs []byte) DeviceID { - var n DeviceID - if len(bs) != len(n) { - panic("incorrect length of byte slice representing device ID") - } - copy(n[:], bs) - return n -} - -// String returns the canonical string representation of the device ID -func (n DeviceID) String() string { - id := base32.StdEncoding.EncodeToString(n[:]) - id = strings.Trim(id, "=") - id, err := luhnify(id) - if err != nil { - // Should never happen - panic(err) - } - id = chunkify(id) - return id -} - -func (n DeviceID) GoString() string { - return n.String() -} - -func (n DeviceID) Compare(other DeviceID) int { - return bytes.Compare(n[:], other[:]) -} - -func (n DeviceID) Equals(other DeviceID) bool { - return bytes.Compare(n[:], other[:]) == 0 -} - -// Short returns an integer representing bits 0-63 of the device ID. -func (n DeviceID) Short() uint64 { - return binary.BigEndian.Uint64(n[:]) -} - -func (n *DeviceID) MarshalText() ([]byte, error) { - return []byte(n.String()), nil -} - -func (n *DeviceID) UnmarshalText(bs []byte) error { - id := string(bs) - id = strings.Trim(id, "=") - id = strings.ToUpper(id) - id = untypeoify(id) - id = unchunkify(id) - - var err error - switch len(id) { - case 56: - // New style, with check digits - id, err = unluhnify(id) - if err != nil { - return err - } - fallthrough - case 52: - // Old style, no check digits - dec, err := base32.StdEncoding.DecodeString(id + "====") - if err != nil { - return err - } - copy(n[:], dec) - return nil - default: - return errors.New("device ID invalid: incorrect length") - } -} - -func luhnify(s string) (string, error) { - if len(s) != 52 { - panic("unsupported string length") - } - - res := make([]string, 0, 4) - for i := 0; i < 4; i++ { - p := s[i*13 : (i+1)*13] - l, err := luhn.Base32.Generate(p) - if err != nil { - return "", err - } - res = append(res, fmt.Sprintf("%s%c", p, l)) - } - return res[0] + res[1] + res[2] + res[3], nil -} - -func unluhnify(s string) (string, error) { - if len(s) != 56 { - return "", fmt.Errorf("unsupported string length %d", len(s)) - } - - res := make([]string, 0, 4) - for i := 0; i < 4; i++ { - p := s[i*14 : (i+1)*14-1] - l, err := luhn.Base32.Generate(p) - if err != nil { - return "", err - } - if g := fmt.Sprintf("%s%c", p, l); g != s[i*14:(i+1)*14] { - return "", errors.New("check digit incorrect") - } - res = append(res, p) - } - return res[0] + res[1] + res[2] + res[3], nil -} - -func chunkify(s string) string { - s = regexp.MustCompile("(.{7})").ReplaceAllString(s, "$1-") - s = strings.Trim(s, "-") - return s -} - -func unchunkify(s string) string { - s = strings.Replace(s, "-", "", -1) - s = strings.Replace(s, " ", "", -1) - return s -} - -func untypeoify(s string) string { - s = strings.Replace(s, "0", "O", -1) - s = strings.Replace(s, "1", "I", -1) - s = strings.Replace(s, "8", "B", -1) - return s -} diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/deviceid_test.go b/Godeps/_workspace/src/github.com/syncthing/protocol/deviceid_test.go deleted file mode 100644 index 613557d32..000000000 --- a/Godeps/_workspace/src/github.com/syncthing/protocol/deviceid_test.go +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright (C) 2014 The Protocol Authors. - -package protocol - -import "testing" - -var formatted = "P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" -var formatCases = []string{ - "P56IOI-7MZJNU-2IQGDR-EYDM2M-GTMGL3-BXNPQ6-W5BTBB-Z4TJXZ-WICQ", - "P56IOI-7MZJNU2Y-IQGDR-EYDM2M-GTI-MGL3-BXNPQ6-W5BM-TBB-Z4TJXZ-WICQ2", - "P56IOI7 MZJNU2I QGDREYD M2MGTMGL 3BXNPQ6W 5BTB BZ4T JXZWICQ", - "P56IOI7 MZJNU2Y IQGDREY DM2MGTI MGL3BXN PQ6W5BM TBBZ4TJ XZWICQ2", - "P56IOI7MZJNU2IQGDREYDM2MGTMGL3BXNPQ6W5BTBBZ4TJXZWICQ", - "p56ioi7mzjnu2iqgdreydm2mgtmgl3bxnpq6w5btbbz4tjxzwicq", - "P56IOI7MZJNU2YIQGDREYDM2MGTIMGL3BXNPQ6W5BMTBBZ4TJXZWICQ2", - "P561017MZJNU2YIQGDREYDM2MGTIMGL3BXNPQ6W5BMT88Z4TJXZWICQ2", - "p56ioi7mzjnu2yiqgdreydm2mgtimgl3bxnpq6w5bmtbbz4tjxzwicq2", - "p561017mzjnu2yiqgdreydm2mgtimgl3bxnpq6w5bmt88z4tjxzwicq2", -} - -func TestFormatDeviceID(t *testing.T) { - for i, tc := range formatCases { - var id DeviceID - err := id.UnmarshalText([]byte(tc)) - if err != nil { - t.Errorf("#%d UnmarshalText(%q); %v", i, tc, err) - } else if f := id.String(); f != formatted { - t.Errorf("#%d FormatDeviceID(%q)\n\t%q !=\n\t%q", i, tc, f, formatted) - } - } -} - -var validateCases = []struct { - s string - ok bool -}{ - {"", false}, - {"P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2", true}, - {"P56IOI7-MZJNU2-IQGDREY-DM2MGT-MGL3BXN-PQ6W5B-TBBZ4TJ-XZWICQ", true}, - {"P56IOI7 MZJNU2I QGDREYD M2MGTMGL 3BXNPQ6W 5BTB BZ4T JXZWICQ", true}, - {"P56IOI7MZJNU2IQGDREYDM2MGTMGL3BXNPQ6W5BTBBZ4TJXZWICQ", true}, - {"P56IOI7MZJNU2IQGDREYDM2MGTMGL3BXNPQ6W5BTBBZ4TJXZWICQCCCC", false}, - {"p56ioi7mzjnu2iqgdreydm2mgtmgl3bxnpq6w5btbbz4tjxzwicq", true}, - {"p56ioi7mzjnu2iqgdreydm2mgtmgl3bxnpq6w5btbbz4tjxzwicqCCCC", false}, -} - -func TestValidateDeviceID(t *testing.T) { - for _, tc := range validateCases { - var id DeviceID - err := id.UnmarshalText([]byte(tc.s)) - if (err == nil && !tc.ok) || (err != nil && tc.ok) { - t.Errorf("ValidateDeviceID(%q); %v != %v", tc.s, err, tc.ok) - } - } -} - -func TestMarshallingDeviceID(t *testing.T) { - n0 := DeviceID{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 10, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32} - n1 := DeviceID{} - n2 := DeviceID{} - - bs, _ := n0.MarshalText() - n1.UnmarshalText(bs) - bs, _ = n1.MarshalText() - n2.UnmarshalText(bs) - - if n2.String() != n0.String() { - t.Errorf("String marshalling error; %q != %q", n2.String(), n0.String()) - } - if !n2.Equals(n0) { - t.Error("Equals error") - } - if n2.Compare(n0) != 0 { - t.Error("Compare error") - } -} diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/doc.go b/Godeps/_workspace/src/github.com/syncthing/protocol/doc.go deleted file mode 100644 index 2c6ea8ef2..000000000 --- a/Godeps/_workspace/src/github.com/syncthing/protocol/doc.go +++ /dev/null @@ -1,4 +0,0 @@ -// Copyright (C) 2014 The Protocol Authors. - -// Package protocol implements the Block Exchange Protocol. -package protocol diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/errors.go b/Godeps/_workspace/src/github.com/syncthing/protocol/errors.go deleted file mode 100644 index 31d27af0d..000000000 --- a/Godeps/_workspace/src/github.com/syncthing/protocol/errors.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright (C) 2014 The Protocol Authors. - -package protocol - -import ( - "errors" -) - -const ( - ecNoError int32 = iota - ecGeneric - ecNoSuchFile - ecInvalid -) - -var ( - ErrNoError error = nil - ErrGeneric = errors.New("generic error") - ErrNoSuchFile = errors.New("no such file") - ErrInvalid = errors.New("file is invalid") -) - -var lookupError = map[int32]error{ - ecNoError: ErrNoError, - ecGeneric: ErrGeneric, - ecNoSuchFile: ErrNoSuchFile, - ecInvalid: ErrInvalid, -} - -var lookupCode = map[error]int32{ - ErrNoError: ecNoError, - ErrGeneric: ecGeneric, - ErrNoSuchFile: ecNoSuchFile, - ErrInvalid: ecInvalid, -} - -func codeToError(errcode int32) error { - err, ok := lookupError[errcode] - if !ok { - return ErrGeneric - } - return err -} - -func errorToCode(err error) int32 { - code, ok := lookupCode[err] - if !ok { - return ecGeneric - } - return code -} diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/fuzz.go b/Godeps/_workspace/src/github.com/syncthing/protocol/fuzz.go deleted file mode 100644 index 9b82abe7c..000000000 --- a/Godeps/_workspace/src/github.com/syncthing/protocol/fuzz.go +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright (C) 2015 The Protocol Authors. - -// +build gofuzz - -package protocol - -import ( - "bytes" - "encoding/binary" - "encoding/hex" - "fmt" - "reflect" - "sync" -) - -func Fuzz(data []byte) int { - // Regenerate the length, or we'll most commonly exit quickly due to an - // unexpected eof which is unintestering. - if len(data) > 8 { - binary.BigEndian.PutUint32(data[4:], uint32(len(data))-8) - } - - // Setup a rawConnection we'll use to parse the message. - c := rawConnection{ - cr: &countingReader{Reader: bytes.NewReader(data)}, - closed: make(chan struct{}), - pool: sync.Pool{ - New: func() interface{} { - return make([]byte, BlockSize) - }, - }, - } - - // Attempt to parse the message. - hdr, msg, err := c.readMessage() - if err != nil { - return 0 - } - - // If parsing worked, attempt to encode it again. - newBs, err := msg.AppendXDR(nil) - if err != nil { - panic("not encodable") - } - - // Create an appriate header for the re-encoding. - newMsg := make([]byte, 8) - binary.BigEndian.PutUint32(newMsg, encodeHeader(hdr)) - binary.BigEndian.PutUint32(newMsg[4:], uint32(len(newBs))) - newMsg = append(newMsg, newBs...) - - // Use the rawConnection to parse the re-encoding. - c.cr = &countingReader{Reader: bytes.NewReader(newMsg)} - hdr2, msg2, err := c.readMessage() - if err != nil { - fmt.Println("Initial:\n" + hex.Dump(data)) - fmt.Println("New:\n" + hex.Dump(newMsg)) - panic("not parseable after re-encode: " + err.Error()) - } - - // Make sure the data is the same as it was before. - if hdr != hdr2 { - panic("headers differ") - } - if !reflect.DeepEqual(msg, msg2) { - panic("contents differ") - } - - return 1 -} diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/fuzz_test.go b/Godeps/_workspace/src/github.com/syncthing/protocol/fuzz_test.go deleted file mode 100644 index 65c2d9010..000000000 --- a/Godeps/_workspace/src/github.com/syncthing/protocol/fuzz_test.go +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright (C) 2015 The Protocol Authors. - -// +build gofuzz - -package protocol - -import ( - "encoding/binary" - "fmt" - "io/ioutil" - "os" - "strings" - "testing" - "testing/quick" -) - -// This can be used to generate a corpus of valid messages as a starting point -// for the fuzzer. -func TestGenerateCorpus(t *testing.T) { - t.Skip("Use to generate initial corpus only") - - n := 0 - check := func(idx IndexMessage) bool { - for i := range idx.Options { - if len(idx.Options[i].Key) > 64 { - idx.Options[i].Key = idx.Options[i].Key[:64] - } - } - hdr := header{ - version: 0, - msgID: 42, - msgType: messageTypeIndex, - compression: false, - } - - msgBs := idx.MustMarshalXDR() - - buf := make([]byte, 8) - binary.BigEndian.PutUint32(buf, encodeHeader(hdr)) - binary.BigEndian.PutUint32(buf[4:], uint32(len(msgBs))) - buf = append(buf, msgBs...) - - ioutil.WriteFile(fmt.Sprintf("testdata/corpus/test-%03d.xdr", n), buf, 0644) - n++ - return true - } - - if err := quick.Check(check, &quick.Config{MaxCount: 1000}); err != nil { - t.Fatal(err) - } -} - -// Tests any crashers found by the fuzzer, for closer investigation. -func TestCrashers(t *testing.T) { - testFiles(t, "testdata/crashers") -} - -// Tests the entire corpus, which should PASS before the fuzzer starts -// fuzzing. -func TestCorpus(t *testing.T) { - testFiles(t, "testdata/corpus") -} - -func testFiles(t *testing.T, dir string) { - fd, err := os.Open(dir) - if err != nil { - t.Fatal(err) - } - crashers, err := fd.Readdirnames(-1) - if err != nil { - t.Fatal(err) - } - for _, name := range crashers { - if strings.HasSuffix(name, ".output") { - continue - } - if strings.HasSuffix(name, ".quoted") { - continue - } - - t.Log(name) - crasher, err := ioutil.ReadFile(dir + "/" + name) - if err != nil { - t.Fatal(err) - } - - Fuzz(crasher) - } -} diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/header.go b/Godeps/_workspace/src/github.com/syncthing/protocol/header.go deleted file mode 100644 index 846ee48cd..000000000 --- a/Godeps/_workspace/src/github.com/syncthing/protocol/header.go +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright (C) 2014 The Protocol Authors. - -package protocol - -import "github.com/calmh/xdr" - -type header struct { - version int - msgID int - msgType int - compression bool -} - -func (h header) encodeXDR(xw *xdr.Writer) (int, error) { - u := encodeHeader(h) - return xw.WriteUint32(u) -} - -func (h *header) decodeXDR(xr *xdr.Reader) error { - u := xr.ReadUint32() - *h = decodeHeader(u) - return xr.Error() -} - -func encodeHeader(h header) uint32 { - var isComp uint32 - if h.compression { - isComp = 1 << 0 // the zeroth bit is the compression bit - } - return uint32(h.version&0xf)<<28 + - uint32(h.msgID&0xfff)<<16 + - uint32(h.msgType&0xff)<<8 + - isComp -} - -func decodeHeader(u uint32) header { - return header{ - version: int(u>>28) & 0xf, - msgID: int(u>>16) & 0xfff, - msgType: int(u>>8) & 0xff, - compression: u&1 == 1, - } -} diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/message.go b/Godeps/_workspace/src/github.com/syncthing/protocol/message.go deleted file mode 100644 index 2a37136b5..000000000 --- a/Godeps/_workspace/src/github.com/syncthing/protocol/message.go +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright (C) 2014 The Protocol Authors. - -//go:generate -command genxdr go run ../syncthing/Godeps/_workspace/src/github.com/calmh/xdr/cmd/genxdr/main.go -//go:generate genxdr -o message_xdr.go message.go - -package protocol - -import "fmt" - -type IndexMessage struct { - Folder string - Files []FileInfo // max:1000000 - Flags uint32 - Options []Option // max:64 -} - -type FileInfo struct { - Name string // max:8192 - Flags uint32 - Modified int64 - Version Vector - LocalVersion int64 - CachedSize int64 // noencode (cache only) - Blocks []BlockInfo // max:1000000 -} - -func (f FileInfo) String() string { - return fmt.Sprintf("File{Name:%q, Flags:0%o, Modified:%d, Version:%v, Size:%d, Blocks:%v}", - f.Name, f.Flags, f.Modified, f.Version, f.Size(), f.Blocks) -} - -func (f FileInfo) Size() (bytes int64) { - if f.IsDeleted() || f.IsDirectory() { - return 128 - } - for _, b := range f.Blocks { - bytes += int64(b.Size) - } - return -} - -func (f FileInfo) IsDeleted() bool { - return f.Flags&FlagDeleted != 0 -} - -func (f FileInfo) IsInvalid() bool { - return f.Flags&FlagInvalid != 0 -} - -func (f FileInfo) IsDirectory() bool { - return f.Flags&FlagDirectory != 0 -} - -func (f FileInfo) IsSymlink() bool { - return f.Flags&FlagSymlink != 0 -} - -func (f FileInfo) HasPermissionBits() bool { - return f.Flags&FlagNoPermBits == 0 -} - -// WinsConflict returns true if "f" is the one to choose when it is in -// conflict with "other". -func (f FileInfo) WinsConflict(other FileInfo) bool { - // If a modification is in conflict with a delete, we pick the - // modification. - if !f.IsDeleted() && other.IsDeleted() { - return true - } - if f.IsDeleted() && !other.IsDeleted() { - return false - } - - // The one with the newer modification time wins. - if f.Modified > other.Modified { - return true - } - if f.Modified < other.Modified { - return false - } - - // The modification times were equal. Use the device ID in the version - // vector as tie breaker. - return f.Version.Compare(other.Version) == ConcurrentGreater -} - -type BlockInfo struct { - Offset int64 // noencode (cache only) - Size int32 - Hash []byte // max:64 -} - -func (b BlockInfo) String() string { - return fmt.Sprintf("Block{%d/%d/%x}", b.Offset, b.Size, b.Hash) -} - -type RequestMessage struct { - Folder string // max:64 - Name string // max:8192 - Offset int64 - Size int32 - Hash []byte // max:64 - Flags uint32 - Options []Option // max:64 -} - -type ResponseMessage struct { - Data []byte - Code int32 -} - -type ClusterConfigMessage struct { - ClientName string // max:64 - ClientVersion string // max:64 - Folders []Folder // max:1000000 - Options []Option // max:64 -} - -func (o *ClusterConfigMessage) GetOption(key string) string { - for _, option := range o.Options { - if option.Key == key { - return option.Value - } - } - return "" -} - -type Folder struct { - ID string // max:64 - Devices []Device // max:1000000 - Flags uint32 - Options []Option // max:64 -} - -type Device struct { - ID []byte // max:32 - MaxLocalVersion int64 - Flags uint32 - Options []Option // max:64 -} - -type Option struct { - Key string // max:64 - Value string // max:1024 -} - -type CloseMessage struct { - Reason string // max:1024 - Code int32 -} - -type EmptyMessage struct{} diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/message_xdr.go b/Godeps/_workspace/src/github.com/syncthing/protocol/message_xdr.go deleted file mode 100644 index 876fbb77c..000000000 --- a/Godeps/_workspace/src/github.com/syncthing/protocol/message_xdr.go +++ /dev/null @@ -1,1136 +0,0 @@ -// ************************************************************ -// This file is automatically generated by genxdr. Do not edit. -// ************************************************************ - -package protocol - -import ( - "bytes" - "io" - - "github.com/calmh/xdr" -) - -/* - -IndexMessage Structure: - - 0 1 2 3 - 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Length of Folder | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -/ / -\ Folder (variable length) \ -/ / -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Number of Files | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -/ / -\ Zero or more FileInfo Structures \ -/ / -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Flags | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Number of Options | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -/ / -\ Zero or more Option Structures \ -/ / -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - - -struct IndexMessage { - string Folder<>; - FileInfo Files<1000000>; - unsigned int Flags; - Option Options<64>; -} - -*/ - -func (o IndexMessage) EncodeXDR(w io.Writer) (int, error) { - var xw = xdr.NewWriter(w) - return o.EncodeXDRInto(xw) -} - -func (o IndexMessage) MarshalXDR() ([]byte, error) { - return o.AppendXDR(make([]byte, 0, 128)) -} - -func (o IndexMessage) MustMarshalXDR() []byte { - bs, err := o.MarshalXDR() - if err != nil { - panic(err) - } - return bs -} - -func (o IndexMessage) AppendXDR(bs []byte) ([]byte, error) { - var aw = xdr.AppendWriter(bs) - var xw = xdr.NewWriter(&aw) - _, err := o.EncodeXDRInto(xw) - return []byte(aw), err -} - -func (o IndexMessage) EncodeXDRInto(xw *xdr.Writer) (int, error) { - xw.WriteString(o.Folder) - if l := len(o.Files); l > 1000000 { - return xw.Tot(), xdr.ElementSizeExceeded("Files", l, 1000000) - } - xw.WriteUint32(uint32(len(o.Files))) - for i := range o.Files { - _, err := o.Files[i].EncodeXDRInto(xw) - if err != nil { - return xw.Tot(), err - } - } - xw.WriteUint32(o.Flags) - if l := len(o.Options); l > 64 { - return xw.Tot(), xdr.ElementSizeExceeded("Options", l, 64) - } - xw.WriteUint32(uint32(len(o.Options))) - for i := range o.Options { - _, err := o.Options[i].EncodeXDRInto(xw) - if err != nil { - return xw.Tot(), err - } - } - return xw.Tot(), xw.Error() -} - -func (o *IndexMessage) DecodeXDR(r io.Reader) error { - xr := xdr.NewReader(r) - return o.DecodeXDRFrom(xr) -} - -func (o *IndexMessage) UnmarshalXDR(bs []byte) error { - var br = bytes.NewReader(bs) - var xr = xdr.NewReader(br) - return o.DecodeXDRFrom(xr) -} - -func (o *IndexMessage) DecodeXDRFrom(xr *xdr.Reader) error { - o.Folder = xr.ReadString() - _FilesSize := int(xr.ReadUint32()) - if _FilesSize < 0 { - return xdr.ElementSizeExceeded("Files", _FilesSize, 1000000) - } - if _FilesSize > 1000000 { - return xdr.ElementSizeExceeded("Files", _FilesSize, 1000000) - } - o.Files = make([]FileInfo, _FilesSize) - for i := range o.Files { - (&o.Files[i]).DecodeXDRFrom(xr) - } - o.Flags = xr.ReadUint32() - _OptionsSize := int(xr.ReadUint32()) - if _OptionsSize < 0 { - return xdr.ElementSizeExceeded("Options", _OptionsSize, 64) - } - if _OptionsSize > 64 { - return xdr.ElementSizeExceeded("Options", _OptionsSize, 64) - } - o.Options = make([]Option, _OptionsSize) - for i := range o.Options { - (&o.Options[i]).DecodeXDRFrom(xr) - } - return xr.Error() -} - -/* - -FileInfo Structure: - - 0 1 2 3 - 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Length of Name | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -/ / -\ Name (variable length) \ -/ / -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Flags | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| | -+ Modified (64 bits) + -| | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -/ / -\ Vector Structure \ -/ / -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| | -+ Local Version (64 bits) + -| | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Number of Blocks | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -/ / -\ Zero or more BlockInfo Structures \ -/ / -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - - -struct FileInfo { - string Name<8192>; - unsigned int Flags; - hyper Modified; - Vector Version; - hyper LocalVersion; - BlockInfo Blocks<1000000>; -} - -*/ - -func (o FileInfo) EncodeXDR(w io.Writer) (int, error) { - var xw = xdr.NewWriter(w) - return o.EncodeXDRInto(xw) -} - -func (o FileInfo) MarshalXDR() ([]byte, error) { - return o.AppendXDR(make([]byte, 0, 128)) -} - -func (o FileInfo) MustMarshalXDR() []byte { - bs, err := o.MarshalXDR() - if err != nil { - panic(err) - } - return bs -} - -func (o FileInfo) AppendXDR(bs []byte) ([]byte, error) { - var aw = xdr.AppendWriter(bs) - var xw = xdr.NewWriter(&aw) - _, err := o.EncodeXDRInto(xw) - return []byte(aw), err -} - -func (o FileInfo) EncodeXDRInto(xw *xdr.Writer) (int, error) { - if l := len(o.Name); l > 8192 { - return xw.Tot(), xdr.ElementSizeExceeded("Name", l, 8192) - } - xw.WriteString(o.Name) - xw.WriteUint32(o.Flags) - xw.WriteUint64(uint64(o.Modified)) - _, err := o.Version.EncodeXDRInto(xw) - if err != nil { - return xw.Tot(), err - } - xw.WriteUint64(uint64(o.LocalVersion)) - if l := len(o.Blocks); l > 1000000 { - return xw.Tot(), xdr.ElementSizeExceeded("Blocks", l, 1000000) - } - xw.WriteUint32(uint32(len(o.Blocks))) - for i := range o.Blocks { - _, err := o.Blocks[i].EncodeXDRInto(xw) - if err != nil { - return xw.Tot(), err - } - } - return xw.Tot(), xw.Error() -} - -func (o *FileInfo) DecodeXDR(r io.Reader) error { - xr := xdr.NewReader(r) - return o.DecodeXDRFrom(xr) -} - -func (o *FileInfo) UnmarshalXDR(bs []byte) error { - var br = bytes.NewReader(bs) - var xr = xdr.NewReader(br) - return o.DecodeXDRFrom(xr) -} - -func (o *FileInfo) DecodeXDRFrom(xr *xdr.Reader) error { - o.Name = xr.ReadStringMax(8192) - o.Flags = xr.ReadUint32() - o.Modified = int64(xr.ReadUint64()) - (&o.Version).DecodeXDRFrom(xr) - o.LocalVersion = int64(xr.ReadUint64()) - _BlocksSize := int(xr.ReadUint32()) - if _BlocksSize < 0 { - return xdr.ElementSizeExceeded("Blocks", _BlocksSize, 1000000) - } - if _BlocksSize > 1000000 { - return xdr.ElementSizeExceeded("Blocks", _BlocksSize, 1000000) - } - o.Blocks = make([]BlockInfo, _BlocksSize) - for i := range o.Blocks { - (&o.Blocks[i]).DecodeXDRFrom(xr) - } - return xr.Error() -} - -/* - -BlockInfo Structure: - - 0 1 2 3 - 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Size | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Length of Hash | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -/ / -\ Hash (variable length) \ -/ / -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - - -struct BlockInfo { - int Size; - opaque Hash<64>; -} - -*/ - -func (o BlockInfo) EncodeXDR(w io.Writer) (int, error) { - var xw = xdr.NewWriter(w) - return o.EncodeXDRInto(xw) -} - -func (o BlockInfo) MarshalXDR() ([]byte, error) { - return o.AppendXDR(make([]byte, 0, 128)) -} - -func (o BlockInfo) MustMarshalXDR() []byte { - bs, err := o.MarshalXDR() - if err != nil { - panic(err) - } - return bs -} - -func (o BlockInfo) AppendXDR(bs []byte) ([]byte, error) { - var aw = xdr.AppendWriter(bs) - var xw = xdr.NewWriter(&aw) - _, err := o.EncodeXDRInto(xw) - return []byte(aw), err -} - -func (o BlockInfo) EncodeXDRInto(xw *xdr.Writer) (int, error) { - xw.WriteUint32(uint32(o.Size)) - if l := len(o.Hash); l > 64 { - return xw.Tot(), xdr.ElementSizeExceeded("Hash", l, 64) - } - xw.WriteBytes(o.Hash) - return xw.Tot(), xw.Error() -} - -func (o *BlockInfo) DecodeXDR(r io.Reader) error { - xr := xdr.NewReader(r) - return o.DecodeXDRFrom(xr) -} - -func (o *BlockInfo) UnmarshalXDR(bs []byte) error { - var br = bytes.NewReader(bs) - var xr = xdr.NewReader(br) - return o.DecodeXDRFrom(xr) -} - -func (o *BlockInfo) DecodeXDRFrom(xr *xdr.Reader) error { - o.Size = int32(xr.ReadUint32()) - o.Hash = xr.ReadBytesMax(64) - return xr.Error() -} - -/* - -RequestMessage Structure: - - 0 1 2 3 - 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Length of Folder | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -/ / -\ Folder (variable length) \ -/ / -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Length of Name | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -/ / -\ Name (variable length) \ -/ / -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| | -+ Offset (64 bits) + -| | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Size | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Length of Hash | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -/ / -\ Hash (variable length) \ -/ / -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Flags | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Number of Options | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -/ / -\ Zero or more Option Structures \ -/ / -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - - -struct RequestMessage { - string Folder<64>; - string Name<8192>; - hyper Offset; - int Size; - opaque Hash<64>; - unsigned int Flags; - Option Options<64>; -} - -*/ - -func (o RequestMessage) EncodeXDR(w io.Writer) (int, error) { - var xw = xdr.NewWriter(w) - return o.EncodeXDRInto(xw) -} - -func (o RequestMessage) MarshalXDR() ([]byte, error) { - return o.AppendXDR(make([]byte, 0, 128)) -} - -func (o RequestMessage) MustMarshalXDR() []byte { - bs, err := o.MarshalXDR() - if err != nil { - panic(err) - } - return bs -} - -func (o RequestMessage) AppendXDR(bs []byte) ([]byte, error) { - var aw = xdr.AppendWriter(bs) - var xw = xdr.NewWriter(&aw) - _, err := o.EncodeXDRInto(xw) - return []byte(aw), err -} - -func (o RequestMessage) EncodeXDRInto(xw *xdr.Writer) (int, error) { - if l := len(o.Folder); l > 64 { - return xw.Tot(), xdr.ElementSizeExceeded("Folder", l, 64) - } - xw.WriteString(o.Folder) - if l := len(o.Name); l > 8192 { - return xw.Tot(), xdr.ElementSizeExceeded("Name", l, 8192) - } - xw.WriteString(o.Name) - xw.WriteUint64(uint64(o.Offset)) - xw.WriteUint32(uint32(o.Size)) - if l := len(o.Hash); l > 64 { - return xw.Tot(), xdr.ElementSizeExceeded("Hash", l, 64) - } - xw.WriteBytes(o.Hash) - xw.WriteUint32(o.Flags) - if l := len(o.Options); l > 64 { - return xw.Tot(), xdr.ElementSizeExceeded("Options", l, 64) - } - xw.WriteUint32(uint32(len(o.Options))) - for i := range o.Options { - _, err := o.Options[i].EncodeXDRInto(xw) - if err != nil { - return xw.Tot(), err - } - } - return xw.Tot(), xw.Error() -} - -func (o *RequestMessage) DecodeXDR(r io.Reader) error { - xr := xdr.NewReader(r) - return o.DecodeXDRFrom(xr) -} - -func (o *RequestMessage) UnmarshalXDR(bs []byte) error { - var br = bytes.NewReader(bs) - var xr = xdr.NewReader(br) - return o.DecodeXDRFrom(xr) -} - -func (o *RequestMessage) DecodeXDRFrom(xr *xdr.Reader) error { - o.Folder = xr.ReadStringMax(64) - o.Name = xr.ReadStringMax(8192) - o.Offset = int64(xr.ReadUint64()) - o.Size = int32(xr.ReadUint32()) - o.Hash = xr.ReadBytesMax(64) - o.Flags = xr.ReadUint32() - _OptionsSize := int(xr.ReadUint32()) - if _OptionsSize < 0 { - return xdr.ElementSizeExceeded("Options", _OptionsSize, 64) - } - if _OptionsSize > 64 { - return xdr.ElementSizeExceeded("Options", _OptionsSize, 64) - } - o.Options = make([]Option, _OptionsSize) - for i := range o.Options { - (&o.Options[i]).DecodeXDRFrom(xr) - } - return xr.Error() -} - -/* - -ResponseMessage Structure: - - 0 1 2 3 - 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Length of Data | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -/ / -\ Data (variable length) \ -/ / -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Code | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - - -struct ResponseMessage { - opaque Data<>; - int Code; -} - -*/ - -func (o ResponseMessage) EncodeXDR(w io.Writer) (int, error) { - var xw = xdr.NewWriter(w) - return o.EncodeXDRInto(xw) -} - -func (o ResponseMessage) MarshalXDR() ([]byte, error) { - return o.AppendXDR(make([]byte, 0, 128)) -} - -func (o ResponseMessage) MustMarshalXDR() []byte { - bs, err := o.MarshalXDR() - if err != nil { - panic(err) - } - return bs -} - -func (o ResponseMessage) AppendXDR(bs []byte) ([]byte, error) { - var aw = xdr.AppendWriter(bs) - var xw = xdr.NewWriter(&aw) - _, err := o.EncodeXDRInto(xw) - return []byte(aw), err -} - -func (o ResponseMessage) EncodeXDRInto(xw *xdr.Writer) (int, error) { - xw.WriteBytes(o.Data) - xw.WriteUint32(uint32(o.Code)) - return xw.Tot(), xw.Error() -} - -func (o *ResponseMessage) DecodeXDR(r io.Reader) error { - xr := xdr.NewReader(r) - return o.DecodeXDRFrom(xr) -} - -func (o *ResponseMessage) UnmarshalXDR(bs []byte) error { - var br = bytes.NewReader(bs) - var xr = xdr.NewReader(br) - return o.DecodeXDRFrom(xr) -} - -func (o *ResponseMessage) DecodeXDRFrom(xr *xdr.Reader) error { - o.Data = xr.ReadBytes() - o.Code = int32(xr.ReadUint32()) - return xr.Error() -} - -/* - -ClusterConfigMessage Structure: - - 0 1 2 3 - 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Length of Client Name | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -/ / -\ Client Name (variable length) \ -/ / -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Length of Client Version | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -/ / -\ Client Version (variable length) \ -/ / -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Number of Folders | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -/ / -\ Zero or more Folder Structures \ -/ / -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Number of Options | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -/ / -\ Zero or more Option Structures \ -/ / -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - - -struct ClusterConfigMessage { - string ClientName<64>; - string ClientVersion<64>; - Folder Folders<1000000>; - Option Options<64>; -} - -*/ - -func (o ClusterConfigMessage) EncodeXDR(w io.Writer) (int, error) { - var xw = xdr.NewWriter(w) - return o.EncodeXDRInto(xw) -} - -func (o ClusterConfigMessage) MarshalXDR() ([]byte, error) { - return o.AppendXDR(make([]byte, 0, 128)) -} - -func (o ClusterConfigMessage) MustMarshalXDR() []byte { - bs, err := o.MarshalXDR() - if err != nil { - panic(err) - } - return bs -} - -func (o ClusterConfigMessage) AppendXDR(bs []byte) ([]byte, error) { - var aw = xdr.AppendWriter(bs) - var xw = xdr.NewWriter(&aw) - _, err := o.EncodeXDRInto(xw) - return []byte(aw), err -} - -func (o ClusterConfigMessage) EncodeXDRInto(xw *xdr.Writer) (int, error) { - if l := len(o.ClientName); l > 64 { - return xw.Tot(), xdr.ElementSizeExceeded("ClientName", l, 64) - } - xw.WriteString(o.ClientName) - if l := len(o.ClientVersion); l > 64 { - return xw.Tot(), xdr.ElementSizeExceeded("ClientVersion", l, 64) - } - xw.WriteString(o.ClientVersion) - if l := len(o.Folders); l > 1000000 { - return xw.Tot(), xdr.ElementSizeExceeded("Folders", l, 1000000) - } - xw.WriteUint32(uint32(len(o.Folders))) - for i := range o.Folders { - _, err := o.Folders[i].EncodeXDRInto(xw) - if err != nil { - return xw.Tot(), err - } - } - if l := len(o.Options); l > 64 { - return xw.Tot(), xdr.ElementSizeExceeded("Options", l, 64) - } - xw.WriteUint32(uint32(len(o.Options))) - for i := range o.Options { - _, err := o.Options[i].EncodeXDRInto(xw) - if err != nil { - return xw.Tot(), err - } - } - return xw.Tot(), xw.Error() -} - -func (o *ClusterConfigMessage) DecodeXDR(r io.Reader) error { - xr := xdr.NewReader(r) - return o.DecodeXDRFrom(xr) -} - -func (o *ClusterConfigMessage) UnmarshalXDR(bs []byte) error { - var br = bytes.NewReader(bs) - var xr = xdr.NewReader(br) - return o.DecodeXDRFrom(xr) -} - -func (o *ClusterConfigMessage) DecodeXDRFrom(xr *xdr.Reader) error { - o.ClientName = xr.ReadStringMax(64) - o.ClientVersion = xr.ReadStringMax(64) - _FoldersSize := int(xr.ReadUint32()) - if _FoldersSize < 0 { - return xdr.ElementSizeExceeded("Folders", _FoldersSize, 1000000) - } - if _FoldersSize > 1000000 { - return xdr.ElementSizeExceeded("Folders", _FoldersSize, 1000000) - } - o.Folders = make([]Folder, _FoldersSize) - for i := range o.Folders { - (&o.Folders[i]).DecodeXDRFrom(xr) - } - _OptionsSize := int(xr.ReadUint32()) - if _OptionsSize < 0 { - return xdr.ElementSizeExceeded("Options", _OptionsSize, 64) - } - if _OptionsSize > 64 { - return xdr.ElementSizeExceeded("Options", _OptionsSize, 64) - } - o.Options = make([]Option, _OptionsSize) - for i := range o.Options { - (&o.Options[i]).DecodeXDRFrom(xr) - } - return xr.Error() -} - -/* - -Folder Structure: - - 0 1 2 3 - 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Length of ID | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -/ / -\ ID (variable length) \ -/ / -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Number of Devices | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -/ / -\ Zero or more Device Structures \ -/ / -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Flags | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Number of Options | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -/ / -\ Zero or more Option Structures \ -/ / -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - - -struct Folder { - string ID<64>; - Device Devices<1000000>; - unsigned int Flags; - Option Options<64>; -} - -*/ - -func (o Folder) EncodeXDR(w io.Writer) (int, error) { - var xw = xdr.NewWriter(w) - return o.EncodeXDRInto(xw) -} - -func (o Folder) MarshalXDR() ([]byte, error) { - return o.AppendXDR(make([]byte, 0, 128)) -} - -func (o Folder) MustMarshalXDR() []byte { - bs, err := o.MarshalXDR() - if err != nil { - panic(err) - } - return bs -} - -func (o Folder) AppendXDR(bs []byte) ([]byte, error) { - var aw = xdr.AppendWriter(bs) - var xw = xdr.NewWriter(&aw) - _, err := o.EncodeXDRInto(xw) - return []byte(aw), err -} - -func (o Folder) EncodeXDRInto(xw *xdr.Writer) (int, error) { - if l := len(o.ID); l > 64 { - return xw.Tot(), xdr.ElementSizeExceeded("ID", l, 64) - } - xw.WriteString(o.ID) - if l := len(o.Devices); l > 1000000 { - return xw.Tot(), xdr.ElementSizeExceeded("Devices", l, 1000000) - } - xw.WriteUint32(uint32(len(o.Devices))) - for i := range o.Devices { - _, err := o.Devices[i].EncodeXDRInto(xw) - if err != nil { - return xw.Tot(), err - } - } - xw.WriteUint32(o.Flags) - if l := len(o.Options); l > 64 { - return xw.Tot(), xdr.ElementSizeExceeded("Options", l, 64) - } - xw.WriteUint32(uint32(len(o.Options))) - for i := range o.Options { - _, err := o.Options[i].EncodeXDRInto(xw) - if err != nil { - return xw.Tot(), err - } - } - return xw.Tot(), xw.Error() -} - -func (o *Folder) DecodeXDR(r io.Reader) error { - xr := xdr.NewReader(r) - return o.DecodeXDRFrom(xr) -} - -func (o *Folder) UnmarshalXDR(bs []byte) error { - var br = bytes.NewReader(bs) - var xr = xdr.NewReader(br) - return o.DecodeXDRFrom(xr) -} - -func (o *Folder) DecodeXDRFrom(xr *xdr.Reader) error { - o.ID = xr.ReadStringMax(64) - _DevicesSize := int(xr.ReadUint32()) - if _DevicesSize < 0 { - return xdr.ElementSizeExceeded("Devices", _DevicesSize, 1000000) - } - if _DevicesSize > 1000000 { - return xdr.ElementSizeExceeded("Devices", _DevicesSize, 1000000) - } - o.Devices = make([]Device, _DevicesSize) - for i := range o.Devices { - (&o.Devices[i]).DecodeXDRFrom(xr) - } - o.Flags = xr.ReadUint32() - _OptionsSize := int(xr.ReadUint32()) - if _OptionsSize < 0 { - return xdr.ElementSizeExceeded("Options", _OptionsSize, 64) - } - if _OptionsSize > 64 { - return xdr.ElementSizeExceeded("Options", _OptionsSize, 64) - } - o.Options = make([]Option, _OptionsSize) - for i := range o.Options { - (&o.Options[i]).DecodeXDRFrom(xr) - } - return xr.Error() -} - -/* - -Device Structure: - - 0 1 2 3 - 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Length of ID | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -/ / -\ ID (variable length) \ -/ / -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| | -+ Max Local Version (64 bits) + -| | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Flags | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Number of Options | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -/ / -\ Zero or more Option Structures \ -/ / -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - - -struct Device { - opaque ID<32>; - hyper MaxLocalVersion; - unsigned int Flags; - Option Options<64>; -} - -*/ - -func (o Device) EncodeXDR(w io.Writer) (int, error) { - var xw = xdr.NewWriter(w) - return o.EncodeXDRInto(xw) -} - -func (o Device) MarshalXDR() ([]byte, error) { - return o.AppendXDR(make([]byte, 0, 128)) -} - -func (o Device) MustMarshalXDR() []byte { - bs, err := o.MarshalXDR() - if err != nil { - panic(err) - } - return bs -} - -func (o Device) AppendXDR(bs []byte) ([]byte, error) { - var aw = xdr.AppendWriter(bs) - var xw = xdr.NewWriter(&aw) - _, err := o.EncodeXDRInto(xw) - return []byte(aw), err -} - -func (o Device) EncodeXDRInto(xw *xdr.Writer) (int, error) { - if l := len(o.ID); l > 32 { - return xw.Tot(), xdr.ElementSizeExceeded("ID", l, 32) - } - xw.WriteBytes(o.ID) - xw.WriteUint64(uint64(o.MaxLocalVersion)) - xw.WriteUint32(o.Flags) - if l := len(o.Options); l > 64 { - return xw.Tot(), xdr.ElementSizeExceeded("Options", l, 64) - } - xw.WriteUint32(uint32(len(o.Options))) - for i := range o.Options { - _, err := o.Options[i].EncodeXDRInto(xw) - if err != nil { - return xw.Tot(), err - } - } - return xw.Tot(), xw.Error() -} - -func (o *Device) DecodeXDR(r io.Reader) error { - xr := xdr.NewReader(r) - return o.DecodeXDRFrom(xr) -} - -func (o *Device) UnmarshalXDR(bs []byte) error { - var br = bytes.NewReader(bs) - var xr = xdr.NewReader(br) - return o.DecodeXDRFrom(xr) -} - -func (o *Device) DecodeXDRFrom(xr *xdr.Reader) error { - o.ID = xr.ReadBytesMax(32) - o.MaxLocalVersion = int64(xr.ReadUint64()) - o.Flags = xr.ReadUint32() - _OptionsSize := int(xr.ReadUint32()) - if _OptionsSize < 0 { - return xdr.ElementSizeExceeded("Options", _OptionsSize, 64) - } - if _OptionsSize > 64 { - return xdr.ElementSizeExceeded("Options", _OptionsSize, 64) - } - o.Options = make([]Option, _OptionsSize) - for i := range o.Options { - (&o.Options[i]).DecodeXDRFrom(xr) - } - return xr.Error() -} - -/* - -Option Structure: - - 0 1 2 3 - 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Length of Key | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -/ / -\ Key (variable length) \ -/ / -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Length of Value | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -/ / -\ Value (variable length) \ -/ / -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - - -struct Option { - string Key<64>; - string Value<1024>; -} - -*/ - -func (o Option) EncodeXDR(w io.Writer) (int, error) { - var xw = xdr.NewWriter(w) - return o.EncodeXDRInto(xw) -} - -func (o Option) MarshalXDR() ([]byte, error) { - return o.AppendXDR(make([]byte, 0, 128)) -} - -func (o Option) MustMarshalXDR() []byte { - bs, err := o.MarshalXDR() - if err != nil { - panic(err) - } - return bs -} - -func (o Option) AppendXDR(bs []byte) ([]byte, error) { - var aw = xdr.AppendWriter(bs) - var xw = xdr.NewWriter(&aw) - _, err := o.EncodeXDRInto(xw) - return []byte(aw), err -} - -func (o Option) EncodeXDRInto(xw *xdr.Writer) (int, error) { - if l := len(o.Key); l > 64 { - return xw.Tot(), xdr.ElementSizeExceeded("Key", l, 64) - } - xw.WriteString(o.Key) - if l := len(o.Value); l > 1024 { - return xw.Tot(), xdr.ElementSizeExceeded("Value", l, 1024) - } - xw.WriteString(o.Value) - return xw.Tot(), xw.Error() -} - -func (o *Option) DecodeXDR(r io.Reader) error { - xr := xdr.NewReader(r) - return o.DecodeXDRFrom(xr) -} - -func (o *Option) UnmarshalXDR(bs []byte) error { - var br = bytes.NewReader(bs) - var xr = xdr.NewReader(br) - return o.DecodeXDRFrom(xr) -} - -func (o *Option) DecodeXDRFrom(xr *xdr.Reader) error { - o.Key = xr.ReadStringMax(64) - o.Value = xr.ReadStringMax(1024) - return xr.Error() -} - -/* - -CloseMessage Structure: - - 0 1 2 3 - 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Length of Reason | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -/ / -\ Reason (variable length) \ -/ / -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Code | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - - -struct CloseMessage { - string Reason<1024>; - int Code; -} - -*/ - -func (o CloseMessage) EncodeXDR(w io.Writer) (int, error) { - var xw = xdr.NewWriter(w) - return o.EncodeXDRInto(xw) -} - -func (o CloseMessage) MarshalXDR() ([]byte, error) { - return o.AppendXDR(make([]byte, 0, 128)) -} - -func (o CloseMessage) MustMarshalXDR() []byte { - bs, err := o.MarshalXDR() - if err != nil { - panic(err) - } - return bs -} - -func (o CloseMessage) AppendXDR(bs []byte) ([]byte, error) { - var aw = xdr.AppendWriter(bs) - var xw = xdr.NewWriter(&aw) - _, err := o.EncodeXDRInto(xw) - return []byte(aw), err -} - -func (o CloseMessage) EncodeXDRInto(xw *xdr.Writer) (int, error) { - if l := len(o.Reason); l > 1024 { - return xw.Tot(), xdr.ElementSizeExceeded("Reason", l, 1024) - } - xw.WriteString(o.Reason) - xw.WriteUint32(uint32(o.Code)) - return xw.Tot(), xw.Error() -} - -func (o *CloseMessage) DecodeXDR(r io.Reader) error { - xr := xdr.NewReader(r) - return o.DecodeXDRFrom(xr) -} - -func (o *CloseMessage) UnmarshalXDR(bs []byte) error { - var br = bytes.NewReader(bs) - var xr = xdr.NewReader(br) - return o.DecodeXDRFrom(xr) -} - -func (o *CloseMessage) DecodeXDRFrom(xr *xdr.Reader) error { - o.Reason = xr.ReadStringMax(1024) - o.Code = int32(xr.ReadUint32()) - return xr.Error() -} - -/* - -EmptyMessage Structure: - - 0 1 2 3 - 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - - -struct EmptyMessage { -} - -*/ - -func (o EmptyMessage) EncodeXDR(w io.Writer) (int, error) { - var xw = xdr.NewWriter(w) - return o.EncodeXDRInto(xw) -} - -func (o EmptyMessage) MarshalXDR() ([]byte, error) { - return o.AppendXDR(make([]byte, 0, 128)) -} - -func (o EmptyMessage) MustMarshalXDR() []byte { - bs, err := o.MarshalXDR() - if err != nil { - panic(err) - } - return bs -} - -func (o EmptyMessage) AppendXDR(bs []byte) ([]byte, error) { - var aw = xdr.AppendWriter(bs) - var xw = xdr.NewWriter(&aw) - _, err := o.EncodeXDRInto(xw) - return []byte(aw), err -} - -func (o EmptyMessage) EncodeXDRInto(xw *xdr.Writer) (int, error) { - return xw.Tot(), xw.Error() -} - -func (o *EmptyMessage) DecodeXDR(r io.Reader) error { - xr := xdr.NewReader(r) - return o.DecodeXDRFrom(xr) -} - -func (o *EmptyMessage) UnmarshalXDR(bs []byte) error { - var br = bytes.NewReader(bs) - var xr = xdr.NewReader(br) - return o.DecodeXDRFrom(xr) -} - -func (o *EmptyMessage) DecodeXDRFrom(xr *xdr.Reader) error { - return xr.Error() -} diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/nativemodel_darwin.go b/Godeps/_workspace/src/github.com/syncthing/protocol/nativemodel_darwin.go deleted file mode 100644 index eb755a6e4..000000000 --- a/Godeps/_workspace/src/github.com/syncthing/protocol/nativemodel_darwin.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright (C) 2014 The Protocol Authors. - -// +build darwin - -package protocol - -// Darwin uses NFD normalization - -import "golang.org/x/text/unicode/norm" - -type nativeModel struct { - next Model -} - -func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { - for i := range files { - files[i].Name = norm.NFD.String(files[i].Name) - } - m.next.Index(deviceID, folder, files, flags, options) -} - -func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { - for i := range files { - files[i].Name = norm.NFD.String(files[i].Name) - } - m.next.IndexUpdate(deviceID, folder, files, flags, options) -} - -func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, flags uint32, options []Option, buf []byte) error { - name = norm.NFD.String(name) - return m.next.Request(deviceID, folder, name, offset, hash, flags, options, buf) -} - -func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) { - m.next.ClusterConfig(deviceID, config) -} - -func (m nativeModel) Close(deviceID DeviceID, err error) { - m.next.Close(deviceID, err) -} diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/nativemodel_unix.go b/Godeps/_workspace/src/github.com/syncthing/protocol/nativemodel_unix.go deleted file mode 100644 index 0611865e1..000000000 --- a/Godeps/_workspace/src/github.com/syncthing/protocol/nativemodel_unix.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (C) 2014 The Protocol Authors. - -// +build !windows,!darwin - -package protocol - -// Normal Unixes uses NFC and slashes, which is the wire format. - -type nativeModel struct { - next Model -} - -func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { - m.next.Index(deviceID, folder, files, flags, options) -} - -func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { - m.next.IndexUpdate(deviceID, folder, files, flags, options) -} - -func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, flags uint32, options []Option, buf []byte) error { - return m.next.Request(deviceID, folder, name, offset, hash, flags, options, buf) -} - -func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) { - m.next.ClusterConfig(deviceID, config) -} - -func (m nativeModel) Close(deviceID DeviceID, err error) { - m.next.Close(deviceID, err) -} diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/nativemodel_windows.go b/Godeps/_workspace/src/github.com/syncthing/protocol/nativemodel_windows.go deleted file mode 100644 index 36a1d2749..000000000 --- a/Godeps/_workspace/src/github.com/syncthing/protocol/nativemodel_windows.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright (C) 2014 The Protocol Authors. - -// +build windows - -package protocol - -// Windows uses backslashes as file separator and disallows a bunch of -// characters in the filename - -import ( - "path/filepath" - "strings" -) - -var disallowedCharacters = string([]rune{ - '<', '>', ':', '"', '|', '?', '*', - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, - 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, - 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, - 31, -}) - -type nativeModel struct { - next Model -} - -func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { - fixupFiles(folder, files) - m.next.Index(deviceID, folder, files, flags, options) -} - -func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) { - fixupFiles(folder, files) - m.next.IndexUpdate(deviceID, folder, files, flags, options) -} - -func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, flags uint32, options []Option, buf []byte) error { - name = filepath.FromSlash(name) - return m.next.Request(deviceID, folder, name, offset, hash, flags, options, buf) -} - -func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) { - m.next.ClusterConfig(deviceID, config) -} - -func (m nativeModel) Close(deviceID DeviceID, err error) { - m.next.Close(deviceID, err) -} - -func fixupFiles(folder string, files []FileInfo) { - for i, f := range files { - if strings.ContainsAny(f.Name, disallowedCharacters) { - if f.IsDeleted() { - // Don't complain if the file is marked as deleted, since it - // can't possibly exist here anyway. - continue - } - files[i].Flags |= FlagInvalid - l.Warnf("File name %q (folder %q) contains invalid characters; marked as invalid.", f.Name, folder) - } - files[i].Name = filepath.FromSlash(files[i].Name) - } -} diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/protocol.go b/Godeps/_workspace/src/github.com/syncthing/protocol/protocol.go deleted file mode 100644 index 4c1364eaf..000000000 --- a/Godeps/_workspace/src/github.com/syncthing/protocol/protocol.go +++ /dev/null @@ -1,782 +0,0 @@ -// Copyright (C) 2014 The Protocol Authors. - -package protocol - -import ( - "encoding/binary" - "encoding/hex" - "errors" - "fmt" - "io" - "sync" - "time" - - lz4 "github.com/bkaradzic/go-lz4" -) - -const ( - // Data block size (128 KiB) - BlockSize = 128 << 10 - - // We reject messages larger than this when encountered on the wire. (64 MiB) - MaxMessageLen = 64 << 20 -) - -const ( - messageTypeClusterConfig = 0 - messageTypeIndex = 1 - messageTypeRequest = 2 - messageTypeResponse = 3 - messageTypePing = 4 - messageTypeIndexUpdate = 6 - messageTypeClose = 7 -) - -const ( - stateInitial = iota - stateReady -) - -// FileInfo flags -const ( - FlagDeleted uint32 = 1 << 12 - FlagInvalid = 1 << 13 - FlagDirectory = 1 << 14 - FlagNoPermBits = 1 << 15 - FlagSymlink = 1 << 16 - FlagSymlinkMissingTarget = 1 << 17 - - FlagsAll = (1 << 18) - 1 - - SymlinkTypeMask = FlagDirectory | FlagSymlinkMissingTarget -) - -// IndexMessage message flags (for IndexUpdate) -const ( - FlagIndexTemporary uint32 = 1 << iota -) - -// Request message flags -const ( - FlagRequestTemporary uint32 = 1 << iota -) - -// ClusterConfigMessage.Folders.Devices flags -const ( - FlagShareTrusted uint32 = 1 << 0 - FlagShareReadOnly = 1 << 1 - FlagIntroducer = 1 << 2 - FlagShareBits = 0x000000ff -) - -var ( - ErrClosed = errors.New("connection closed") - ErrTimeout = errors.New("read timeout") -) - -// Specific variants of empty messages... -type pingMessage struct{ EmptyMessage } - -type Model interface { - // An index was received from the peer device - Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) - // An index update was received from the peer device - IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) - // A request was made by the peer device - Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, flags uint32, options []Option, buf []byte) error - // A cluster configuration message was received - ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) - // The peer device closed the connection - Close(deviceID DeviceID, err error) -} - -type Connection interface { - Start() - ID() DeviceID - Name() string - Index(folder string, files []FileInfo, flags uint32, options []Option) error - IndexUpdate(folder string, files []FileInfo, flags uint32, options []Option) error - Request(folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) - ClusterConfig(config ClusterConfigMessage) - Statistics() Statistics -} - -type rawConnection struct { - id DeviceID - name string - receiver Model - - cr *countingReader - cw *countingWriter - - awaiting [4096]chan asyncResult - awaitingMut sync.Mutex - - idxMut sync.Mutex // ensures serialization of Index calls - - nextID chan int - outbox chan hdrMsg - closed chan struct{} - once sync.Once - pool sync.Pool - compression Compression - - rdbuf0 []byte // used & reused by readMessage - rdbuf1 []byte // used & reused by readMessage -} - -type asyncResult struct { - val []byte - err error -} - -type hdrMsg struct { - hdr header - msg encodable - done chan struct{} -} - -type encodable interface { - AppendXDR([]byte) ([]byte, error) -} - -type isEofer interface { - IsEOF() bool -} - -const ( - // We make sure to send a message at least this often, by triggering pings. - PingSendInterval = 90 * time.Second - // If we haven't received a message from the other side for this long, close the connection. - ReceiveTimeout = 300 * time.Second -) - -func NewConnection(deviceID DeviceID, reader io.Reader, writer io.Writer, receiver Model, name string, compress Compression) Connection { - cr := &countingReader{Reader: reader} - cw := &countingWriter{Writer: writer} - - c := rawConnection{ - id: deviceID, - name: name, - receiver: nativeModel{receiver}, - cr: cr, - cw: cw, - outbox: make(chan hdrMsg), - nextID: make(chan int), - closed: make(chan struct{}), - pool: sync.Pool{ - New: func() interface{} { - return make([]byte, BlockSize) - }, - }, - compression: compress, - } - - return wireFormatConnection{&c} -} - -// Start creates the goroutines for sending and receiving of messages. It must -// be called exactly once after creating a connection. -func (c *rawConnection) Start() { - go c.readerLoop() - go c.writerLoop() - go c.pingSender() - go c.pingReceiver() - go c.idGenerator() -} - -func (c *rawConnection) ID() DeviceID { - return c.id -} - -func (c *rawConnection) Name() string { - return c.name -} - -// Index writes the list of file information to the connected peer device -func (c *rawConnection) Index(folder string, idx []FileInfo, flags uint32, options []Option) error { - select { - case <-c.closed: - return ErrClosed - default: - } - c.idxMut.Lock() - c.send(-1, messageTypeIndex, IndexMessage{ - Folder: folder, - Files: idx, - Flags: flags, - Options: options, - }, nil) - c.idxMut.Unlock() - return nil -} - -// IndexUpdate writes the list of file information to the connected peer device as an update -func (c *rawConnection) IndexUpdate(folder string, idx []FileInfo, flags uint32, options []Option) error { - select { - case <-c.closed: - return ErrClosed - default: - } - c.idxMut.Lock() - c.send(-1, messageTypeIndexUpdate, IndexMessage{ - Folder: folder, - Files: idx, - Flags: flags, - Options: options, - }, nil) - c.idxMut.Unlock() - return nil -} - -// Request returns the bytes for the specified block after fetching them from the connected peer. -func (c *rawConnection) Request(folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) { - var id int - select { - case id = <-c.nextID: - case <-c.closed: - return nil, ErrClosed - } - - c.awaitingMut.Lock() - if ch := c.awaiting[id]; ch != nil { - panic("id taken") - } - rc := make(chan asyncResult, 1) - c.awaiting[id] = rc - c.awaitingMut.Unlock() - - ok := c.send(id, messageTypeRequest, RequestMessage{ - Folder: folder, - Name: name, - Offset: offset, - Size: int32(size), - Hash: hash, - Flags: flags, - Options: options, - }, nil) - if !ok { - return nil, ErrClosed - } - - res, ok := <-rc - if !ok { - return nil, ErrClosed - } - return res.val, res.err -} - -// ClusterConfig send the cluster configuration message to the peer and returns any error -func (c *rawConnection) ClusterConfig(config ClusterConfigMessage) { - c.send(-1, messageTypeClusterConfig, config, nil) -} - -func (c *rawConnection) ping() bool { - var id int - select { - case id = <-c.nextID: - case <-c.closed: - return false - } - - return c.send(id, messageTypePing, nil, nil) -} - -func (c *rawConnection) readerLoop() (err error) { - defer func() { - c.close(err) - }() - - state := stateInitial - for { - select { - case <-c.closed: - return ErrClosed - default: - } - - hdr, msg, err := c.readMessage() - if err != nil { - return err - } - - switch msg := msg.(type) { - case ClusterConfigMessage: - if state != stateInitial { - return fmt.Errorf("protocol error: cluster config message in state %d", state) - } - go c.receiver.ClusterConfig(c.id, msg) - state = stateReady - - case IndexMessage: - switch hdr.msgType { - case messageTypeIndex: - if state != stateReady { - return fmt.Errorf("protocol error: index message in state %d", state) - } - c.handleIndex(msg) - state = stateReady - - case messageTypeIndexUpdate: - if state != stateReady { - return fmt.Errorf("protocol error: index update message in state %d", state) - } - c.handleIndexUpdate(msg) - state = stateReady - } - - case RequestMessage: - if state != stateReady { - return fmt.Errorf("protocol error: request message in state %d", state) - } - // Requests are handled asynchronously - go c.handleRequest(hdr.msgID, msg) - - case ResponseMessage: - if state != stateReady { - return fmt.Errorf("protocol error: response message in state %d", state) - } - c.handleResponse(hdr.msgID, msg) - - case pingMessage: - if state != stateReady { - return fmt.Errorf("protocol error: ping message in state %d", state) - } - // Nothing - - case CloseMessage: - return errors.New(msg.Reason) - - default: - return fmt.Errorf("protocol error: %s: unknown message type %#x", c.id, hdr.msgType) - } - } -} - -func (c *rawConnection) readMessage() (hdr header, msg encodable, err error) { - if cap(c.rdbuf0) < 8 { - c.rdbuf0 = make([]byte, 8) - } else { - c.rdbuf0 = c.rdbuf0[:8] - } - _, err = io.ReadFull(c.cr, c.rdbuf0) - if err != nil { - return - } - - hdr = decodeHeader(binary.BigEndian.Uint32(c.rdbuf0[0:4])) - msglen := int(binary.BigEndian.Uint32(c.rdbuf0[4:8])) - - if debug { - l.Debugf("read header %v (msglen=%d)", hdr, msglen) - } - - if msglen > MaxMessageLen { - err = fmt.Errorf("message length %d exceeds maximum %d", msglen, MaxMessageLen) - return - } - - if hdr.version != 0 { - err = fmt.Errorf("unknown protocol version 0x%x", hdr.version) - return - } - - if cap(c.rdbuf0) < msglen { - c.rdbuf0 = make([]byte, msglen) - } else { - c.rdbuf0 = c.rdbuf0[:msglen] - } - _, err = io.ReadFull(c.cr, c.rdbuf0) - if err != nil { - return - } - - if debug { - l.Debugf("read %d bytes", len(c.rdbuf0)) - } - - msgBuf := c.rdbuf0 - if hdr.compression && msglen > 0 { - c.rdbuf1 = c.rdbuf1[:cap(c.rdbuf1)] - c.rdbuf1, err = lz4.Decode(c.rdbuf1, c.rdbuf0) - if err != nil { - return - } - msgBuf = c.rdbuf1 - if debug { - l.Debugf("decompressed to %d bytes", len(msgBuf)) - } - } - - if debug { - if len(msgBuf) > 1024 { - l.Debugf("message data:\n%s", hex.Dump(msgBuf[:1024])) - } else { - l.Debugf("message data:\n%s", hex.Dump(msgBuf)) - } - } - - // We check each returned error for the XDRError.IsEOF() method. - // IsEOF()==true here means that the message contained fewer fields than - // expected. It does not signify an EOF on the socket, because we've - // successfully read a size value and that many bytes already. New fields - // we expected but the other peer didn't send should be interpreted as - // zero/nil, and if that's not valid we'll verify it somewhere else. - - switch hdr.msgType { - case messageTypeIndex, messageTypeIndexUpdate: - var idx IndexMessage - err = idx.UnmarshalXDR(msgBuf) - if xdrErr, ok := err.(isEofer); ok && xdrErr.IsEOF() { - err = nil - } - msg = idx - - case messageTypeRequest: - var req RequestMessage - err = req.UnmarshalXDR(msgBuf) - if xdrErr, ok := err.(isEofer); ok && xdrErr.IsEOF() { - err = nil - } - msg = req - - case messageTypeResponse: - var resp ResponseMessage - err = resp.UnmarshalXDR(msgBuf) - if xdrErr, ok := err.(isEofer); ok && xdrErr.IsEOF() { - err = nil - } - msg = resp - - case messageTypePing: - msg = pingMessage{} - - case messageTypeClusterConfig: - var cc ClusterConfigMessage - err = cc.UnmarshalXDR(msgBuf) - if xdrErr, ok := err.(isEofer); ok && xdrErr.IsEOF() { - err = nil - } - msg = cc - - case messageTypeClose: - var cm CloseMessage - err = cm.UnmarshalXDR(msgBuf) - if xdrErr, ok := err.(isEofer); ok && xdrErr.IsEOF() { - err = nil - } - msg = cm - - default: - err = fmt.Errorf("protocol error: %s: unknown message type %#x", c.id, hdr.msgType) - } - - return -} - -func (c *rawConnection) handleIndex(im IndexMessage) { - if debug { - l.Debugf("Index(%v, %v, %d file, flags %x, opts: %s)", c.id, im.Folder, len(im.Files), im.Flags, im.Options) - } - c.receiver.Index(c.id, im.Folder, filterIndexMessageFiles(im.Files), im.Flags, im.Options) -} - -func (c *rawConnection) handleIndexUpdate(im IndexMessage) { - if debug { - l.Debugf("queueing IndexUpdate(%v, %v, %d files, flags %x, opts: %s)", c.id, im.Folder, len(im.Files), im.Flags, im.Options) - } - c.receiver.IndexUpdate(c.id, im.Folder, filterIndexMessageFiles(im.Files), im.Flags, im.Options) -} - -func filterIndexMessageFiles(fs []FileInfo) []FileInfo { - var out []FileInfo - for i, f := range fs { - switch f.Name { - case "", ".", "..", "/": // A few obviously invalid filenames - l.Infof("Dropping invalid filename %q from incoming index", f.Name) - if out == nil { - // Most incoming updates won't contain anything invalid, so we - // delay the allocation and copy to output slice until we - // really need to do it, then copy all the so var valid files - // to it. - out = make([]FileInfo, i, len(fs)-1) - copy(out, fs) - } - default: - if out != nil { - out = append(out, f) - } - } - } - if out != nil { - return out - } - return fs -} - -func (c *rawConnection) handleRequest(msgID int, req RequestMessage) { - size := int(req.Size) - usePool := size <= BlockSize - - var buf []byte - var done chan struct{} - - if usePool { - buf = c.pool.Get().([]byte)[:size] - done = make(chan struct{}) - } else { - buf = make([]byte, size) - } - - err := c.receiver.Request(c.id, req.Folder, req.Name, int64(req.Offset), req.Hash, req.Flags, req.Options, buf) - if err != nil { - c.send(msgID, messageTypeResponse, ResponseMessage{ - Data: nil, - Code: errorToCode(err), - }, done) - } else { - c.send(msgID, messageTypeResponse, ResponseMessage{ - Data: buf, - Code: errorToCode(err), - }, done) - } - - if usePool { - <-done - c.pool.Put(buf) - } -} - -func (c *rawConnection) handleResponse(msgID int, resp ResponseMessage) { - c.awaitingMut.Lock() - if rc := c.awaiting[msgID]; rc != nil { - c.awaiting[msgID] = nil - rc <- asyncResult{resp.Data, codeToError(resp.Code)} - close(rc) - } - c.awaitingMut.Unlock() -} - -func (c *rawConnection) handlePong(msgID int) { - c.awaitingMut.Lock() - if rc := c.awaiting[msgID]; rc != nil { - c.awaiting[msgID] = nil - rc <- asyncResult{} - close(rc) - } - c.awaitingMut.Unlock() -} - -func (c *rawConnection) send(msgID int, msgType int, msg encodable, done chan struct{}) bool { - if msgID < 0 { - select { - case id := <-c.nextID: - msgID = id - case <-c.closed: - return false - } - } - - hdr := header{ - version: 0, - msgID: msgID, - msgType: msgType, - } - - select { - case c.outbox <- hdrMsg{hdr, msg, done}: - return true - case <-c.closed: - return false - } -} - -func (c *rawConnection) writerLoop() { - var msgBuf = make([]byte, 8) // buffer for wire format message, kept and reused - var uncBuf []byte // buffer for uncompressed message, kept and reused - for { - var tempBuf []byte - var err error - - select { - case hm := <-c.outbox: - if hm.msg != nil { - // Uncompressed message in uncBuf - uncBuf, err = hm.msg.AppendXDR(uncBuf[:0]) - if hm.done != nil { - close(hm.done) - } - if err != nil { - c.close(err) - return - } - - compress := false - switch c.compression { - case CompressAlways: - compress = true - case CompressMetadata: - compress = hm.hdr.msgType != messageTypeResponse - } - - if compress && len(uncBuf) >= compressionThreshold { - // Use compression for large messages - hm.hdr.compression = true - - // Make sure we have enough space for the compressed message plus header in msgBug - msgBuf = msgBuf[:cap(msgBuf)] - if maxLen := lz4.CompressBound(len(uncBuf)) + 8; maxLen > len(msgBuf) { - msgBuf = make([]byte, maxLen) - } - - // Compressed is written to msgBuf, we keep tb for the length only - tempBuf, err = lz4.Encode(msgBuf[8:], uncBuf) - binary.BigEndian.PutUint32(msgBuf[4:8], uint32(len(tempBuf))) - msgBuf = msgBuf[0 : len(tempBuf)+8] - - if debug { - l.Debugf("write compressed message; %v (len=%d)", hm.hdr, len(tempBuf)) - } - } else { - // No point in compressing very short messages - hm.hdr.compression = false - - msgBuf = msgBuf[:cap(msgBuf)] - if l := len(uncBuf) + 8; l > len(msgBuf) { - msgBuf = make([]byte, l) - } - - binary.BigEndian.PutUint32(msgBuf[4:8], uint32(len(uncBuf))) - msgBuf = msgBuf[0 : len(uncBuf)+8] - copy(msgBuf[8:], uncBuf) - - if debug { - l.Debugf("write uncompressed message; %v (len=%d)", hm.hdr, len(uncBuf)) - } - } - } else { - if debug { - l.Debugf("write empty message; %v", hm.hdr) - } - binary.BigEndian.PutUint32(msgBuf[4:8], 0) - msgBuf = msgBuf[:8] - } - - binary.BigEndian.PutUint32(msgBuf[0:4], encodeHeader(hm.hdr)) - - if err == nil { - var n int - n, err = c.cw.Write(msgBuf) - if debug { - l.Debugf("wrote %d bytes on the wire", n) - } - } - if err != nil { - c.close(err) - return - } - case <-c.closed: - return - } - } -} - -func (c *rawConnection) close(err error) { - c.once.Do(func() { - close(c.closed) - - c.awaitingMut.Lock() - for i, ch := range c.awaiting { - if ch != nil { - close(ch) - c.awaiting[i] = nil - } - } - c.awaitingMut.Unlock() - - go c.receiver.Close(c.id, err) - }) -} - -func (c *rawConnection) idGenerator() { - nextID := 0 - for { - nextID = (nextID + 1) & 0xfff - select { - case c.nextID <- nextID: - case <-c.closed: - return - } - } -} - -// The pingSender makes sure that we've sent a message within the last -// PingSendInterval. If we already have something sent in the last -// PingSendInterval/2, we do nothing. Otherwise we send a ping message. This -// results in an effecting ping interval of somewhere between -// PingSendInterval/2 and PingSendInterval. -func (c *rawConnection) pingSender() { - ticker := time.Tick(PingSendInterval / 2) - - for { - select { - case <-ticker: - d := time.Since(c.cw.Last()) - if d < PingSendInterval/2 { - if debug { - l.Debugln(c.id, "ping skipped after wr", d) - } - continue - } - - if debug { - l.Debugln(c.id, "ping -> after", d) - } - c.ping() - - case <-c.closed: - return - } - } -} - -// The pingReciever checks that we've received a message (any message will do, -// but we expect pings in the absence of other messages) within the last -// ReceiveTimeout. If not, we close the connection with an ErrTimeout. -func (c *rawConnection) pingReceiver() { - ticker := time.Tick(ReceiveTimeout / 2) - - for { - select { - case <-ticker: - d := time.Since(c.cr.Last()) - if d > ReceiveTimeout { - if debug { - l.Debugln(c.id, "ping timeout", d) - } - c.close(ErrTimeout) - } - - if debug { - l.Debugln(c.id, "last read within", d) - } - - case <-c.closed: - return - } - } -} - -type Statistics struct { - At time.Time - InBytesTotal int64 - OutBytesTotal int64 -} - -func (c *rawConnection) Statistics() Statistics { - return Statistics{ - At: time.Now(), - InBytesTotal: c.cr.Tot(), - OutBytesTotal: c.cw.Tot(), - } -} diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/protocol_test.go b/Godeps/_workspace/src/github.com/syncthing/protocol/protocol_test.go deleted file mode 100644 index 8a4708843..000000000 --- a/Godeps/_workspace/src/github.com/syncthing/protocol/protocol_test.go +++ /dev/null @@ -1,316 +0,0 @@ -// Copyright (C) 2014 The Protocol Authors. - -package protocol - -import ( - "bytes" - "encoding/hex" - "encoding/json" - "fmt" - "io" - "io/ioutil" - "os" - "reflect" - "strings" - "testing" - "testing/quick" - - "github.com/calmh/xdr" -) - -var ( - c0ID = NewDeviceID([]byte{1}) - c1ID = NewDeviceID([]byte{2}) -) - -func TestHeaderFunctions(t *testing.T) { - f := func(ver, id, typ int) bool { - ver = int(uint(ver) % 16) - id = int(uint(id) % 4096) - typ = int(uint(typ) % 256) - h0 := header{version: ver, msgID: id, msgType: typ} - h1 := decodeHeader(encodeHeader(h0)) - return h0 == h1 - } - if err := quick.Check(f, nil); err != nil { - t.Error(err) - } -} - -func TestHeaderLayout(t *testing.T) { - var e, a uint32 - - // Version are the first four bits - e = 0xf0000000 - a = encodeHeader(header{version: 0xf}) - if a != e { - t.Errorf("Header layout incorrect; %08x != %08x", a, e) - } - - // Message ID are the following 12 bits - e = 0x0fff0000 - a = encodeHeader(header{msgID: 0xfff}) - if a != e { - t.Errorf("Header layout incorrect; %08x != %08x", a, e) - } - - // Type are the last 8 bits before reserved - e = 0x0000ff00 - a = encodeHeader(header{msgType: 0xff}) - if a != e { - t.Errorf("Header layout incorrect; %08x != %08x", a, e) - } -} - -func TestPing(t *testing.T) { - ar, aw := io.Pipe() - br, bw := io.Pipe() - - c0 := NewConnection(c0ID, ar, bw, newTestModel(), "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) - c0.Start() - c1 := NewConnection(c1ID, br, aw, newTestModel(), "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) - c1.Start() - c0.ClusterConfig(ClusterConfigMessage{}) - c1.ClusterConfig(ClusterConfigMessage{}) - - if ok := c0.ping(); !ok { - t.Error("c0 ping failed") - } - if ok := c1.ping(); !ok { - t.Error("c1 ping failed") - } -} - -func TestVersionErr(t *testing.T) { - m0 := newTestModel() - m1 := newTestModel() - - ar, aw := io.Pipe() - br, bw := io.Pipe() - - c0 := NewConnection(c0ID, ar, bw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) - c0.Start() - c1 := NewConnection(c1ID, br, aw, m1, "name", CompressAlways) - c1.Start() - c0.ClusterConfig(ClusterConfigMessage{}) - c1.ClusterConfig(ClusterConfigMessage{}) - - w := xdr.NewWriter(c0.cw) - w.WriteUint32(encodeHeader(header{ - version: 2, - msgID: 0, - msgType: 0, - })) - w.WriteUint32(0) // Avoids reader closing due to EOF - - if !m1.isClosed() { - t.Error("Connection should close due to unknown version") - } -} - -func TestTypeErr(t *testing.T) { - m0 := newTestModel() - m1 := newTestModel() - - ar, aw := io.Pipe() - br, bw := io.Pipe() - - c0 := NewConnection(c0ID, ar, bw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) - c0.Start() - c1 := NewConnection(c1ID, br, aw, m1, "name", CompressAlways) - c1.Start() - c0.ClusterConfig(ClusterConfigMessage{}) - c1.ClusterConfig(ClusterConfigMessage{}) - - w := xdr.NewWriter(c0.cw) - w.WriteUint32(encodeHeader(header{ - version: 0, - msgID: 0, - msgType: 42, - })) - w.WriteUint32(0) // Avoids reader closing due to EOF - - if !m1.isClosed() { - t.Error("Connection should close due to unknown message type") - } -} - -func TestClose(t *testing.T) { - m0 := newTestModel() - m1 := newTestModel() - - ar, aw := io.Pipe() - br, bw := io.Pipe() - - c0 := NewConnection(c0ID, ar, bw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection) - c0.Start() - c1 := NewConnection(c1ID, br, aw, m1, "name", CompressAlways) - c1.Start() - c0.ClusterConfig(ClusterConfigMessage{}) - c1.ClusterConfig(ClusterConfigMessage{}) - - c0.close(nil) - - <-c0.closed - if !m0.isClosed() { - t.Fatal("Connection should be closed") - } - - // None of these should panic, some should return an error - - if c0.ping() { - t.Error("Ping should not return true") - } - - c0.Index("default", nil, 0, nil) - c0.Index("default", nil, 0, nil) - - if _, err := c0.Request("default", "foo", 0, 0, nil, 0, nil); err == nil { - t.Error("Request should return an error") - } -} - -func TestElementSizeExceededNested(t *testing.T) { - m := ClusterConfigMessage{ - Folders: []Folder{ - {ID: "longstringlongstringlongstringinglongstringlongstringlonlongstringlongstringlon"}, - }, - } - _, err := m.EncodeXDR(ioutil.Discard) - if err == nil { - t.Errorf("ID length %d > max 64, but no error", len(m.Folders[0].ID)) - } -} - -func TestMarshalIndexMessage(t *testing.T) { - var quickCfg = &quick.Config{MaxCountScale: 10} - if testing.Short() { - quickCfg = nil - } - - f := func(m1 IndexMessage) bool { - for i, f := range m1.Files { - m1.Files[i].CachedSize = 0 - for j := range f.Blocks { - f.Blocks[j].Offset = 0 - if len(f.Blocks[j].Hash) == 0 { - f.Blocks[j].Hash = nil - } - } - } - - return testMarshal(t, "index", &m1, &IndexMessage{}) - } - - if err := quick.Check(f, quickCfg); err != nil { - t.Error(err) - } -} - -func TestMarshalRequestMessage(t *testing.T) { - var quickCfg = &quick.Config{MaxCountScale: 10} - if testing.Short() { - quickCfg = nil - } - - f := func(m1 RequestMessage) bool { - return testMarshal(t, "request", &m1, &RequestMessage{}) - } - - if err := quick.Check(f, quickCfg); err != nil { - t.Error(err) - } -} - -func TestMarshalResponseMessage(t *testing.T) { - var quickCfg = &quick.Config{MaxCountScale: 10} - if testing.Short() { - quickCfg = nil - } - - f := func(m1 ResponseMessage) bool { - if len(m1.Data) == 0 { - m1.Data = nil - } - return testMarshal(t, "response", &m1, &ResponseMessage{}) - } - - if err := quick.Check(f, quickCfg); err != nil { - t.Error(err) - } -} - -func TestMarshalClusterConfigMessage(t *testing.T) { - var quickCfg = &quick.Config{MaxCountScale: 10} - if testing.Short() { - quickCfg = nil - } - - f := func(m1 ClusterConfigMessage) bool { - return testMarshal(t, "clusterconfig", &m1, &ClusterConfigMessage{}) - } - - if err := quick.Check(f, quickCfg); err != nil { - t.Error(err) - } -} - -func TestMarshalCloseMessage(t *testing.T) { - var quickCfg = &quick.Config{MaxCountScale: 10} - if testing.Short() { - quickCfg = nil - } - - f := func(m1 CloseMessage) bool { - return testMarshal(t, "close", &m1, &CloseMessage{}) - } - - if err := quick.Check(f, quickCfg); err != nil { - t.Error(err) - } -} - -type message interface { - EncodeXDR(io.Writer) (int, error) - DecodeXDR(io.Reader) error -} - -func testMarshal(t *testing.T, prefix string, m1, m2 message) bool { - var buf bytes.Buffer - - failed := func(bc []byte) { - bs, _ := json.MarshalIndent(m1, "", " ") - ioutil.WriteFile(prefix+"-1.txt", bs, 0644) - bs, _ = json.MarshalIndent(m2, "", " ") - ioutil.WriteFile(prefix+"-2.txt", bs, 0644) - if len(bc) > 0 { - f, _ := os.Create(prefix + "-data.txt") - fmt.Fprint(f, hex.Dump(bc)) - f.Close() - } - } - - _, err := m1.EncodeXDR(&buf) - if err != nil && strings.Contains(err.Error(), "exceeds size") { - return true - } - if err != nil { - failed(nil) - t.Fatal(err) - } - - bc := make([]byte, len(buf.Bytes())) - copy(bc, buf.Bytes()) - - err = m2.DecodeXDR(&buf) - if err != nil { - failed(bc) - t.Fatal(err) - } - - ok := reflect.DeepEqual(m1, m2) - if !ok { - failed(bc) - } - return ok -} diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/vector.go b/Godeps/_workspace/src/github.com/syncthing/protocol/vector.go deleted file mode 100644 index edd156143..000000000 --- a/Godeps/_workspace/src/github.com/syncthing/protocol/vector.go +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright (C) 2015 The Protocol Authors. - -package protocol - -// The Vector type represents a version vector. The zero value is a usable -// version vector. The vector has slice semantics and some operations on it -// are "append-like" in that they may return the same vector modified, or a -// new allocated Vector with the modified contents. -type Vector []Counter - -// Counter represents a single counter in the version vector. -type Counter struct { - ID uint64 - Value uint64 -} - -// Update returns a Vector with the index for the specific ID incremented by -// one. If it is possible, the vector v is updated and returned. If it is not, -// a copy will be created, updated and returned. -func (v Vector) Update(ID uint64) Vector { - for i := range v { - if v[i].ID == ID { - // Update an existing index - v[i].Value++ - return v - } else if v[i].ID > ID { - // Insert a new index - nv := make(Vector, len(v)+1) - copy(nv, v[:i]) - nv[i].ID = ID - nv[i].Value = 1 - copy(nv[i+1:], v[i:]) - return nv - } - } - // Append a new new index - return append(v, Counter{ID, 1}) -} - -// Merge returns the vector containing the maximum indexes from a and b. If it -// is possible, the vector a is updated and returned. If it is not, a copy -// will be created, updated and returned. -func (a Vector) Merge(b Vector) Vector { - var ai, bi int - for bi < len(b) { - if ai == len(a) { - // We've reach the end of a, all that remains are appends - return append(a, b[bi:]...) - } - - if a[ai].ID > b[bi].ID { - // The index from b should be inserted here - n := make(Vector, len(a)+1) - copy(n, a[:ai]) - n[ai] = b[bi] - copy(n[ai+1:], a[ai:]) - a = n - } - - if a[ai].ID == b[bi].ID { - if v := b[bi].Value; v > a[ai].Value { - a[ai].Value = v - } - } - - if bi < len(b) && a[ai].ID == b[bi].ID { - bi++ - } - ai++ - } - - return a -} - -// Copy returns an identical vector that is not shared with v. -func (v Vector) Copy() Vector { - nv := make(Vector, len(v)) - copy(nv, v) - return nv -} - -// Equal returns true when the two vectors are equivalent. -func (a Vector) Equal(b Vector) bool { - return a.Compare(b) == Equal -} - -// LesserEqual returns true when the two vectors are equivalent or a is Lesser -// than b. -func (a Vector) LesserEqual(b Vector) bool { - comp := a.Compare(b) - return comp == Lesser || comp == Equal -} - -// LesserEqual returns true when the two vectors are equivalent or a is Greater -// than b. -func (a Vector) GreaterEqual(b Vector) bool { - comp := a.Compare(b) - return comp == Greater || comp == Equal -} - -// Concurrent returns true when the two vectors are concrurrent. -func (a Vector) Concurrent(b Vector) bool { - comp := a.Compare(b) - return comp == ConcurrentGreater || comp == ConcurrentLesser -} - -// Counter returns the current value of the given counter ID. -func (v Vector) Counter(id uint64) uint64 { - for _, c := range v { - if c.ID == id { - return c.Value - } - } - return 0 -} diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/vector_compare.go b/Godeps/_workspace/src/github.com/syncthing/protocol/vector_compare.go deleted file mode 100644 index 9735ec9d1..000000000 --- a/Godeps/_workspace/src/github.com/syncthing/protocol/vector_compare.go +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright (C) 2015 The Protocol Authors. - -package protocol - -// Ordering represents the relationship between two Vectors. -type Ordering int - -const ( - Equal Ordering = iota - Greater - Lesser - ConcurrentLesser - ConcurrentGreater -) - -// There's really no such thing as "concurrent lesser" and "concurrent -// greater" in version vectors, just "concurrent". But it's useful to be able -// to get a strict ordering between versions for stable sorts and so on, so we -// return both variants. The convenience method Concurrent() can be used to -// check for either case. - -// Compare returns the Ordering that describes a's relation to b. -func (a Vector) Compare(b Vector) Ordering { - var ai, bi int // index into a and b - var av, bv Counter // value at current index - - result := Equal - - for ai < len(a) || bi < len(b) { - var aMissing, bMissing bool - - if ai < len(a) { - av = a[ai] - } else { - av = Counter{} - aMissing = true - } - - if bi < len(b) { - bv = b[bi] - } else { - bv = Counter{} - bMissing = true - } - - switch { - case av.ID == bv.ID: - // We have a counter value for each side - if av.Value > bv.Value { - if result == Lesser { - return ConcurrentLesser - } - result = Greater - } else if av.Value < bv.Value { - if result == Greater { - return ConcurrentGreater - } - result = Lesser - } - - case !aMissing && av.ID < bv.ID || bMissing: - // Value is missing on the b side - if av.Value > 0 { - if result == Lesser { - return ConcurrentLesser - } - result = Greater - } - - case !bMissing && bv.ID < av.ID || aMissing: - // Value is missing on the a side - if bv.Value > 0 { - if result == Greater { - return ConcurrentGreater - } - result = Lesser - } - } - - if ai < len(a) && (av.ID <= bv.ID || bMissing) { - ai++ - } - if bi < len(b) && (bv.ID <= av.ID || aMissing) { - bi++ - } - } - - return result -} diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/vector_compare_test.go b/Godeps/_workspace/src/github.com/syncthing/protocol/vector_compare_test.go deleted file mode 100644 index 78b6abe43..000000000 --- a/Godeps/_workspace/src/github.com/syncthing/protocol/vector_compare_test.go +++ /dev/null @@ -1,249 +0,0 @@ -// Copyright (C) 2015 The Protocol Authors. - -package protocol - -import ( - "math" - "testing" -) - -func TestCompare(t *testing.T) { - testcases := []struct { - a, b Vector - r Ordering - }{ - // Empty vectors are identical - {Vector{}, Vector{}, Equal}, - {Vector{}, nil, Equal}, - {nil, Vector{}, Equal}, - {nil, Vector{Counter{42, 0}}, Equal}, - {Vector{}, Vector{Counter{42, 0}}, Equal}, - {Vector{Counter{42, 0}}, nil, Equal}, - {Vector{Counter{42, 0}}, Vector{}, Equal}, - - // Zero is the implied value for a missing Counter - { - Vector{Counter{42, 0}}, - Vector{Counter{77, 0}}, - Equal, - }, - - // Equal vectors are equal - { - Vector{Counter{42, 33}}, - Vector{Counter{42, 33}}, - Equal, - }, - { - Vector{Counter{42, 33}, Counter{77, 24}}, - Vector{Counter{42, 33}, Counter{77, 24}}, - Equal, - }, - - // These a-vectors are all greater than the b-vector - { - Vector{Counter{42, 1}}, - nil, - Greater, - }, - { - Vector{Counter{42, 1}}, - Vector{}, - Greater, - }, - { - Vector{Counter{0, 1}}, - Vector{Counter{0, 0}}, - Greater, - }, - { - Vector{Counter{42, 1}}, - Vector{Counter{42, 0}}, - Greater, - }, - { - Vector{Counter{math.MaxUint64, 1}}, - Vector{Counter{math.MaxUint64, 0}}, - Greater, - }, - { - Vector{Counter{0, math.MaxUint64}}, - Vector{Counter{0, 0}}, - Greater, - }, - { - Vector{Counter{42, math.MaxUint64}}, - Vector{Counter{42, 0}}, - Greater, - }, - { - Vector{Counter{math.MaxUint64, math.MaxUint64}}, - Vector{Counter{math.MaxUint64, 0}}, - Greater, - }, - { - Vector{Counter{0, math.MaxUint64}}, - Vector{Counter{0, math.MaxUint64 - 1}}, - Greater, - }, - { - Vector{Counter{42, math.MaxUint64}}, - Vector{Counter{42, math.MaxUint64 - 1}}, - Greater, - }, - { - Vector{Counter{math.MaxUint64, math.MaxUint64}}, - Vector{Counter{math.MaxUint64, math.MaxUint64 - 1}}, - Greater, - }, - { - Vector{Counter{42, 2}}, - Vector{Counter{42, 1}}, - Greater, - }, - { - Vector{Counter{22, 22}, Counter{42, 2}}, - Vector{Counter{22, 22}, Counter{42, 1}}, - Greater, - }, - { - Vector{Counter{42, 2}, Counter{77, 3}}, - Vector{Counter{42, 1}, Counter{77, 3}}, - Greater, - }, - { - Vector{Counter{22, 22}, Counter{42, 2}, Counter{77, 3}}, - Vector{Counter{22, 22}, Counter{42, 1}, Counter{77, 3}}, - Greater, - }, - { - Vector{Counter{22, 23}, Counter{42, 2}, Counter{77, 4}}, - Vector{Counter{22, 22}, Counter{42, 1}, Counter{77, 3}}, - Greater, - }, - - // These a-vectors are all lesser than the b-vector - {nil, Vector{Counter{42, 1}}, Lesser}, - {Vector{}, Vector{Counter{42, 1}}, Lesser}, - { - Vector{Counter{42, 0}}, - Vector{Counter{42, 1}}, - Lesser, - }, - { - Vector{Counter{42, 1}}, - Vector{Counter{42, 2}}, - Lesser, - }, - { - Vector{Counter{22, 22}, Counter{42, 1}}, - Vector{Counter{22, 22}, Counter{42, 2}}, - Lesser, - }, - { - Vector{Counter{42, 1}, Counter{77, 3}}, - Vector{Counter{42, 2}, Counter{77, 3}}, - Lesser, - }, - { - Vector{Counter{22, 22}, Counter{42, 1}, Counter{77, 3}}, - Vector{Counter{22, 22}, Counter{42, 2}, Counter{77, 3}}, - Lesser, - }, - { - Vector{Counter{22, 22}, Counter{42, 1}, Counter{77, 3}}, - Vector{Counter{22, 23}, Counter{42, 2}, Counter{77, 4}}, - Lesser, - }, - - // These are all in conflict - { - Vector{Counter{42, 2}}, - Vector{Counter{43, 1}}, - ConcurrentGreater, - }, - { - Vector{Counter{43, 1}}, - Vector{Counter{42, 2}}, - ConcurrentLesser, - }, - { - Vector{Counter{22, 23}, Counter{42, 1}}, - Vector{Counter{22, 22}, Counter{42, 2}}, - ConcurrentGreater, - }, - { - Vector{Counter{22, 21}, Counter{42, 2}}, - Vector{Counter{22, 22}, Counter{42, 1}}, - ConcurrentLesser, - }, - { - Vector{Counter{22, 21}, Counter{42, 2}, Counter{43, 1}}, - Vector{Counter{20, 1}, Counter{22, 22}, Counter{42, 1}}, - ConcurrentLesser, - }, - } - - for i, tc := range testcases { - // Test real Compare - if r := tc.a.Compare(tc.b); r != tc.r { - t.Errorf("%d: %+v.Compare(%+v) == %v (expected %v)", i, tc.a, tc.b, r, tc.r) - } - - // Test convenience functions - switch tc.r { - case Greater: - if tc.a.Equal(tc.b) { - t.Errorf("%+v == %+v", tc.a, tc.b) - } - if tc.a.Concurrent(tc.b) { - t.Errorf("%+v concurrent %+v", tc.a, tc.b) - } - if !tc.a.GreaterEqual(tc.b) { - t.Errorf("%+v not >= %+v", tc.a, tc.b) - } - if tc.a.LesserEqual(tc.b) { - t.Errorf("%+v <= %+v", tc.a, tc.b) - } - case Lesser: - if tc.a.Concurrent(tc.b) { - t.Errorf("%+v concurrent %+v", tc.a, tc.b) - } - if tc.a.Equal(tc.b) { - t.Errorf("%+v == %+v", tc.a, tc.b) - } - if tc.a.GreaterEqual(tc.b) { - t.Errorf("%+v >= %+v", tc.a, tc.b) - } - if !tc.a.LesserEqual(tc.b) { - t.Errorf("%+v not <= %+v", tc.a, tc.b) - } - case Equal: - if tc.a.Concurrent(tc.b) { - t.Errorf("%+v concurrent %+v", tc.a, tc.b) - } - if !tc.a.Equal(tc.b) { - t.Errorf("%+v not == %+v", tc.a, tc.b) - } - if !tc.a.GreaterEqual(tc.b) { - t.Errorf("%+v not <= %+v", tc.a, tc.b) - } - if !tc.a.LesserEqual(tc.b) { - t.Errorf("%+v not <= %+v", tc.a, tc.b) - } - case ConcurrentLesser, ConcurrentGreater: - if !tc.a.Concurrent(tc.b) { - t.Errorf("%+v not concurrent %+v", tc.a, tc.b) - } - if tc.a.Equal(tc.b) { - t.Errorf("%+v == %+v", tc.a, tc.b) - } - if tc.a.GreaterEqual(tc.b) { - t.Errorf("%+v >= %+v", tc.a, tc.b) - } - if tc.a.LesserEqual(tc.b) { - t.Errorf("%+v <= %+v", tc.a, tc.b) - } - } - } -} diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/vector_test.go b/Godeps/_workspace/src/github.com/syncthing/protocol/vector_test.go deleted file mode 100644 index c01255e7a..000000000 --- a/Godeps/_workspace/src/github.com/syncthing/protocol/vector_test.go +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright (C) 2015 The Protocol Authors. - -package protocol - -import "testing" - -func TestUpdate(t *testing.T) { - var v Vector - - // Append - - v = v.Update(42) - expected := Vector{Counter{42, 1}} - - if v.Compare(expected) != Equal { - t.Errorf("Update error, %+v != %+v", v, expected) - } - - // Insert at front - - v = v.Update(36) - expected = Vector{Counter{36, 1}, Counter{42, 1}} - - if v.Compare(expected) != Equal { - t.Errorf("Update error, %+v != %+v", v, expected) - } - - // Insert in moddle - - v = v.Update(37) - expected = Vector{Counter{36, 1}, Counter{37, 1}, Counter{42, 1}} - - if v.Compare(expected) != Equal { - t.Errorf("Update error, %+v != %+v", v, expected) - } - - // Update existing - - v = v.Update(37) - expected = Vector{Counter{36, 1}, Counter{37, 2}, Counter{42, 1}} - - if v.Compare(expected) != Equal { - t.Errorf("Update error, %+v != %+v", v, expected) - } -} - -func TestCopy(t *testing.T) { - v0 := Vector{Counter{42, 1}} - v1 := v0.Copy() - v1.Update(42) - if v0.Compare(v1) != Lesser { - t.Errorf("Copy error, %+v should be ancestor of %+v", v0, v1) - } -} - -func TestMerge(t *testing.T) { - testcases := []struct { - a, b, m Vector - }{ - // No-ops - { - Vector{}, - Vector{}, - Vector{}, - }, - { - Vector{Counter{22, 1}, Counter{42, 1}}, - Vector{Counter{22, 1}, Counter{42, 1}}, - Vector{Counter{22, 1}, Counter{42, 1}}, - }, - - // Appends - { - Vector{}, - Vector{Counter{22, 1}, Counter{42, 1}}, - Vector{Counter{22, 1}, Counter{42, 1}}, - }, - { - Vector{Counter{22, 1}}, - Vector{Counter{42, 1}}, - Vector{Counter{22, 1}, Counter{42, 1}}, - }, - { - Vector{Counter{22, 1}}, - Vector{Counter{22, 1}, Counter{42, 1}}, - Vector{Counter{22, 1}, Counter{42, 1}}, - }, - - // Insert - { - Vector{Counter{22, 1}, Counter{42, 1}}, - Vector{Counter{22, 1}, Counter{23, 2}, Counter{42, 1}}, - Vector{Counter{22, 1}, Counter{23, 2}, Counter{42, 1}}, - }, - { - Vector{Counter{42, 1}}, - Vector{Counter{22, 1}}, - Vector{Counter{22, 1}, Counter{42, 1}}, - }, - - // Update - { - Vector{Counter{22, 1}, Counter{42, 2}}, - Vector{Counter{22, 2}, Counter{42, 1}}, - Vector{Counter{22, 2}, Counter{42, 2}}, - }, - - // All of the above - { - Vector{Counter{10, 1}, Counter{20, 2}, Counter{30, 1}}, - Vector{Counter{5, 1}, Counter{10, 2}, Counter{15, 1}, Counter{20, 1}, Counter{25, 1}, Counter{35, 1}}, - Vector{Counter{5, 1}, Counter{10, 2}, Counter{15, 1}, Counter{20, 2}, Counter{25, 1}, Counter{30, 1}, Counter{35, 1}}, - }, - } - - for i, tc := range testcases { - if m := tc.a.Merge(tc.b); m.Compare(tc.m) != Equal { - t.Errorf("%d: %+v.Merge(%+v) == %+v (expected %+v)", i, tc.a, tc.b, m, tc.m) - } - } -} - -func TestCounterValue(t *testing.T) { - v0 := Vector{Counter{42, 1}, Counter{64, 5}} - if v0.Counter(42) != 1 { - t.Error("Counter error, %d != %d", v0.Counter(42), 1) - } - if v0.Counter(64) != 5 { - t.Error("Counter error, %d != %d", v0.Counter(64), 5) - } - if v0.Counter(72) != 0 { - t.Error("Counter error, %d != %d", v0.Counter(72), 0) - } -} diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/vector_xdr.go b/Godeps/_workspace/src/github.com/syncthing/protocol/vector_xdr.go deleted file mode 100644 index 01efa7e4e..000000000 --- a/Godeps/_workspace/src/github.com/syncthing/protocol/vector_xdr.go +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright (C) 2015 The Protocol Authors. - -package protocol - -import "github.com/calmh/xdr" - -// This stuff is hacked up manually because genxdr doesn't support 'type -// Vector []Counter' declarations and it was tricky when I tried to add it... - -type xdrWriter interface { - WriteUint32(uint32) (int, error) - WriteUint64(uint64) (int, error) -} -type xdrReader interface { - ReadUint32() uint32 - ReadUint64() uint64 -} - -// EncodeXDRInto encodes the vector as an XDR object into the given XDR -// encoder. -func (v Vector) EncodeXDRInto(w xdrWriter) (int, error) { - w.WriteUint32(uint32(len(v))) - for i := range v { - w.WriteUint64(v[i].ID) - w.WriteUint64(v[i].Value) - } - return 4 + 16*len(v), nil -} - -// DecodeXDRFrom decodes the XDR objects from the given reader into itself. -func (v *Vector) DecodeXDRFrom(r xdrReader) error { - l := int(r.ReadUint32()) - if l > 1e6 { - return xdr.ElementSizeExceeded("number of counters", l, 1e6) - } - n := make(Vector, l) - for i := range n { - n[i].ID = r.ReadUint64() - n[i].Value = r.ReadUint64() - } - *v = n - return nil -} diff --git a/Godeps/_workspace/src/github.com/syncthing/protocol/wireformat.go b/Godeps/_workspace/src/github.com/syncthing/protocol/wireformat.go deleted file mode 100644 index 66b02ed6f..000000000 --- a/Godeps/_workspace/src/github.com/syncthing/protocol/wireformat.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright (C) 2014 The Protocol Authors. - -package protocol - -import ( - "path/filepath" - - "golang.org/x/text/unicode/norm" -) - -type wireFormatConnection struct { - next Connection -} - -func (c wireFormatConnection) Start() { - c.next.Start() -} - -func (c wireFormatConnection) ID() DeviceID { - return c.next.ID() -} - -func (c wireFormatConnection) Name() string { - return c.next.Name() -} - -func (c wireFormatConnection) Index(folder string, fs []FileInfo, flags uint32, options []Option) error { - var myFs = make([]FileInfo, len(fs)) - copy(myFs, fs) - - for i := range fs { - myFs[i].Name = norm.NFC.String(filepath.ToSlash(myFs[i].Name)) - } - - return c.next.Index(folder, myFs, flags, options) -} - -func (c wireFormatConnection) IndexUpdate(folder string, fs []FileInfo, flags uint32, options []Option) error { - var myFs = make([]FileInfo, len(fs)) - copy(myFs, fs) - - for i := range fs { - myFs[i].Name = norm.NFC.String(filepath.ToSlash(myFs[i].Name)) - } - - return c.next.IndexUpdate(folder, myFs, flags, options) -} - -func (c wireFormatConnection) Request(folder, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) { - name = norm.NFC.String(filepath.ToSlash(name)) - return c.next.Request(folder, name, offset, size, hash, flags, options) -} - -func (c wireFormatConnection) ClusterConfig(config ClusterConfigMessage) { - c.next.ClusterConfig(config) -} - -func (c wireFormatConnection) Statistics() Statistics { - return c.next.Statistics() -} diff --git a/Godeps/_workspace/src/github.com/syncthing/relaysrv/client/client.go b/Godeps/_workspace/src/github.com/syncthing/relaysrv/client/client.go deleted file mode 100644 index 2e1b51e03..000000000 --- a/Godeps/_workspace/src/github.com/syncthing/relaysrv/client/client.go +++ /dev/null @@ -1,298 +0,0 @@ -// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file). - -package client - -import ( - "crypto/tls" - "fmt" - "log" - "net" - "net/url" - "time" - - syncthingprotocol "github.com/syncthing/syncthing/lib/protocol" - "github.com/syncthing/syncthing/lib/relay/protocol" - "github.com/syncthing/syncthing/lib/sync" -) - -type ProtocolClient struct { - URI *url.URL - Invitations chan protocol.SessionInvitation - - closeInvitationsOnFinish bool - - config *tls.Config - - timeout time.Duration - - stop chan struct{} - stopped chan struct{} - - conn *tls.Conn - - mut sync.RWMutex - connected bool - latency time.Duration -} - -func NewProtocolClient(uri *url.URL, certs []tls.Certificate, invitations chan protocol.SessionInvitation) *ProtocolClient { - closeInvitationsOnFinish := false - if invitations == nil { - closeInvitationsOnFinish = true - invitations = make(chan protocol.SessionInvitation) - } - - return &ProtocolClient{ - URI: uri, - Invitations: invitations, - - closeInvitationsOnFinish: closeInvitationsOnFinish, - - config: configForCerts(certs), - - timeout: time.Minute * 2, - - stop: make(chan struct{}), - stopped: make(chan struct{}), - - mut: sync.NewRWMutex(), - connected: false, - } -} - -func (c *ProtocolClient) Serve() { - c.stop = make(chan struct{}) - c.stopped = make(chan struct{}) - defer close(c.stopped) - - if err := c.connect(); err != nil { - if debug { - l.Debugln("Relay connect:", err) - } - return - } - - if debug { - l.Debugln(c, "connected", c.conn.RemoteAddr()) - } - - if err := c.join(); err != nil { - c.conn.Close() - l.Infoln("Relay join:", err) - return - } - - if err := c.conn.SetDeadline(time.Time{}); err != nil { - l.Infoln("Relay set deadline:", err) - return - } - - if debug { - l.Debugln(c, "joined", c.conn.RemoteAddr(), "via", c.conn.LocalAddr()) - } - - defer c.cleanup() - c.mut.Lock() - c.connected = true - c.mut.Unlock() - - messages := make(chan interface{}) - errors := make(chan error, 1) - - go messageReader(c.conn, messages, errors) - - timeout := time.NewTimer(c.timeout) - - for { - select { - case message := <-messages: - timeout.Reset(c.timeout) - if debug { - log.Printf("%s received message %T", c, message) - } - - switch msg := message.(type) { - case protocol.Ping: - if err := protocol.WriteMessage(c.conn, protocol.Pong{}); err != nil { - l.Infoln("Relay write:", err) - return - - } - if debug { - l.Debugln(c, "sent pong") - } - - case protocol.SessionInvitation: - ip := net.IP(msg.Address) - if len(ip) == 0 || ip.IsUnspecified() { - msg.Address = c.conn.RemoteAddr().(*net.TCPAddr).IP[:] - } - c.Invitations <- msg - - default: - l.Infoln("Relay: protocol error: unexpected message %v", msg) - return - } - - case <-c.stop: - if debug { - l.Debugln(c, "stopping") - } - return - - case err := <-errors: - l.Infoln("Relay received:", err) - return - - case <-timeout.C: - if debug { - l.Debugln(c, "timed out") - } - return - } - } -} - -func (c *ProtocolClient) Stop() { - if c.stop == nil { - return - } - - close(c.stop) - <-c.stopped -} - -func (c *ProtocolClient) StatusOK() bool { - c.mut.RLock() - con := c.connected - c.mut.RUnlock() - return con -} - -func (c *ProtocolClient) Latency() time.Duration { - c.mut.RLock() - lat := c.latency - c.mut.RUnlock() - return lat -} - -func (c *ProtocolClient) String() string { - return fmt.Sprintf("ProtocolClient@%p", c) -} - -func (c *ProtocolClient) connect() error { - if c.URI.Scheme != "relay" { - return fmt.Errorf("Unsupported relay schema:", c.URI.Scheme) - } - - t0 := time.Now() - tcpConn, err := net.Dial("tcp", c.URI.Host) - if err != nil { - return err - } - - c.mut.Lock() - c.latency = time.Since(t0) - c.mut.Unlock() - - conn := tls.Client(tcpConn, c.config) - if err = conn.Handshake(); err != nil { - return err - } - - if err := conn.SetDeadline(time.Now().Add(10 * time.Second)); err != nil { - conn.Close() - return err - } - - if err := performHandshakeAndValidation(conn, c.URI); err != nil { - conn.Close() - return err - } - - c.conn = conn - return nil -} - -func (c *ProtocolClient) cleanup() { - if c.closeInvitationsOnFinish { - close(c.Invitations) - c.Invitations = make(chan protocol.SessionInvitation) - } - - if debug { - l.Debugln(c, "cleaning up") - } - - c.mut.Lock() - c.connected = false - c.mut.Unlock() - - c.conn.Close() -} - -func (c *ProtocolClient) join() error { - if err := protocol.WriteMessage(c.conn, protocol.JoinRelayRequest{}); err != nil { - return err - } - - message, err := protocol.ReadMessage(c.conn) - if err != nil { - return err - } - - switch msg := message.(type) { - case protocol.Response: - if msg.Code != 0 { - return fmt.Errorf("Incorrect response code %d: %s", msg.Code, msg.Message) - } - - default: - return fmt.Errorf("protocol error: expecting response got %v", msg) - } - - return nil -} - -func performHandshakeAndValidation(conn *tls.Conn, uri *url.URL) error { - if err := conn.Handshake(); err != nil { - return err - } - - cs := conn.ConnectionState() - if !cs.NegotiatedProtocolIsMutual || cs.NegotiatedProtocol != protocol.ProtocolName { - return fmt.Errorf("protocol negotiation error") - } - - q := uri.Query() - relayIDs := q.Get("id") - if relayIDs != "" { - relayID, err := syncthingprotocol.DeviceIDFromString(relayIDs) - if err != nil { - return fmt.Errorf("relay address contains invalid verification id: %s", err) - } - - certs := cs.PeerCertificates - if cl := len(certs); cl != 1 { - return fmt.Errorf("unexpected certificate count: %d", cl) - } - - remoteID := syncthingprotocol.NewDeviceID(certs[0].Raw) - if remoteID != relayID { - return fmt.Errorf("relay id does not match. Expected %v got %v", relayID, remoteID) - } - } - - return nil -} - -func messageReader(conn net.Conn, messages chan<- interface{}, errors chan<- error) { - for { - msg, err := protocol.ReadMessage(conn) - if err != nil { - errors <- err - return - } - messages <- msg - } -} diff --git a/Godeps/_workspace/src/github.com/syncthing/relaysrv/client/debug.go b/Godeps/_workspace/src/github.com/syncthing/relaysrv/client/debug.go deleted file mode 100644 index 935e9fe62..000000000 --- a/Godeps/_workspace/src/github.com/syncthing/relaysrv/client/debug.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file). - -package client - -import ( - "os" - "strings" - - "github.com/calmh/logger" -) - -var ( - debug = strings.Contains(os.Getenv("STTRACE"), "relay") || os.Getenv("STTRACE") == "all" - l = logger.DefaultLogger -) diff --git a/Godeps/_workspace/src/github.com/syncthing/relaysrv/client/methods.go b/Godeps/_workspace/src/github.com/syncthing/relaysrv/client/methods.go deleted file mode 100644 index ced788b8a..000000000 --- a/Godeps/_workspace/src/github.com/syncthing/relaysrv/client/methods.go +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file). - -package client - -import ( - "crypto/tls" - "fmt" - "net" - "net/url" - "strconv" - "strings" - "time" - - syncthingprotocol "github.com/syncthing/syncthing/lib/protocol" - "github.com/syncthing/syncthing/lib/relay/protocol" -) - -func GetInvitationFromRelay(uri *url.URL, id syncthingprotocol.DeviceID, certs []tls.Certificate) (protocol.SessionInvitation, error) { - if uri.Scheme != "relay" { - return protocol.SessionInvitation{}, fmt.Errorf("Unsupported relay scheme:", uri.Scheme) - } - - conn, err := tls.Dial("tcp", uri.Host, configForCerts(certs)) - if err != nil { - return protocol.SessionInvitation{}, err - } - conn.SetDeadline(time.Now().Add(10 * time.Second)) - - if err := performHandshakeAndValidation(conn, uri); err != nil { - return protocol.SessionInvitation{}, err - } - - defer conn.Close() - - request := protocol.ConnectRequest{ - ID: id[:], - } - - if err := protocol.WriteMessage(conn, request); err != nil { - return protocol.SessionInvitation{}, err - } - - message, err := protocol.ReadMessage(conn) - if err != nil { - return protocol.SessionInvitation{}, err - } - - switch msg := message.(type) { - case protocol.Response: - return protocol.SessionInvitation{}, fmt.Errorf("Incorrect response code %d: %s", msg.Code, msg.Message) - case protocol.SessionInvitation: - if debug { - l.Debugln("Received invitation", msg, "via", conn.LocalAddr()) - } - ip := net.IP(msg.Address) - if len(ip) == 0 || ip.IsUnspecified() { - msg.Address = conn.RemoteAddr().(*net.TCPAddr).IP[:] - } - return msg, nil - default: - return protocol.SessionInvitation{}, fmt.Errorf("protocol error: unexpected message %v", msg) - } -} - -func JoinSession(invitation protocol.SessionInvitation) (net.Conn, error) { - addr := net.JoinHostPort(net.IP(invitation.Address).String(), strconv.Itoa(int(invitation.Port))) - - conn, err := net.Dial("tcp", addr) - if err != nil { - return nil, err - } - - request := protocol.JoinSessionRequest{ - Key: invitation.Key, - } - - conn.SetDeadline(time.Now().Add(10 * time.Second)) - err = protocol.WriteMessage(conn, request) - if err != nil { - return nil, err - } - - message, err := protocol.ReadMessage(conn) - if err != nil { - return nil, err - } - - conn.SetDeadline(time.Time{}) - - switch msg := message.(type) { - case protocol.Response: - if msg.Code != 0 { - return nil, fmt.Errorf("Incorrect response code %d: %s", msg.Code, msg.Message) - } - return conn, nil - default: - return nil, fmt.Errorf("protocol error: expecting response got %v", msg) - } -} - -func TestRelay(uri *url.URL, certs []tls.Certificate, sleep time.Duration, times int) bool { - id := syncthingprotocol.NewDeviceID(certs[0].Certificate[0]) - invs := make(chan protocol.SessionInvitation, 1) - c := NewProtocolClient(uri, certs, invs) - go c.Serve() - defer func() { - close(invs) - c.Stop() - }() - - for i := 0; i < times; i++ { - _, err := GetInvitationFromRelay(uri, id, certs) - if err == nil { - return true - } - if !strings.Contains(err.Error(), "Incorrect response code") { - return false - } - time.Sleep(sleep) - } - return false -} - -func configForCerts(certs []tls.Certificate) *tls.Config { - return &tls.Config{ - Certificates: certs, - NextProtos: []string{protocol.ProtocolName}, - ClientAuth: tls.RequestClientCert, - SessionTicketsDisabled: true, - InsecureSkipVerify: true, - MinVersion: tls.VersionTLS12, - CipherSuites: []uint16{ - tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, - tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, - tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, - tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, - tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, - tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, - }, - } -} diff --git a/Godeps/_workspace/src/github.com/syncthing/relaysrv/protocol/packets.go b/Godeps/_workspace/src/github.com/syncthing/relaysrv/protocol/packets.go deleted file mode 100644 index f11954caa..000000000 --- a/Godeps/_workspace/src/github.com/syncthing/relaysrv/protocol/packets.go +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file). - -//go:generate -command genxdr go run ../../syncthing/Godeps/_workspace/src/github.com/calmh/xdr/cmd/genxdr/main.go -//go:generate genxdr -o packets_xdr.go packets.go - -package protocol - -import ( - "fmt" - "net" - - syncthingprotocol "github.com/syncthing/syncthing/lib/protocol" -) - -const ( - messageTypePing int32 = iota - messageTypePong - messageTypeJoinRelayRequest - messageTypeJoinSessionRequest - messageTypeResponse - messageTypeConnectRequest - messageTypeSessionInvitation -) - -type header struct { - magic uint32 - messageType int32 - messageLength int32 -} - -type Ping struct{} -type Pong struct{} -type JoinRelayRequest struct{} - -type JoinSessionRequest struct { - Key []byte // max:32 -} - -type Response struct { - Code int32 - Message string -} - -type ConnectRequest struct { - ID []byte // max:32 -} - -type SessionInvitation struct { - From []byte // max:32 - Key []byte // max:32 - Address []byte // max:32 - Port uint16 - ServerSocket bool -} - -func (i SessionInvitation) String() string { - return fmt.Sprintf("%s@%s", syncthingprotocol.DeviceIDFromBytes(i.From), i.AddressString()) -} - -func (i SessionInvitation) GoString() string { - return i.String() -} - -func (i SessionInvitation) AddressString() string { - return fmt.Sprintf("%s:%d", net.IP(i.Address), i.Port) -} diff --git a/Godeps/_workspace/src/github.com/syncthing/relaysrv/protocol/packets_xdr.go b/Godeps/_workspace/src/github.com/syncthing/relaysrv/protocol/packets_xdr.go deleted file mode 100644 index f18e18c18..000000000 --- a/Godeps/_workspace/src/github.com/syncthing/relaysrv/protocol/packets_xdr.go +++ /dev/null @@ -1,567 +0,0 @@ -// ************************************************************ -// This file is automatically generated by genxdr. Do not edit. -// ************************************************************ - -package protocol - -import ( - "bytes" - "io" - - "github.com/calmh/xdr" -) - -/* - -header Structure: - - 0 1 2 3 - 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| magic | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| message Type | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| message Length | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - - -struct header { - unsigned int magic; - int messageType; - int messageLength; -} - -*/ - -func (o header) EncodeXDR(w io.Writer) (int, error) { - var xw = xdr.NewWriter(w) - return o.EncodeXDRInto(xw) -} - -func (o header) MarshalXDR() ([]byte, error) { - return o.AppendXDR(make([]byte, 0, 128)) -} - -func (o header) MustMarshalXDR() []byte { - bs, err := o.MarshalXDR() - if err != nil { - panic(err) - } - return bs -} - -func (o header) AppendXDR(bs []byte) ([]byte, error) { - var aw = xdr.AppendWriter(bs) - var xw = xdr.NewWriter(&aw) - _, err := o.EncodeXDRInto(xw) - return []byte(aw), err -} - -func (o header) EncodeXDRInto(xw *xdr.Writer) (int, error) { - xw.WriteUint32(o.magic) - xw.WriteUint32(uint32(o.messageType)) - xw.WriteUint32(uint32(o.messageLength)) - return xw.Tot(), xw.Error() -} - -func (o *header) DecodeXDR(r io.Reader) error { - xr := xdr.NewReader(r) - return o.DecodeXDRFrom(xr) -} - -func (o *header) UnmarshalXDR(bs []byte) error { - var br = bytes.NewReader(bs) - var xr = xdr.NewReader(br) - return o.DecodeXDRFrom(xr) -} - -func (o *header) DecodeXDRFrom(xr *xdr.Reader) error { - o.magic = xr.ReadUint32() - o.messageType = int32(xr.ReadUint32()) - o.messageLength = int32(xr.ReadUint32()) - return xr.Error() -} - -/* - -Ping Structure: - - 0 1 2 3 - 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - - -struct Ping { -} - -*/ - -func (o Ping) EncodeXDR(w io.Writer) (int, error) { - var xw = xdr.NewWriter(w) - return o.EncodeXDRInto(xw) -} - -func (o Ping) MarshalXDR() ([]byte, error) { - return o.AppendXDR(make([]byte, 0, 128)) -} - -func (o Ping) MustMarshalXDR() []byte { - bs, err := o.MarshalXDR() - if err != nil { - panic(err) - } - return bs -} - -func (o Ping) AppendXDR(bs []byte) ([]byte, error) { - var aw = xdr.AppendWriter(bs) - var xw = xdr.NewWriter(&aw) - _, err := o.EncodeXDRInto(xw) - return []byte(aw), err -} - -func (o Ping) EncodeXDRInto(xw *xdr.Writer) (int, error) { - return xw.Tot(), xw.Error() -} - -func (o *Ping) DecodeXDR(r io.Reader) error { - xr := xdr.NewReader(r) - return o.DecodeXDRFrom(xr) -} - -func (o *Ping) UnmarshalXDR(bs []byte) error { - var br = bytes.NewReader(bs) - var xr = xdr.NewReader(br) - return o.DecodeXDRFrom(xr) -} - -func (o *Ping) DecodeXDRFrom(xr *xdr.Reader) error { - return xr.Error() -} - -/* - -Pong Structure: - - 0 1 2 3 - 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - - -struct Pong { -} - -*/ - -func (o Pong) EncodeXDR(w io.Writer) (int, error) { - var xw = xdr.NewWriter(w) - return o.EncodeXDRInto(xw) -} - -func (o Pong) MarshalXDR() ([]byte, error) { - return o.AppendXDR(make([]byte, 0, 128)) -} - -func (o Pong) MustMarshalXDR() []byte { - bs, err := o.MarshalXDR() - if err != nil { - panic(err) - } - return bs -} - -func (o Pong) AppendXDR(bs []byte) ([]byte, error) { - var aw = xdr.AppendWriter(bs) - var xw = xdr.NewWriter(&aw) - _, err := o.EncodeXDRInto(xw) - return []byte(aw), err -} - -func (o Pong) EncodeXDRInto(xw *xdr.Writer) (int, error) { - return xw.Tot(), xw.Error() -} - -func (o *Pong) DecodeXDR(r io.Reader) error { - xr := xdr.NewReader(r) - return o.DecodeXDRFrom(xr) -} - -func (o *Pong) UnmarshalXDR(bs []byte) error { - var br = bytes.NewReader(bs) - var xr = xdr.NewReader(br) - return o.DecodeXDRFrom(xr) -} - -func (o *Pong) DecodeXDRFrom(xr *xdr.Reader) error { - return xr.Error() -} - -/* - -JoinRelayRequest Structure: - - 0 1 2 3 - 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - - -struct JoinRelayRequest { -} - -*/ - -func (o JoinRelayRequest) EncodeXDR(w io.Writer) (int, error) { - var xw = xdr.NewWriter(w) - return o.EncodeXDRInto(xw) -} - -func (o JoinRelayRequest) MarshalXDR() ([]byte, error) { - return o.AppendXDR(make([]byte, 0, 128)) -} - -func (o JoinRelayRequest) MustMarshalXDR() []byte { - bs, err := o.MarshalXDR() - if err != nil { - panic(err) - } - return bs -} - -func (o JoinRelayRequest) AppendXDR(bs []byte) ([]byte, error) { - var aw = xdr.AppendWriter(bs) - var xw = xdr.NewWriter(&aw) - _, err := o.EncodeXDRInto(xw) - return []byte(aw), err -} - -func (o JoinRelayRequest) EncodeXDRInto(xw *xdr.Writer) (int, error) { - return xw.Tot(), xw.Error() -} - -func (o *JoinRelayRequest) DecodeXDR(r io.Reader) error { - xr := xdr.NewReader(r) - return o.DecodeXDRFrom(xr) -} - -func (o *JoinRelayRequest) UnmarshalXDR(bs []byte) error { - var br = bytes.NewReader(bs) - var xr = xdr.NewReader(br) - return o.DecodeXDRFrom(xr) -} - -func (o *JoinRelayRequest) DecodeXDRFrom(xr *xdr.Reader) error { - return xr.Error() -} - -/* - -JoinSessionRequest Structure: - - 0 1 2 3 - 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Length of Key | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -/ / -\ Key (variable length) \ -/ / -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - - -struct JoinSessionRequest { - opaque Key<32>; -} - -*/ - -func (o JoinSessionRequest) EncodeXDR(w io.Writer) (int, error) { - var xw = xdr.NewWriter(w) - return o.EncodeXDRInto(xw) -} - -func (o JoinSessionRequest) MarshalXDR() ([]byte, error) { - return o.AppendXDR(make([]byte, 0, 128)) -} - -func (o JoinSessionRequest) MustMarshalXDR() []byte { - bs, err := o.MarshalXDR() - if err != nil { - panic(err) - } - return bs -} - -func (o JoinSessionRequest) AppendXDR(bs []byte) ([]byte, error) { - var aw = xdr.AppendWriter(bs) - var xw = xdr.NewWriter(&aw) - _, err := o.EncodeXDRInto(xw) - return []byte(aw), err -} - -func (o JoinSessionRequest) EncodeXDRInto(xw *xdr.Writer) (int, error) { - if l := len(o.Key); l > 32 { - return xw.Tot(), xdr.ElementSizeExceeded("Key", l, 32) - } - xw.WriteBytes(o.Key) - return xw.Tot(), xw.Error() -} - -func (o *JoinSessionRequest) DecodeXDR(r io.Reader) error { - xr := xdr.NewReader(r) - return o.DecodeXDRFrom(xr) -} - -func (o *JoinSessionRequest) UnmarshalXDR(bs []byte) error { - var br = bytes.NewReader(bs) - var xr = xdr.NewReader(br) - return o.DecodeXDRFrom(xr) -} - -func (o *JoinSessionRequest) DecodeXDRFrom(xr *xdr.Reader) error { - o.Key = xr.ReadBytesMax(32) - return xr.Error() -} - -/* - -Response Structure: - - 0 1 2 3 - 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Code | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Length of Message | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -/ / -\ Message (variable length) \ -/ / -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - - -struct Response { - int Code; - string Message<>; -} - -*/ - -func (o Response) EncodeXDR(w io.Writer) (int, error) { - var xw = xdr.NewWriter(w) - return o.EncodeXDRInto(xw) -} - -func (o Response) MarshalXDR() ([]byte, error) { - return o.AppendXDR(make([]byte, 0, 128)) -} - -func (o Response) MustMarshalXDR() []byte { - bs, err := o.MarshalXDR() - if err != nil { - panic(err) - } - return bs -} - -func (o Response) AppendXDR(bs []byte) ([]byte, error) { - var aw = xdr.AppendWriter(bs) - var xw = xdr.NewWriter(&aw) - _, err := o.EncodeXDRInto(xw) - return []byte(aw), err -} - -func (o Response) EncodeXDRInto(xw *xdr.Writer) (int, error) { - xw.WriteUint32(uint32(o.Code)) - xw.WriteString(o.Message) - return xw.Tot(), xw.Error() -} - -func (o *Response) DecodeXDR(r io.Reader) error { - xr := xdr.NewReader(r) - return o.DecodeXDRFrom(xr) -} - -func (o *Response) UnmarshalXDR(bs []byte) error { - var br = bytes.NewReader(bs) - var xr = xdr.NewReader(br) - return o.DecodeXDRFrom(xr) -} - -func (o *Response) DecodeXDRFrom(xr *xdr.Reader) error { - o.Code = int32(xr.ReadUint32()) - o.Message = xr.ReadString() - return xr.Error() -} - -/* - -ConnectRequest Structure: - - 0 1 2 3 - 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Length of ID | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -/ / -\ ID (variable length) \ -/ / -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - - -struct ConnectRequest { - opaque ID<32>; -} - -*/ - -func (o ConnectRequest) EncodeXDR(w io.Writer) (int, error) { - var xw = xdr.NewWriter(w) - return o.EncodeXDRInto(xw) -} - -func (o ConnectRequest) MarshalXDR() ([]byte, error) { - return o.AppendXDR(make([]byte, 0, 128)) -} - -func (o ConnectRequest) MustMarshalXDR() []byte { - bs, err := o.MarshalXDR() - if err != nil { - panic(err) - } - return bs -} - -func (o ConnectRequest) AppendXDR(bs []byte) ([]byte, error) { - var aw = xdr.AppendWriter(bs) - var xw = xdr.NewWriter(&aw) - _, err := o.EncodeXDRInto(xw) - return []byte(aw), err -} - -func (o ConnectRequest) EncodeXDRInto(xw *xdr.Writer) (int, error) { - if l := len(o.ID); l > 32 { - return xw.Tot(), xdr.ElementSizeExceeded("ID", l, 32) - } - xw.WriteBytes(o.ID) - return xw.Tot(), xw.Error() -} - -func (o *ConnectRequest) DecodeXDR(r io.Reader) error { - xr := xdr.NewReader(r) - return o.DecodeXDRFrom(xr) -} - -func (o *ConnectRequest) UnmarshalXDR(bs []byte) error { - var br = bytes.NewReader(bs) - var xr = xdr.NewReader(br) - return o.DecodeXDRFrom(xr) -} - -func (o *ConnectRequest) DecodeXDRFrom(xr *xdr.Reader) error { - o.ID = xr.ReadBytesMax(32) - return xr.Error() -} - -/* - -SessionInvitation Structure: - - 0 1 2 3 - 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Length of From | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -/ / -\ From (variable length) \ -/ / -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Length of Key | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -/ / -\ Key (variable length) \ -/ / -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Length of Address | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -/ / -\ Address (variable length) \ -/ / -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| 0x0000 | Port | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| Server Socket (V=0 or 1) |V| -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - - -struct SessionInvitation { - opaque From<32>; - opaque Key<32>; - opaque Address<32>; - unsigned int Port; - bool ServerSocket; -} - -*/ - -func (o SessionInvitation) EncodeXDR(w io.Writer) (int, error) { - var xw = xdr.NewWriter(w) - return o.EncodeXDRInto(xw) -} - -func (o SessionInvitation) MarshalXDR() ([]byte, error) { - return o.AppendXDR(make([]byte, 0, 128)) -} - -func (o SessionInvitation) MustMarshalXDR() []byte { - bs, err := o.MarshalXDR() - if err != nil { - panic(err) - } - return bs -} - -func (o SessionInvitation) AppendXDR(bs []byte) ([]byte, error) { - var aw = xdr.AppendWriter(bs) - var xw = xdr.NewWriter(&aw) - _, err := o.EncodeXDRInto(xw) - return []byte(aw), err -} - -func (o SessionInvitation) EncodeXDRInto(xw *xdr.Writer) (int, error) { - if l := len(o.From); l > 32 { - return xw.Tot(), xdr.ElementSizeExceeded("From", l, 32) - } - xw.WriteBytes(o.From) - if l := len(o.Key); l > 32 { - return xw.Tot(), xdr.ElementSizeExceeded("Key", l, 32) - } - xw.WriteBytes(o.Key) - if l := len(o.Address); l > 32 { - return xw.Tot(), xdr.ElementSizeExceeded("Address", l, 32) - } - xw.WriteBytes(o.Address) - xw.WriteUint16(o.Port) - xw.WriteBool(o.ServerSocket) - return xw.Tot(), xw.Error() -} - -func (o *SessionInvitation) DecodeXDR(r io.Reader) error { - xr := xdr.NewReader(r) - return o.DecodeXDRFrom(xr) -} - -func (o *SessionInvitation) UnmarshalXDR(bs []byte) error { - var br = bytes.NewReader(bs) - var xr = xdr.NewReader(br) - return o.DecodeXDRFrom(xr) -} - -func (o *SessionInvitation) DecodeXDRFrom(xr *xdr.Reader) error { - o.From = xr.ReadBytesMax(32) - o.Key = xr.ReadBytesMax(32) - o.Address = xr.ReadBytesMax(32) - o.Port = xr.ReadUint16() - o.ServerSocket = xr.ReadBool() - return xr.Error() -} diff --git a/Godeps/_workspace/src/github.com/syncthing/relaysrv/protocol/protocol.go b/Godeps/_workspace/src/github.com/syncthing/relaysrv/protocol/protocol.go deleted file mode 100644 index 57a967ac8..000000000 --- a/Godeps/_workspace/src/github.com/syncthing/relaysrv/protocol/protocol.go +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file). - -package protocol - -import ( - "fmt" - "io" -) - -const ( - magic = 0x9E79BC40 - ProtocolName = "bep-relay" -) - -var ( - ResponseSuccess = Response{0, "success"} - ResponseNotFound = Response{1, "not found"} - ResponseAlreadyConnected = Response{2, "already connected"} - ResponseInternalError = Response{99, "internal error"} - ResponseUnexpectedMessage = Response{100, "unexpected message"} -) - -func WriteMessage(w io.Writer, message interface{}) error { - header := header{ - magic: magic, - } - - var payload []byte - var err error - - switch msg := message.(type) { - case Ping: - payload, err = msg.MarshalXDR() - header.messageType = messageTypePing - case Pong: - payload, err = msg.MarshalXDR() - header.messageType = messageTypePong - case JoinRelayRequest: - payload, err = msg.MarshalXDR() - header.messageType = messageTypeJoinRelayRequest - case JoinSessionRequest: - payload, err = msg.MarshalXDR() - header.messageType = messageTypeJoinSessionRequest - case Response: - payload, err = msg.MarshalXDR() - header.messageType = messageTypeResponse - case ConnectRequest: - payload, err = msg.MarshalXDR() - header.messageType = messageTypeConnectRequest - case SessionInvitation: - payload, err = msg.MarshalXDR() - header.messageType = messageTypeSessionInvitation - default: - err = fmt.Errorf("Unknown message type") - } - - if err != nil { - return err - } - - header.messageLength = int32(len(payload)) - - headerpayload, err := header.MarshalXDR() - if err != nil { - return err - } - - _, err = w.Write(append(headerpayload, payload...)) - return err -} - -func ReadMessage(r io.Reader) (interface{}, error) { - var header header - if err := header.DecodeXDR(r); err != nil { - return nil, err - } - - if header.magic != magic { - return nil, fmt.Errorf("magic mismatch") - } - - switch header.messageType { - case messageTypePing: - var msg Ping - err := msg.DecodeXDR(r) - return msg, err - case messageTypePong: - var msg Pong - err := msg.DecodeXDR(r) - return msg, err - case messageTypeJoinRelayRequest: - var msg JoinRelayRequest - err := msg.DecodeXDR(r) - return msg, err - case messageTypeJoinSessionRequest: - var msg JoinSessionRequest - err := msg.DecodeXDR(r) - return msg, err - case messageTypeResponse: - var msg Response - err := msg.DecodeXDR(r) - return msg, err - case messageTypeConnectRequest: - var msg ConnectRequest - err := msg.DecodeXDR(r) - return msg, err - case messageTypeSessionInvitation: - var msg SessionInvitation - err := msg.DecodeXDR(r) - return msg, err - } - - return nil, fmt.Errorf("Unknown message type") -}