diff --git a/cmd/stfileinfo/main.go b/cmd/stfileinfo/main.go
index 4dda3ca7a..2efb99611 100644
--- a/cmd/stfileinfo/main.go
+++ b/cmd/stfileinfo/main.go
@@ -68,8 +68,8 @@ func main() {
}
blockSize := int(fi.Size())
- if *standardBlocks || blockSize < protocol.BlockSize {
- blockSize = protocol.BlockSize
+ if *standardBlocks || blockSize < protocol.MinBlockSize {
+ blockSize = protocol.BlockSize(fi.Size())
}
bs, err := scanner.Blocks(context.TODO(), fd, blockSize, fi.Size(), nil, true)
if err != nil {
diff --git a/cmd/syncthing/gui.go b/cmd/syncthing/gui.go
index 0adca93c8..29fc35b9e 100644
--- a/cmd/syncthing/gui.go
+++ b/cmd/syncthing/gui.go
@@ -94,7 +94,7 @@ type modelIntf interface {
CurrentFolderFile(folder string, file string) (protocol.FileInfo, bool)
CurrentGlobalFile(folder string, file string) (protocol.FileInfo, bool)
ResetFolder(folder string)
- Availability(folder, file string, version protocol.Vector, block protocol.BlockInfo) []model.Availability
+ Availability(folder string, file protocol.FileInfo, block protocol.BlockInfo) []model.Availability
GetIgnores(folder string) ([]string, []string, error)
GetFolderVersions(folder string) (map[string][]versioner.FileVersion, error)
RestoreFolderVersions(folder string, versions map[string]time.Time) (map[string]string, error)
@@ -828,7 +828,7 @@ func (s *apiService) getDBFile(w http.ResponseWriter, r *http.Request) {
return
}
- av := s.model.Availability(folder, file, protocol.Vector{}, protocol.BlockInfo{})
+ av := s.model.Availability(folder, gf, protocol.BlockInfo{})
sendJSON(w, map[string]interface{}{
"global": jsonFileInfo(gf),
"local": jsonFileInfo(lf),
diff --git a/cmd/syncthing/mocked_model_test.go b/cmd/syncthing/mocked_model_test.go
index 13aed6db2..5841b99a0 100644
--- a/cmd/syncthing/mocked_model_test.go
+++ b/cmd/syncthing/mocked_model_test.go
@@ -64,7 +64,7 @@ func (m *mockedModel) CurrentGlobalFile(folder string, file string) (protocol.Fi
func (m *mockedModel) ResetFolder(folder string) {
}
-func (m *mockedModel) Availability(folder, file string, version protocol.Vector, block protocol.BlockInfo) []model.Availability {
+func (m *mockedModel) Availability(folder string, file protocol.FileInfo, block protocol.BlockInfo) []model.Availability {
return nil
}
diff --git a/cmd/syncthing/usage_report.go b/cmd/syncthing/usage_report.go
index 0c1da16cf..403dab39a 100644
--- a/cmd/syncthing/usage_report.go
+++ b/cmd/syncthing/usage_report.go
@@ -399,7 +399,7 @@ func (usageReportingService) String() string {
// cpuBench returns CPU performance as a measure of single threaded SHA-256 MiB/s
func cpuBench(iterations int, duration time.Duration, useWeakHash bool) float64 {
- dataSize := 16 * protocol.BlockSize
+ dataSize := 16 * protocol.MinBlockSize
bs := make([]byte, dataSize)
rand.Reader.Read(bs)
@@ -420,7 +420,7 @@ func cpuBenchOnce(duration time.Duration, useWeakHash bool, bs []byte) float64 {
b := 0
for time.Since(t0) < duration {
r := bytes.NewReader(bs)
- blocksResult, _ = scanner.Blocks(context.TODO(), r, protocol.BlockSize, int64(len(bs)), nil, useWeakHash)
+ blocksResult, _ = scanner.Blocks(context.TODO(), r, protocol.MinBlockSize, int64(len(bs)), nil, useWeakHash)
b += len(bs)
}
d := time.Since(t0)
diff --git a/lib/config/folderconfiguration.go b/lib/config/folderconfiguration.go
index 4300691a3..d40ab6513 100644
--- a/lib/config/folderconfiguration.go
+++ b/lib/config/folderconfiguration.go
@@ -52,6 +52,7 @@ type FolderConfiguration struct {
Paused bool `xml:"paused" json:"paused"`
WeakHashThresholdPct int `xml:"weakHashThresholdPct" json:"weakHashThresholdPct"` // Use weak hash if more than X percent of the file has changed. Set to -1 to always use weak hash.
MarkerName string `xml:"markerName" json:"markerName"`
+ UseLargeBlocks bool `xml:"useLargeBlocks" json:"useLargeBlocks"`
cachedFilesystem fs.Filesystem
diff --git a/lib/db/set.go b/lib/db/set.go
index 2d78f1a66..d6cf2630d 100644
--- a/lib/db/set.go
+++ b/lib/db/set.go
@@ -43,6 +43,7 @@ type FileIntf interface {
IsSymlink() bool
HasPermissionBits() bool
SequenceNo() int64
+ BlockSize() int
}
// The Iterator is called with either a protocol.FileInfo or a
diff --git a/lib/db/structs.go b/lib/db/structs.go
index c8d4e6ed5..60ff35823 100644
--- a/lib/db/structs.go
+++ b/lib/db/structs.go
@@ -17,8 +17,8 @@ import (
)
func (f FileInfoTruncated) String() string {
- return fmt.Sprintf("File{Name:%q, Permissions:0%o, Modified:%v, Version:%v, Length:%d, Deleted:%v, Invalid:%v, NoPermissions:%v}",
- f.Name, f.Permissions, f.ModTime(), f.Version, f.Size, f.Deleted, f.Invalid, f.NoPermissions)
+ return fmt.Sprintf("File{Name:%q, Permissions:0%o, Modified:%v, Version:%v, Length:%d, Deleted:%v, Invalid:%v, NoPermissions:%v, BlockSize:%d}",
+ f.Name, f.Permissions, f.ModTime(), f.Version, f.Size, f.Deleted, f.Invalid, f.NoPermissions, f.RawBlockSize)
}
func (f FileInfoTruncated) IsDeleted() bool {
@@ -56,6 +56,13 @@ func (f FileInfoTruncated) FileSize() int64 {
return f.Size
}
+func (f FileInfoTruncated) BlockSize() int {
+ if f.RawBlockSize == 0 {
+ return protocol.MinBlockSize
+ }
+ return int(f.RawBlockSize)
+}
+
func (f FileInfoTruncated) FileName() string {
return f.Name
}
@@ -70,12 +77,13 @@ func (f FileInfoTruncated) SequenceNo() int64 {
func (f FileInfoTruncated) ConvertToInvalidFileInfo(invalidatedBy protocol.ShortID) protocol.FileInfo {
return protocol.FileInfo{
- Name: f.Name,
- Type: f.Type,
- ModifiedS: f.ModifiedS,
- ModifiedNs: f.ModifiedNs,
- ModifiedBy: invalidatedBy,
- Invalid: true,
- Version: f.Version,
+ Name: f.Name,
+ Type: f.Type,
+ ModifiedS: f.ModifiedS,
+ ModifiedNs: f.ModifiedNs,
+ ModifiedBy: invalidatedBy,
+ Invalid: true,
+ Version: f.Version,
+ RawBlockSize: f.RawBlockSize,
}
}
diff --git a/lib/db/structs.pb.go b/lib/db/structs.pb.go
index fffa16e97..ae4235a3f 100644
--- a/lib/db/structs.pb.go
+++ b/lib/db/structs.pb.go
@@ -70,7 +70,9 @@ type FileInfoTruncated struct {
NoPermissions bool `protobuf:"varint,8,opt,name=no_permissions,json=noPermissions,proto3" json:"no_permissions,omitempty"`
Version protocol.Vector `protobuf:"bytes,9,opt,name=version" json:"version"`
Sequence int64 `protobuf:"varint,10,opt,name=sequence,proto3" json:"sequence,omitempty"`
- SymlinkTarget string `protobuf:"bytes,17,opt,name=symlink_target,json=symlinkTarget,proto3" json:"symlink_target,omitempty"`
+ RawBlockSize int32 `protobuf:"varint,13,opt,name=block_size,json=blockSize,proto3" json:"block_size,omitempty"`
+ // repeated BlockInfo Blocks = 16
+ SymlinkTarget string `protobuf:"bytes,17,opt,name=symlink_target,json=symlinkTarget,proto3" json:"symlink_target,omitempty"`
}
func (m *FileInfoTruncated) Reset() { *m = FileInfoTruncated{} }
@@ -277,6 +279,11 @@ func (m *FileInfoTruncated) MarshalTo(dAtA []byte) (int, error) {
i++
i = encodeVarintStructs(dAtA, i, uint64(m.ModifiedBy))
}
+ if m.RawBlockSize != 0 {
+ dAtA[i] = 0x68
+ i++
+ i = encodeVarintStructs(dAtA, i, uint64(m.RawBlockSize))
+ }
if len(m.SymlinkTarget) > 0 {
dAtA[i] = 0x8a
i++
@@ -472,6 +479,9 @@ func (m *FileInfoTruncated) ProtoSize() (n int) {
if m.ModifiedBy != 0 {
n += 1 + sovStructs(uint64(m.ModifiedBy))
}
+ if m.RawBlockSize != 0 {
+ n += 1 + sovStructs(uint64(m.RawBlockSize))
+ }
l = len(m.SymlinkTarget)
if l > 0 {
n += 2 + l + sovStructs(uint64(l))
@@ -1028,6 +1038,25 @@ func (m *FileInfoTruncated) Unmarshal(dAtA []byte) error {
break
}
}
+ case 13:
+ if wireType != 0 {
+ return fmt.Errorf("proto: wrong wireType = %d for field RawBlockSize", wireType)
+ }
+ m.RawBlockSize = 0
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowStructs
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ m.RawBlockSize |= (int32(b) & 0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
case 17:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field SymlinkTarget", wireType)
@@ -1481,43 +1510,45 @@ var (
func init() { proto.RegisterFile("structs.proto", fileDescriptorStructs) }
var fileDescriptorStructs = []byte{
- // 598 bytes of a gzipped FileDescriptorProto
- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x53, 0xcd, 0x6a, 0xdb, 0x4c,
- 0x14, 0xb5, 0x62, 0xc9, 0xb1, 0x47, 0x71, 0xbe, 0x2f, 0x43, 0x08, 0xc2, 0x50, 0x5b, 0x18, 0x0a,
- 0xa2, 0x50, 0xa5, 0x4d, 0xe8, 0xa6, 0xdd, 0xb9, 0x21, 0x10, 0x28, 0x6d, 0x99, 0x84, 0xac, 0x0a,
- 0xc1, 0x92, 0xae, 0x9d, 0xa1, 0xf2, 0x8c, 0xa3, 0x19, 0x07, 0xd4, 0x27, 0xe9, 0x32, 0x0f, 0xd3,
- 0x45, 0x96, 0x5d, 0x77, 0x11, 0x5a, 0xf7, 0x39, 0x0a, 0x45, 0x77, 0x24, 0x45, 0xe9, 0xaa, 0xdd,
- 0xdd, 0x73, 0xff, 0xef, 0x39, 0x33, 0xa4, 0xaf, 0x74, 0xb6, 0x8a, 0xb5, 0x0a, 0x97, 0x99, 0xd4,
- 0x92, 0x6e, 0x24, 0xd1, 0xe0, 0xe9, 0x9c, 0xeb, 0xcb, 0x55, 0x14, 0xc6, 0x72, 0xb1, 0x3f, 0x97,
- 0x73, 0xb9, 0x8f, 0xa1, 0x68, 0x35, 0x43, 0x84, 0x00, 0x2d, 0x53, 0x32, 0x78, 0xd1, 0x48, 0x57,
- 0xb9, 0x88, 0xf5, 0x25, 0x17, 0xf3, 0x86, 0x95, 0xf2, 0xc8, 0x74, 0x88, 0x65, 0xba, 0x1f, 0xc1,
- 0xd2, 0x94, 0x8d, 0xaf, 0x88, 0x7b, 0xcc, 0x53, 0x38, 0x87, 0x4c, 0x71, 0x29, 0xe8, 0x33, 0xb2,
- 0x79, 0x6d, 0x4c, 0xcf, 0xf2, 0xad, 0xc0, 0x3d, 0xf8, 0x3f, 0xac, 0x8a, 0xc2, 0x73, 0x88, 0xb5,
- 0xcc, 0x26, 0xf6, 0xed, 0xdd, 0xa8, 0xc5, 0xaa, 0x34, 0xba, 0x47, 0x3a, 0x09, 0x5c, 0xf3, 0x18,
- 0xbc, 0x0d, 0xdf, 0x0a, 0xb6, 0x58, 0x89, 0xa8, 0x47, 0x36, 0xb9, 0xb8, 0x9e, 0xa6, 0x3c, 0xf1,
- 0xda, 0xbe, 0x15, 0x74, 0x59, 0x05, 0xc7, 0xc7, 0xc4, 0x2d, 0xc7, 0xbd, 0xe1, 0x4a, 0xd3, 0xe7,
- 0xa4, 0x5b, 0xf6, 0x52, 0x9e, 0xe5, 0xb7, 0x03, 0xf7, 0xe0, 0xbf, 0x30, 0x89, 0xc2, 0xc6, 0x56,
- 0xe5, 0xc8, 0x3a, 0xed, 0xa5, 0xfd, 0xf9, 0x66, 0xd4, 0x1a, 0xff, 0x6a, 0x93, 0x9d, 0x22, 0xeb,
- 0x44, 0xcc, 0xe4, 0x59, 0xb6, 0x12, 0xf1, 0x54, 0x43, 0x42, 0x29, 0xb1, 0xc5, 0x74, 0x01, 0xb8,
- 0x7e, 0x8f, 0xa1, 0x4d, 0x9f, 0x10, 0x5b, 0xe7, 0x4b, 0xb3, 0xe1, 0xf6, 0xc1, 0xde, 0xfd, 0x49,
- 0x75, 0x79, 0xbe, 0x04, 0x86, 0x39, 0x45, 0xbd, 0xe2, 0x9f, 0x00, 0x97, 0x6e, 0x33, 0xb4, 0xa9,
- 0x4f, 0xdc, 0x25, 0x64, 0x0b, 0xae, 0xcc, 0x96, 0xb6, 0x6f, 0x05, 0x7d, 0xd6, 0x74, 0xd1, 0x47,
- 0x84, 0x2c, 0x64, 0xc2, 0x67, 0x1c, 0x92, 0x0b, 0xe5, 0x39, 0x58, 0xdb, 0xab, 0x3c, 0xa7, 0x05,
- 0x19, 0x09, 0xa4, 0xa0, 0x21, 0xf1, 0x3a, 0x86, 0x8c, 0x12, 0x36, 0x69, 0xda, 0x7c, 0x40, 0x13,
- 0x7d, 0x4c, 0xb6, 0x85, 0xbc, 0x68, 0xce, 0xed, 0x62, 0x42, 0x5f, 0xc8, 0xf7, 0x8d, 0xc9, 0x0d,
- 0xc5, 0x7a, 0x7f, 0xa7, 0xd8, 0x80, 0x74, 0x15, 0x5c, 0xad, 0x40, 0xc4, 0xe0, 0x11, 0xdc, 0xb4,
- 0xc6, 0x74, 0x44, 0xdc, 0xfa, 0x0e, 0xa1, 0x3c, 0xd7, 0xb7, 0x02, 0x87, 0xd5, 0xa7, 0xbd, 0x55,
- 0xf4, 0x43, 0x23, 0x21, 0xca, 0xbd, 0x2d, 0xdf, 0x0a, 0xec, 0xc9, 0xab, 0x62, 0xc0, 0xb7, 0xbb,
- 0xd1, 0xe1, 0x3f, 0xbc, 0xc1, 0xf0, 0xf4, 0x52, 0x66, 0xfa, 0xe4, 0xe8, 0xbe, 0xfb, 0x24, 0x2f,
- 0x6e, 0x56, 0xf9, 0x22, 0xe5, 0xe2, 0xe3, 0x85, 0x9e, 0x66, 0x73, 0xd0, 0xde, 0x0e, 0xca, 0xd8,
- 0x2f, 0xbd, 0x67, 0xe8, 0x2c, 0xf5, 0xff, 0x62, 0x91, 0xce, 0x6b, 0xb9, 0x12, 0x5a, 0xd1, 0x5d,
- 0xe2, 0xcc, 0x78, 0x0a, 0x0a, 0x55, 0x77, 0x98, 0x01, 0x85, 0x6c, 0x09, 0xcf, 0x90, 0x03, 0x0e,
- 0x0a, 0xd5, 0x77, 0x58, 0xd3, 0x85, 0x54, 0x98, 0xce, 0x0a, 0x05, 0x77, 0x58, 0x8d, 0x9b, 0x9a,
- 0xd9, 0x18, 0xaa, 0x35, 0xdb, 0x25, 0x4e, 0x94, 0x6b, 0xa8, 0x74, 0x36, 0xe0, 0x01, 0xad, 0x9d,
- 0x3f, 0x68, 0x1d, 0x90, 0xae, 0xf9, 0x16, 0x27, 0x47, 0x78, 0xd1, 0x16, 0xab, 0xf1, 0xf8, 0x1d,
- 0xe9, 0x99, 0x2b, 0x4e, 0x41, 0xd3, 0x80, 0x74, 0x62, 0x04, 0xe5, 0x57, 0x20, 0xc5, 0x57, 0x30,
- 0xe1, 0x52, 0xc6, 0x32, 0x5e, 0xac, 0x17, 0x67, 0x50, 0x3c, 0x79, 0x3c, 0xac, 0xcd, 0x2a, 0x38,
- 0xd9, 0xbd, 0xfd, 0x31, 0x6c, 0xdd, 0xae, 0x87, 0xd6, 0xd7, 0xf5, 0xd0, 0xfa, 0xbe, 0x1e, 0xb6,
- 0x6e, 0x7e, 0x0e, 0xad, 0xa8, 0x83, 0xc4, 0x1f, 0xfe, 0x0e, 0x00, 0x00, 0xff, 0xff, 0x6e, 0x86,
- 0x97, 0x21, 0x6a, 0x04, 0x00, 0x00,
+ // 630 bytes of a gzipped FileDescriptorProto
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x53, 0xcf, 0x6a, 0xdb, 0x4c,
+ 0x10, 0xb7, 0x62, 0xc9, 0xb1, 0x47, 0x76, 0xbe, 0x64, 0x09, 0x41, 0x18, 0x3e, 0x5b, 0x18, 0x0a,
+ 0xa2, 0x50, 0xb9, 0x4d, 0xe8, 0xa5, 0xbd, 0xb9, 0x21, 0x10, 0x28, 0x6d, 0x59, 0x87, 0x9c, 0x0a,
+ 0xc6, 0x92, 0xd6, 0xce, 0x12, 0x79, 0xd7, 0xd1, 0xae, 0x52, 0x94, 0x27, 0xe9, 0x31, 0x4f, 0xd0,
+ 0xa7, 0xe8, 0x21, 0xc7, 0x9e, 0x7b, 0x08, 0xad, 0xfb, 0x22, 0x45, 0xbb, 0x92, 0xa2, 0xf4, 0xd4,
+ 0xde, 0xe6, 0x37, 0xff, 0x67, 0x7e, 0x33, 0xd0, 0x13, 0x32, 0x49, 0x43, 0x29, 0xfc, 0x75, 0xc2,
+ 0x25, 0x47, 0x5b, 0x51, 0xd0, 0x7f, 0xb6, 0xa4, 0xf2, 0x22, 0x0d, 0xfc, 0x90, 0xaf, 0xc6, 0x4b,
+ 0xbe, 0xe4, 0x63, 0x65, 0x0a, 0xd2, 0x85, 0x42, 0x0a, 0x28, 0x49, 0x87, 0xf4, 0x5f, 0xd6, 0xdc,
+ 0x45, 0xc6, 0x42, 0x79, 0x41, 0xd9, 0xb2, 0x26, 0xc5, 0x34, 0xd0, 0x19, 0x42, 0x1e, 0x8f, 0x03,
+ 0xb2, 0xd6, 0x61, 0xa3, 0x2b, 0xb0, 0x4f, 0x68, 0x4c, 0xce, 0x49, 0x22, 0x28, 0x67, 0xe8, 0x39,
+ 0x6c, 0x5f, 0x6b, 0xd1, 0x31, 0x5c, 0xc3, 0xb3, 0x0f, 0x77, 0xfd, 0x32, 0xc8, 0x3f, 0x27, 0xa1,
+ 0xe4, 0xc9, 0xc4, 0xbc, 0xbb, 0x1f, 0x36, 0x70, 0xe9, 0x86, 0x0e, 0xa0, 0x15, 0x91, 0x6b, 0x1a,
+ 0x12, 0x67, 0xcb, 0x35, 0xbc, 0x2e, 0x2e, 0x10, 0x72, 0x60, 0x9b, 0xb2, 0xeb, 0x79, 0x4c, 0x23,
+ 0xa7, 0xe9, 0x1a, 0x5e, 0x1b, 0x97, 0x70, 0x74, 0x02, 0x76, 0x51, 0xee, 0x2d, 0x15, 0x12, 0xbd,
+ 0x80, 0x76, 0x91, 0x4b, 0x38, 0x86, 0xdb, 0xf4, 0xec, 0xc3, 0xff, 0xfc, 0x28, 0xf0, 0x6b, 0x5d,
+ 0x15, 0x25, 0x2b, 0xb7, 0x57, 0xe6, 0xe7, 0xdb, 0x61, 0x63, 0xf4, 0xc5, 0x84, 0xbd, 0xdc, 0xeb,
+ 0x94, 0x2d, 0xf8, 0x59, 0x92, 0xb2, 0x70, 0x2e, 0x49, 0x84, 0x10, 0x98, 0x6c, 0xbe, 0x22, 0xaa,
+ 0xfd, 0x0e, 0x56, 0x32, 0x7a, 0x0a, 0xa6, 0xcc, 0xd6, 0xba, 0xc3, 0x9d, 0xc3, 0x83, 0x87, 0x91,
+ 0xaa, 0xf0, 0x6c, 0x4d, 0xb0, 0xf2, 0xc9, 0xe3, 0x05, 0xbd, 0x21, 0xaa, 0xe9, 0x26, 0x56, 0x32,
+ 0x72, 0xc1, 0x5e, 0x93, 0x64, 0x45, 0x85, 0xee, 0xd2, 0x74, 0x0d, 0xaf, 0x87, 0xeb, 0x2a, 0xf4,
+ 0x3f, 0xc0, 0x8a, 0x47, 0x74, 0x41, 0x49, 0x34, 0x13, 0x8e, 0xa5, 0x62, 0x3b, 0xa5, 0x66, 0x9a,
+ 0x2f, 0x23, 0x22, 0x31, 0x91, 0x24, 0x72, 0x5a, 0x7a, 0x19, 0x05, 0xac, 0xaf, 0x69, 0xfb, 0xd1,
+ 0x9a, 0xd0, 0x13, 0xd8, 0x61, 0x7c, 0x56, 0xaf, 0xdb, 0x56, 0x0e, 0x3d, 0xc6, 0x3f, 0xd4, 0x2a,
+ 0xd7, 0x18, 0xeb, 0xfc, 0x1d, 0x63, 0x7d, 0x68, 0x0b, 0x72, 0x95, 0x12, 0x16, 0x12, 0x07, 0x54,
+ 0xa7, 0x15, 0x46, 0x43, 0xb0, 0xab, 0x39, 0x98, 0x70, 0x6c, 0xd7, 0xf0, 0x2c, 0x5c, 0x8d, 0xf6,
+ 0x4e, 0xa0, 0x8f, 0x35, 0x87, 0x20, 0x73, 0xba, 0xae, 0xe1, 0x99, 0x93, 0xd7, 0x79, 0x81, 0xef,
+ 0xf7, 0xc3, 0xa3, 0x7f, 0xb8, 0x41, 0x7f, 0x7a, 0xc1, 0x13, 0x79, 0x7a, 0xfc, 0x90, 0x7d, 0x92,
+ 0xa1, 0x31, 0x40, 0x10, 0xf3, 0xf0, 0x72, 0xa6, 0x28, 0xe8, 0xe5, 0xd5, 0x27, 0xbb, 0x9b, 0xfb,
+ 0x61, 0x17, 0xcf, 0x3f, 0x4d, 0x72, 0xc3, 0x94, 0xde, 0x10, 0xdc, 0x09, 0x4a, 0x31, 0x5f, 0x92,
+ 0xc8, 0x56, 0x31, 0x65, 0x97, 0x33, 0x39, 0x4f, 0x96, 0x44, 0x3a, 0x7b, 0x8a, 0xf7, 0x5e, 0xa1,
+ 0x3d, 0x53, 0xca, 0xe2, 0x60, 0xbe, 0x1a, 0xd0, 0x7a, 0xc3, 0x53, 0x26, 0x05, 0xda, 0x07, 0x6b,
+ 0x41, 0x63, 0x22, 0xd4, 0x99, 0x58, 0x58, 0x83, 0x9c, 0xe7, 0x88, 0x26, 0x6a, 0x69, 0x94, 0x08,
+ 0x75, 0x2e, 0x16, 0xae, 0xab, 0xd4, 0xee, 0x74, 0x66, 0xa1, 0x2e, 0xc4, 0xc2, 0x15, 0xae, 0x93,
+ 0x6c, 0x2a, 0x53, 0x45, 0xf2, 0x3e, 0x58, 0x41, 0x26, 0x49, 0x79, 0x18, 0x1a, 0x3c, 0xe2, 0xa1,
+ 0xf5, 0x07, 0x0f, 0x7d, 0x68, 0xeb, 0x3f, 0x3a, 0x3d, 0x56, 0x13, 0x75, 0x71, 0x85, 0x47, 0xef,
+ 0xa1, 0xa3, 0xa7, 0x98, 0x12, 0x89, 0x3c, 0x68, 0x85, 0x0a, 0x14, 0xbf, 0x03, 0xf9, 0xef, 0x68,
+ 0x73, 0xc1, 0x7b, 0x61, 0xcf, 0xdb, 0x0b, 0x13, 0x92, 0xff, 0x88, 0x1a, 0xac, 0x89, 0x4b, 0x38,
+ 0xd9, 0xbf, 0xfb, 0x39, 0x68, 0xdc, 0x6d, 0x06, 0xc6, 0xb7, 0xcd, 0xc0, 0xf8, 0xb1, 0x19, 0x34,
+ 0x6e, 0x7f, 0x0d, 0x8c, 0xa0, 0xa5, 0x98, 0x3a, 0xfa, 0x1d, 0x00, 0x00, 0xff, 0xff, 0x28, 0xfa,
+ 0x1e, 0x63, 0x9b, 0x04, 0x00, 0x00,
}
diff --git a/lib/db/structs.proto b/lib/db/structs.proto
index 5af65cad2..1aaac22d1 100644
--- a/lib/db/structs.proto
+++ b/lib/db/structs.proto
@@ -35,6 +35,9 @@ message FileInfoTruncated {
bool no_permissions = 8;
protocol.Vector version = 9 [(gogoproto.nullable) = false];
int64 sequence = 10;
+ int32 block_size = 13 [(gogoproto.customname) = "RawBlockSize"];
+
+ // repeated BlockInfo Blocks = 16
string symlink_target = 17;
}
diff --git a/lib/model/model.go b/lib/model/model.go
index 44b0c5b70..0fb8860ed 100644
--- a/lib/model/model.go
+++ b/lib/model/model.go
@@ -626,7 +626,7 @@ func (m *Model) Completion(device protocol.DeviceID, folder string) FolderComple
}
// This might might be more than it really is, because some blocks can be of a smaller size.
- downloaded = int64(counts[ft.Name] * protocol.BlockSize)
+ downloaded = int64(counts[ft.Name] * int(ft.BlockSize()))
fileNeed = ft.FileSize() - downloaded
if fileNeed < 0 {
@@ -1968,7 +1968,6 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su
Folder: folderCfg.ID,
Subs: subDirs,
Matcher: ignores,
- BlockSize: protocol.BlockSize,
TempLifetime: time.Duration(m.cfg.Options().KeepTemporariesH) * time.Hour,
CurrentFiler: cFiler{m, folder},
Filesystem: mtimefs,
@@ -1978,6 +1977,7 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su
ShortID: m.shortID,
ProgressTickIntervalS: folderCfg.ScanProgressIntervalS,
UseWeakHashes: weakhash.Enabled,
+ UseLargeBlocks: folderCfg.UseLargeBlocks,
})
if err := runner.CheckHealth(); err != nil {
@@ -2513,7 +2513,7 @@ func (m *Model) RestoreFolderVersions(folder string, versions map[string]time.Ti
return errors, nil
}
-func (m *Model) Availability(folder, file string, version protocol.Vector, block protocol.BlockInfo) []Availability {
+func (m *Model) Availability(folder string, file protocol.FileInfo, block protocol.BlockInfo) []Availability {
// The slightly unusual locking sequence here is because we need to hold
// pmut for the duration (as the value returned from foldersFiles can
// get heavily modified on Close()), but also must acquire fmut before
@@ -2532,7 +2532,7 @@ func (m *Model) Availability(folder, file string, version protocol.Vector, block
var availabilities []Availability
next:
- for _, device := range fs.Availability(file) {
+ for _, device := range fs.Availability(file.Name) {
for _, pausedFolder := range m.remotePausedFolders[device] {
if pausedFolder == folder {
continue next
@@ -2545,7 +2545,7 @@ next:
}
for device := range devices {
- if m.deviceDownloads[device].Has(folder, file, version, int32(block.Offset/protocol.BlockSize)) {
+ if m.deviceDownloads[device].Has(folder, file.Name, file.Version, int32(block.Offset/int64(file.BlockSize()))) {
availabilities = append(availabilities, Availability{ID: device, FromTemporary: true})
}
}
diff --git a/lib/model/model_test.go b/lib/model/model_test.go
index 493e05ad8..7b93affd0 100644
--- a/lib/model/model_test.go
+++ b/lib/model/model_test.go
@@ -177,7 +177,7 @@ func TestRequest(t *testing.T) {
defer m.Stop()
m.ScanFolder("default")
- bs := make([]byte, protocol.BlockSize)
+ bs := make([]byte, protocol.MinBlockSize)
// Existing, shared file
bs = bs[:6]
@@ -409,20 +409,22 @@ func (f *fakeConnection) addFile(name string, flags uint32, ftype protocol.FileI
f.mut.Lock()
defer f.mut.Unlock()
- blocks, _ := scanner.Blocks(context.TODO(), bytes.NewReader(data), protocol.BlockSize, int64(len(data)), nil, true)
+ blockSize := protocol.BlockSize(int64(len(data)))
+ blocks, _ := scanner.Blocks(context.TODO(), bytes.NewReader(data), blockSize, int64(len(data)), nil, true)
var version protocol.Vector
version = version.Update(f.id.Short())
if ftype == protocol.FileInfoTypeFile || ftype == protocol.FileInfoTypeDirectory {
f.files = append(f.files, protocol.FileInfo{
- Name: name,
- Type: ftype,
- Size: int64(len(data)),
- ModifiedS: time.Now().Unix(),
- Permissions: flags,
- Version: version,
- Sequence: time.Now().UnixNano(),
- Blocks: blocks,
+ Name: name,
+ Type: ftype,
+ Size: int64(len(data)),
+ ModifiedS: time.Now().Unix(),
+ Permissions: flags,
+ Version: version,
+ Sequence: time.Now().UnixNano(),
+ RawBlockSize: int32(blockSize),
+ Blocks: blocks,
})
} else {
// Symlink
@@ -2803,7 +2805,7 @@ func TestNoRequestsFromPausedDevices(t *testing.T) {
files.Update(device1, []protocol.FileInfo{file})
files.Update(device2, []protocol.FileInfo{file})
- avail := m.Availability("default", file.Name, file.Version, file.Blocks[0])
+ avail := m.Availability("default", file, file.Blocks[0])
if len(avail) != 0 {
t.Errorf("should not be available, no connections")
}
@@ -2813,7 +2815,7 @@ func TestNoRequestsFromPausedDevices(t *testing.T) {
// !!! This is not what I'd expect to happen, as we don't even know if the peer has the original index !!!
- avail = m.Availability("default", file.Name, file.Version, file.Blocks[0])
+ avail = m.Availability("default", file, file.Blocks[0])
if len(avail) != 2 {
t.Errorf("should have two available")
}
@@ -2833,7 +2835,7 @@ func TestNoRequestsFromPausedDevices(t *testing.T) {
m.ClusterConfig(device1, cc)
m.ClusterConfig(device2, cc)
- avail = m.Availability("default", file.Name, file.Version, file.Blocks[0])
+ avail = m.Availability("default", file, file.Blocks[0])
if len(avail) != 2 {
t.Errorf("should have two available")
}
@@ -2841,7 +2843,7 @@ func TestNoRequestsFromPausedDevices(t *testing.T) {
m.Closed(&fakeConnection{id: device1}, errDeviceUnknown)
m.Closed(&fakeConnection{id: device2}, errDeviceUnknown)
- avail = m.Availability("default", file.Name, file.Version, file.Blocks[0])
+ avail = m.Availability("default", file, file.Blocks[0])
if len(avail) != 0 {
t.Errorf("should have no available")
}
@@ -2856,7 +2858,7 @@ func TestNoRequestsFromPausedDevices(t *testing.T) {
ccp.Folders[0].Paused = true
m.ClusterConfig(device1, ccp)
- avail = m.Availability("default", file.Name, file.Version, file.Blocks[0])
+ avail = m.Availability("default", file, file.Blocks[0])
if len(avail) != 1 {
t.Errorf("should have one available")
}
diff --git a/lib/model/rwfolder.go b/lib/model/rwfolder.go
index 9d4292ae7..0c69f851b 100644
--- a/lib/model/rwfolder.go
+++ b/lib/model/rwfolder.go
@@ -126,7 +126,7 @@ func newSendReceiveFolder(model *Model, cfg config.FolderConfiguration, ver vers
if f.PullerMaxPendingKiB == 0 {
f.PullerMaxPendingKiB = defaultPullerPendingKiB
}
- if blockSizeKiB := protocol.BlockSize / 1024; f.PullerMaxPendingKiB < blockSizeKiB {
+ if blockSizeKiB := protocol.MaxBlockSize / 1024; f.PullerMaxPendingKiB < blockSizeKiB {
f.PullerMaxPendingKiB = blockSizeKiB
}
@@ -1010,7 +1010,7 @@ func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, copyChan chan<- c
// Check for an old temporary file which might have some blocks we could
// reuse.
- tempBlocks, err := scanner.HashFile(f.ctx, f.fs, tempName, protocol.BlockSize, nil, false)
+ tempBlocks, err := scanner.HashFile(f.ctx, f.fs, tempName, file.BlockSize(), nil, false)
if err == nil {
// Check for any reusable blocks in the temp file
tempCopyBlocks, _ := blockDiff(tempBlocks, file.Blocks)
@@ -1160,7 +1160,7 @@ func (f *sendReceiveFolder) shortcutFile(file protocol.FileInfo) error {
// copierRoutine reads copierStates until the in channel closes and performs
// the relevant copies when possible, or passes it to the puller routine.
func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan chan<- pullBlockState, out chan<- *sharedPullerState) {
- buf := make([]byte, protocol.BlockSize)
+ buf := make([]byte, protocol.MinBlockSize)
for state := range in {
dstFd, err := state.tempFile()
@@ -1203,7 +1203,7 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
if len(hashesToFind) > 0 {
file, err = f.fs.Open(state.file.Name)
if err == nil {
- weakHashFinder, err = weakhash.NewFinder(file, protocol.BlockSize, hashesToFind)
+ weakHashFinder, err = weakhash.NewFinder(file, int(state.file.BlockSize()), hashesToFind)
if err != nil {
l.Debugln("weak hasher", err)
}
@@ -1268,7 +1268,7 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
return false
}
- _, err = fd.ReadAt(buf, protocol.BlockSize*int64(index))
+ _, err = fd.ReadAt(buf, int64(state.file.BlockSize())*int64(index))
fd.Close()
if err != nil {
return false
@@ -1391,7 +1391,7 @@ func (f *sendReceiveFolder) pullBlock(state pullBlockState, out chan<- *sharedPu
}
var lastError error
- candidates := f.model.Availability(f.folderID, state.file.Name, state.file.Version, state.block)
+ candidates := f.model.Availability(f.folderID, state.file, state.block)
for {
// Select the least busy device to pull the block from. If we found no
// feasible device at all, fail the block (and in the long run, the
diff --git a/lib/model/rwfolder_test.go b/lib/model/rwfolder_test.go
index 6415924b0..e555a5855 100644
--- a/lib/model/rwfolder_test.go
+++ b/lib/model/rwfolder_test.go
@@ -256,7 +256,7 @@ func TestCopierFinder(t *testing.T) {
}
// Verify that the fetched blocks have actually been written to the temp file
- blks, err := scanner.HashFile(context.TODO(), fs.NewFilesystem(fs.FilesystemTypeBasic, "."), tempFile, protocol.BlockSize, nil, false)
+ blks, err := scanner.HashFile(context.TODO(), fs.NewFilesystem(fs.FilesystemTypeBasic, "."), tempFile, protocol.MinBlockSize, nil, false)
if err != nil {
t.Log(err)
}
@@ -273,8 +273,8 @@ func TestWeakHash(t *testing.T) {
tempFile := filepath.Join("testdata", fs.TempName("weakhash"))
var shift int64 = 10
var size int64 = 1 << 20
- expectBlocks := int(size / protocol.BlockSize)
- expectPulls := int(shift / protocol.BlockSize)
+ expectBlocks := int(size / protocol.MinBlockSize)
+ expectPulls := int(shift / protocol.MinBlockSize)
if shift > 0 {
expectPulls++
}
@@ -307,7 +307,7 @@ func TestWeakHash(t *testing.T) {
// File 1: abcdefgh
// File 2: xyabcdef
f.Seek(0, os.SEEK_SET)
- existing, err := scanner.Blocks(context.TODO(), f, protocol.BlockSize, size, nil, true)
+ existing, err := scanner.Blocks(context.TODO(), f, protocol.MinBlockSize, size, nil, true)
if err != nil {
t.Error(err)
}
@@ -316,7 +316,7 @@ func TestWeakHash(t *testing.T) {
remainder := io.LimitReader(f, size-shift)
prefix := io.LimitReader(rand.Reader, shift)
nf := io.MultiReader(prefix, remainder)
- desired, err := scanner.Blocks(context.TODO(), nf, protocol.BlockSize, size, nil, true)
+ desired, err := scanner.Blocks(context.TODO(), nf, protocol.MinBlockSize, size, nil, true)
if err != nil {
t.Error(err)
}
diff --git a/lib/model/sharedpullerstate.go b/lib/model/sharedpullerstate.go
index 1c048536c..a09722f47 100644
--- a/lib/model/sharedpullerstate.go
+++ b/lib/model/sharedpullerstate.go
@@ -217,7 +217,7 @@ func (s *sharedPullerState) copyDone(block protocol.BlockInfo) {
s.mut.Lock()
s.copyNeeded--
s.updated = time.Now()
- s.available = append(s.available, int32(block.Offset/protocol.BlockSize))
+ s.available = append(s.available, int32(block.Offset/int64(s.file.BlockSize())))
s.availableUpdated = time.Now()
l.Debugln("sharedPullerState", s.folder, s.file.Name, "copyNeeded ->", s.copyNeeded)
s.mut.Unlock()
@@ -253,7 +253,7 @@ func (s *sharedPullerState) pullDone(block protocol.BlockInfo) {
s.mut.Lock()
s.pullNeeded--
s.updated = time.Now()
- s.available = append(s.available, int32(block.Offset/protocol.BlockSize))
+ s.available = append(s.available, int32(block.Offset/int64(s.file.BlockSize())))
s.availableUpdated = time.Now()
l.Debugln("sharedPullerState", s.folder, s.file.Name, "pullNeeded done ->", s.pullNeeded)
s.mut.Unlock()
@@ -314,8 +314,8 @@ func (s *sharedPullerState) Progress() *pullerProgress {
CopiedFromElsewhere: s.copyTotal - s.copyNeeded - s.copyOrigin,
Pulled: s.pullTotal - s.pullNeeded,
Pulling: s.pullNeeded,
- BytesTotal: blocksToSize(total),
- BytesDone: blocksToSize(done),
+ BytesTotal: blocksToSize(s.file.BlockSize(), total),
+ BytesDone: blocksToSize(s.file.BlockSize(), done),
}
}
@@ -343,9 +343,9 @@ func (s *sharedPullerState) Available() []int32 {
return blocks
}
-func blocksToSize(num int) int64 {
+func blocksToSize(size int, num int) int64 {
if num < 2 {
- return protocol.BlockSize / 2
+ return int64(size / 2)
}
- return int64(num-1)*protocol.BlockSize + protocol.BlockSize/2
+ return int64(num-1)*int64(size) + int64(size/2)
}
diff --git a/lib/protocol/bep.pb.go b/lib/protocol/bep.pb.go
index cb018572e..a98cef07e 100644
--- a/lib/protocol/bep.pb.go
+++ b/lib/protocol/bep.pb.go
@@ -306,6 +306,7 @@ type FileInfo struct {
NoPermissions bool `protobuf:"varint,8,opt,name=no_permissions,json=noPermissions,proto3" json:"no_permissions,omitempty"`
Version Vector `protobuf:"bytes,9,opt,name=version" json:"version"`
Sequence int64 `protobuf:"varint,10,opt,name=sequence,proto3" json:"sequence,omitempty"`
+ RawBlockSize int32 `protobuf:"varint,13,opt,name=block_size,json=blockSize,proto3" json:"block_size,omitempty"`
Blocks []BlockInfo `protobuf:"bytes,16,rep,name=Blocks" json:"Blocks"`
SymlinkTarget string `protobuf:"bytes,17,opt,name=symlink_target,json=symlinkTarget,proto3" json:"symlink_target,omitempty"`
}
@@ -876,6 +877,11 @@ func (m *FileInfo) MarshalTo(dAtA []byte) (int, error) {
i++
i = encodeVarintBep(dAtA, i, uint64(m.ModifiedBy))
}
+ if m.RawBlockSize != 0 {
+ dAtA[i] = 0x68
+ i++
+ i = encodeVarintBep(dAtA, i, uint64(m.RawBlockSize))
+ }
if len(m.Blocks) > 0 {
for _, msg := range m.Blocks {
dAtA[i] = 0x82
@@ -1427,6 +1433,9 @@ func (m *FileInfo) ProtoSize() (n int) {
if m.ModifiedBy != 0 {
n += 1 + sovBep(uint64(m.ModifiedBy))
}
+ if m.RawBlockSize != 0 {
+ n += 1 + sovBep(uint64(m.RawBlockSize))
+ }
if len(m.Blocks) > 0 {
for _, e := range m.Blocks {
l = e.ProtoSize()
@@ -2904,6 +2913,25 @@ func (m *FileInfo) Unmarshal(dAtA []byte) error {
break
}
}
+ case 13:
+ if wireType != 0 {
+ return fmt.Errorf("proto: wrong wireType = %d for field RawBlockSize", wireType)
+ }
+ m.RawBlockSize = 0
+ for shift := uint(0); ; shift += 7 {
+ if shift >= 64 {
+ return ErrIntOverflowBep
+ }
+ if iNdEx >= l {
+ return io.ErrUnexpectedEOF
+ }
+ b := dAtA[iNdEx]
+ iNdEx++
+ m.RawBlockSize |= (int32(b) & 0x7F) << shift
+ if b < 0x80 {
+ break
+ }
+ }
case 16:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Blocks", wireType)
@@ -4164,114 +4192,115 @@ var (
func init() { proto.RegisterFile("bep.proto", fileDescriptorBep) }
var fileDescriptorBep = []byte{
- // 1729 bytes of a gzipped FileDescriptorProto
+ // 1757 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0x4f, 0x8f, 0xdb, 0xc6,
- 0x15, 0x17, 0x25, 0xea, 0xdf, 0x93, 0x76, 0xc3, 0x1d, 0xdb, 0x5b, 0x96, 0xd9, 0x68, 0x69, 0xc5,
- 0x8e, 0x37, 0x8b, 0x64, 0xe3, 0x26, 0x69, 0x8b, 0x16, 0x6d, 0x01, 0xfd, 0xe1, 0xae, 0x85, 0xca,
- 0x94, 0x3a, 0xd2, 0x3a, 0x75, 0x0e, 0x25, 0x28, 0x71, 0xa4, 0x25, 0x4c, 0x71, 0x54, 0x92, 0x5a,
- 0x5b, 0xfd, 0x08, 0xfa, 0x04, 0xbd, 0x08, 0x08, 0xd0, 0x53, 0xef, 0xfd, 0x10, 0x3e, 0x06, 0x3d,
- 0xf4, 0xd0, 0x83, 0xd1, 0x6c, 0x2f, 0x3d, 0xf6, 0x5e, 0xa0, 0x28, 0x38, 0x43, 0x52, 0xd4, 0xae,
- 0x37, 0xf0, 0xa1, 0x27, 0xce, 0xbc, 0xf7, 0x9b, 0x3f, 0xef, 0xf7, 0xde, 0xef, 0x0d, 0xa1, 0x3c,
- 0x22, 0xf3, 0x93, 0xb9, 0x47, 0x03, 0x8a, 0x4a, 0xec, 0x33, 0xa6, 0x8e, 0xf2, 0xe9, 0xd4, 0x0e,
- 0x2e, 0x16, 0xa3, 0x93, 0x31, 0x9d, 0x7d, 0x36, 0xa5, 0x53, 0xfa, 0x19, 0xf3, 0x8c, 0x16, 0x13,
- 0x36, 0x63, 0x13, 0x36, 0xe2, 0x0b, 0xeb, 0x73, 0xc8, 0x3f, 0x21, 0x8e, 0x43, 0xd1, 0x21, 0x54,
- 0x2c, 0x72, 0x69, 0x8f, 0x89, 0xe1, 0x9a, 0x33, 0x22, 0x0b, 0xaa, 0x70, 0x54, 0xc6, 0xc0, 0x4d,
- 0xba, 0x39, 0x23, 0x21, 0x60, 0xec, 0xd8, 0xc4, 0x0d, 0x38, 0x20, 0xcb, 0x01, 0xdc, 0xc4, 0x00,
- 0x0f, 0x61, 0x37, 0x02, 0x5c, 0x12, 0xcf, 0xb7, 0xa9, 0x2b, 0xe7, 0x18, 0x66, 0x87, 0x5b, 0x9f,
- 0x71, 0x63, 0xdd, 0x87, 0xc2, 0x13, 0x62, 0x5a, 0xc4, 0x43, 0x1f, 0x83, 0x18, 0x2c, 0xe7, 0xfc,
- 0xac, 0xdd, 0xcf, 0xef, 0x9d, 0xc4, 0x31, 0x9c, 0x3c, 0x25, 0xbe, 0x6f, 0x4e, 0xc9, 0x70, 0x39,
- 0x27, 0x98, 0x41, 0xd0, 0xaf, 0xa0, 0x32, 0xa6, 0xb3, 0xb9, 0x47, 0x7c, 0xb6, 0x71, 0x96, 0xad,
- 0x38, 0xb8, 0xb1, 0xa2, 0xb5, 0xc1, 0xe0, 0xf4, 0x82, 0x7a, 0x03, 0x76, 0x5a, 0xce, 0xc2, 0x0f,
- 0x88, 0xd7, 0xa2, 0xee, 0xc4, 0x9e, 0xa2, 0xc7, 0x50, 0x9c, 0x50, 0xc7, 0x22, 0x9e, 0x2f, 0x0b,
- 0x6a, 0xee, 0xa8, 0xf2, 0xb9, 0xb4, 0xd9, 0xec, 0x94, 0x39, 0x9a, 0xe2, 0xeb, 0x37, 0x87, 0x19,
- 0x1c, 0xc3, 0xea, 0x7f, 0xca, 0x42, 0x81, 0x7b, 0xd0, 0x3e, 0x64, 0x6d, 0x8b, 0x53, 0xd4, 0x2c,
- 0x5c, 0xbd, 0x39, 0xcc, 0x76, 0xda, 0x38, 0x6b, 0x5b, 0xe8, 0x2e, 0xe4, 0x1d, 0x73, 0x44, 0x9c,
- 0x88, 0x1c, 0x3e, 0x41, 0xef, 0x43, 0xd9, 0x23, 0xa6, 0x65, 0x50, 0xd7, 0x59, 0x32, 0x4a, 0x4a,
- 0xb8, 0x14, 0x1a, 0x7a, 0xae, 0xb3, 0x44, 0x9f, 0x02, 0xb2, 0xa7, 0x2e, 0xf5, 0x88, 0x31, 0x27,
- 0xde, 0xcc, 0x66, 0xb7, 0xf5, 0x65, 0x91, 0xa1, 0xf6, 0xb8, 0xa7, 0xbf, 0x71, 0xa0, 0x0f, 0x61,
- 0x27, 0x82, 0x5b, 0xc4, 0x21, 0x01, 0x91, 0xf3, 0x0c, 0x59, 0xe5, 0xc6, 0x36, 0xb3, 0xa1, 0xc7,
- 0x70, 0xd7, 0xb2, 0x7d, 0x73, 0xe4, 0x10, 0x23, 0x20, 0xb3, 0xb9, 0x61, 0xbb, 0x16, 0x79, 0x45,
- 0x7c, 0xb9, 0xc0, 0xb0, 0x28, 0xf2, 0x0d, 0xc9, 0x6c, 0xde, 0xe1, 0x1e, 0xb4, 0x0f, 0x85, 0xb9,
- 0xb9, 0xf0, 0x89, 0x25, 0x17, 0x19, 0x26, 0x9a, 0x85, 0x2c, 0xf1, 0x0a, 0xf0, 0x65, 0xe9, 0x3a,
- 0x4b, 0x6d, 0xe6, 0x88, 0x59, 0x8a, 0x60, 0xf5, 0x7f, 0x67, 0xa1, 0xc0, 0x3d, 0xe8, 0xa3, 0x84,
- 0xa5, 0x6a, 0x73, 0x3f, 0x44, 0xfd, 0xfd, 0xcd, 0x61, 0x89, 0xfb, 0x3a, 0xed, 0x14, 0x6b, 0x08,
- 0xc4, 0x54, 0x45, 0xb1, 0x31, 0x3a, 0x80, 0xb2, 0x69, 0x59, 0x61, 0xf6, 0x88, 0x2f, 0xe7, 0xd4,
- 0xdc, 0x51, 0x19, 0x6f, 0x0c, 0xe8, 0xa7, 0xdb, 0xd5, 0x20, 0x5e, 0xaf, 0x9f, 0xdb, 0xca, 0x20,
- 0x4c, 0xc5, 0x98, 0x78, 0x51, 0x05, 0xe7, 0xd9, 0x79, 0xa5, 0xd0, 0xc0, 0xea, 0xf7, 0x3e, 0x54,
- 0x67, 0xe6, 0x2b, 0xc3, 0x27, 0xbf, 0x5f, 0x10, 0x77, 0x4c, 0x18, 0x5d, 0x39, 0x5c, 0x99, 0x99,
- 0xaf, 0x06, 0x91, 0x09, 0xd5, 0x00, 0x6c, 0x37, 0xf0, 0xa8, 0xb5, 0x18, 0x13, 0x2f, 0xe2, 0x2a,
- 0x65, 0x41, 0x3f, 0x86, 0x12, 0x23, 0xdb, 0xb0, 0x2d, 0xb9, 0xa4, 0x0a, 0x47, 0x62, 0x53, 0x89,
- 0x02, 0x2f, 0x32, 0xaa, 0x59, 0xdc, 0xf1, 0x10, 0x17, 0x19, 0xb6, 0x63, 0xa1, 0x5f, 0x80, 0xe2,
- 0xbf, 0xb0, 0xc3, 0x44, 0xf1, 0x9d, 0x02, 0x9b, 0xba, 0x86, 0x47, 0x66, 0xf4, 0xd2, 0x74, 0x7c,
- 0xb9, 0xcc, 0x8e, 0x91, 0x43, 0x44, 0x27, 0x05, 0xc0, 0x91, 0xbf, 0xde, 0x83, 0x3c, 0xdb, 0x31,
- 0xcc, 0x22, 0x2f, 0xd6, 0x48, 0xbd, 0xd1, 0x0c, 0x9d, 0x40, 0x7e, 0x62, 0x3b, 0xc4, 0x97, 0xb3,
- 0x2c, 0x87, 0x28, 0x55, 0xe9, 0xb6, 0x43, 0x3a, 0xee, 0x84, 0x46, 0x59, 0xe4, 0xb0, 0xfa, 0x39,
- 0x54, 0xd8, 0x86, 0xe7, 0x73, 0xcb, 0x0c, 0xc8, 0xff, 0x6d, 0xdb, 0xff, 0xe4, 0xa0, 0x14, 0x7b,
- 0x92, 0xa4, 0x0b, 0xa9, 0xa4, 0x1f, 0x47, 0xfd, 0x80, 0xab, 0x7b, 0xff, 0xe6, 0x7e, 0xa9, 0x86,
- 0x80, 0x40, 0xf4, 0xed, 0x3f, 0x10, 0xa6, 0xa7, 0x1c, 0x66, 0x63, 0xa4, 0x42, 0xe5, 0xba, 0x88,
- 0x76, 0x70, 0xda, 0x84, 0x3e, 0x00, 0x98, 0x51, 0xcb, 0x9e, 0xd8, 0xc4, 0x32, 0x7c, 0x56, 0x00,
- 0x39, 0x5c, 0x8e, 0x2d, 0x03, 0x24, 0x87, 0xe5, 0x1e, 0x4a, 0xc8, 0x8a, 0xb4, 0x12, 0x4f, 0x43,
- 0x8f, 0xed, 0x5e, 0x9a, 0x8e, 0x1d, 0x2b, 0x24, 0x9e, 0x86, 0x5d, 0xcf, 0xa5, 0x5b, 0xe2, 0x2d,
- 0x31, 0xc0, 0x8e, 0x4b, 0xd3, 0xc2, 0x7d, 0x0c, 0xc5, 0xb8, 0x2b, 0x86, 0xf9, 0xdc, 0x52, 0xd2,
- 0x33, 0x32, 0x0e, 0x68, 0xd2, 0x6f, 0x22, 0x18, 0x52, 0xa0, 0x94, 0x94, 0x22, 0xb0, 0x9b, 0x26,
- 0xf3, 0xb0, 0x17, 0x27, 0x71, 0xb8, 0xbe, 0x5c, 0x51, 0x85, 0xa3, 0x3c, 0x4e, 0x42, 0xd3, 0xc3,
- 0xe3, 0x36, 0x80, 0xd1, 0x52, 0xae, 0xb2, 0x5a, 0x7c, 0x2f, 0xae, 0xc5, 0xc1, 0x05, 0xf5, 0x82,
- 0x4e, 0x7b, 0xb3, 0xa2, 0xb9, 0x44, 0x3f, 0x82, 0x42, 0xd3, 0xa1, 0xe3, 0x17, 0xb1, 0xd2, 0xef,
- 0x6c, 0xee, 0xc7, 0xec, 0xa9, 0x7c, 0x46, 0xc0, 0x30, 0x74, 0x7f, 0x39, 0x73, 0x6c, 0xf7, 0x85,
- 0x11, 0x98, 0xde, 0x94, 0x04, 0xf2, 0x1e, 0x6f, 0xf8, 0x91, 0x75, 0xc8, 0x8c, 0x3f, 0x17, 0xff,
- 0xf8, 0xcd, 0x61, 0xa6, 0xee, 0x42, 0x39, 0xd9, 0x27, 0x2c, 0x29, 0x3a, 0x99, 0xf8, 0x24, 0x60,
- 0xf9, 0xcf, 0xe1, 0x68, 0x96, 0x64, 0x35, 0xcb, 0x02, 0xe2, 0x59, 0x45, 0x20, 0x5e, 0x98, 0xfe,
- 0x05, 0xcb, 0x74, 0x15, 0xb3, 0x71, 0xa8, 0xe3, 0x97, 0xc4, 0x7c, 0x61, 0x30, 0x07, 0xcf, 0x73,
- 0x29, 0x34, 0x3c, 0x31, 0xfd, 0x8b, 0xe8, 0xbc, 0x5f, 0x42, 0x81, 0xf3, 0x8a, 0xbe, 0x80, 0xd2,
- 0x98, 0x2e, 0xdc, 0x60, 0xd3, 0xeb, 0xf7, 0xd2, 0xad, 0x82, 0x79, 0xa2, 0xc8, 0x12, 0x60, 0xfd,
- 0x14, 0x8a, 0x91, 0x0b, 0x3d, 0x4c, 0xfa, 0x98, 0xd8, 0xbc, 0x77, 0x8d, 0xc2, 0xed, 0xe6, 0x7f,
- 0x69, 0x3a, 0x0b, 0x7e, 0x79, 0x11, 0xf3, 0x49, 0xfd, 0x2f, 0x02, 0x14, 0x71, 0x98, 0x36, 0x3f,
- 0x48, 0x3d, 0x1b, 0xf9, 0xad, 0x67, 0x63, 0x23, 0xb0, 0xec, 0x96, 0xc0, 0x62, 0x8d, 0xe4, 0x52,
- 0x1a, 0xd9, 0x30, 0x27, 0xbe, 0x95, 0xb9, 0xfc, 0x5b, 0x98, 0x2b, 0xa4, 0x98, 0x7b, 0x08, 0xbb,
- 0x13, 0x8f, 0xce, 0xd8, 0xc3, 0x40, 0x3d, 0xd3, 0x5b, 0x46, 0xf5, 0xbc, 0x13, 0x5a, 0x87, 0xb1,
- 0xb1, 0x6e, 0x40, 0x09, 0x13, 0x7f, 0x4e, 0x5d, 0x9f, 0xdc, 0x7a, 0x6d, 0x04, 0xa2, 0x65, 0x06,
- 0x26, 0xbb, 0x74, 0x15, 0xb3, 0x31, 0x7a, 0x04, 0xe2, 0x98, 0x5a, 0xfc, 0xca, 0xbb, 0xe9, 0x1a,
- 0xd2, 0x3c, 0x8f, 0x7a, 0x2d, 0x6a, 0x11, 0xcc, 0x00, 0xf5, 0x39, 0x48, 0x6d, 0xfa, 0xd2, 0x75,
- 0xa8, 0x69, 0xf5, 0x3d, 0x3a, 0x0d, 0x1b, 0xf4, 0xad, 0x8d, 0xa6, 0x0d, 0xc5, 0x05, 0x6b, 0x45,
- 0x71, 0xab, 0x79, 0xb0, 0xdd, 0x1a, 0xae, 0x6f, 0xc4, 0xfb, 0x56, 0xac, 0xa7, 0x68, 0x69, 0xfd,
- 0x6f, 0x02, 0x28, 0xb7, 0xa3, 0x51, 0x07, 0x2a, 0x1c, 0x69, 0xa4, 0xfe, 0x49, 0x8e, 0xde, 0xe5,
- 0x20, 0xd6, 0x95, 0x60, 0x91, 0x8c, 0xdf, 0xfa, 0xa0, 0xa5, 0xf4, 0x9f, 0x7b, 0x37, 0xfd, 0x3f,
- 0x82, 0x9d, 0x51, 0x28, 0x98, 0xe4, 0xf9, 0x16, 0xd5, 0xdc, 0x51, 0xbe, 0x99, 0x95, 0x32, 0xb8,
- 0x3a, 0xe2, 0x4a, 0x62, 0xf6, 0x7a, 0x01, 0xc4, 0xbe, 0xed, 0x4e, 0xeb, 0x87, 0x90, 0x6f, 0x39,
- 0x94, 0x25, 0xac, 0xe0, 0x11, 0xd3, 0xa7, 0x6e, 0xcc, 0x23, 0x9f, 0x1d, 0xff, 0x35, 0x0b, 0x95,
- 0xd4, 0xaf, 0x15, 0x7a, 0x0c, 0xbb, 0xad, 0xee, 0xf9, 0x60, 0xa8, 0x61, 0xa3, 0xd5, 0xd3, 0x4f,
- 0x3b, 0x67, 0x52, 0x46, 0x39, 0x58, 0xad, 0x55, 0x79, 0xb6, 0x01, 0x6d, 0xff, 0x35, 0x1d, 0x42,
- 0xbe, 0xa3, 0xb7, 0xb5, 0xdf, 0x4a, 0x82, 0x72, 0x77, 0xb5, 0x56, 0xa5, 0x14, 0x90, 0x3f, 0x41,
- 0x9f, 0x40, 0x95, 0x01, 0x8c, 0xf3, 0x7e, 0xbb, 0x31, 0xd4, 0xa4, 0xac, 0xa2, 0xac, 0xd6, 0xea,
- 0xfe, 0x75, 0x5c, 0xc4, 0xf9, 0x87, 0x50, 0xc4, 0xda, 0x6f, 0xce, 0xb5, 0xc1, 0x50, 0xca, 0x29,
- 0xfb, 0xab, 0xb5, 0x8a, 0x52, 0xc0, 0x58, 0x35, 0x0f, 0xa1, 0x84, 0xb5, 0x41, 0xbf, 0xa7, 0x0f,
- 0x34, 0x49, 0x54, 0x7e, 0xb0, 0x5a, 0xab, 0x77, 0xb6, 0x50, 0x51, 0x95, 0xfe, 0x04, 0xf6, 0xda,
- 0xbd, 0xaf, 0xf4, 0x6e, 0xaf, 0xd1, 0x36, 0xfa, 0xb8, 0x77, 0x86, 0xb5, 0xc1, 0x40, 0xca, 0x2b,
- 0x87, 0xab, 0xb5, 0xfa, 0x7e, 0x0a, 0x7f, 0xa3, 0xe8, 0x3e, 0x00, 0xb1, 0xdf, 0xd1, 0xcf, 0xa4,
- 0x82, 0x72, 0x67, 0xb5, 0x56, 0xdf, 0x4b, 0x41, 0x43, 0x52, 0xc3, 0x88, 0x5b, 0xdd, 0xde, 0x40,
- 0x93, 0x8a, 0x37, 0x22, 0x66, 0x64, 0x1f, 0xff, 0x0e, 0xd0, 0xcd, 0x9f, 0x4f, 0xf4, 0x00, 0x44,
- 0xbd, 0xa7, 0x6b, 0x52, 0x86, 0xc7, 0x7f, 0x13, 0xa1, 0x53, 0x97, 0xa0, 0x3a, 0xe4, 0xba, 0x5f,
- 0x7f, 0x29, 0x09, 0xca, 0x0f, 0x57, 0x6b, 0xf5, 0xde, 0x4d, 0x50, 0xf7, 0xeb, 0x2f, 0x8f, 0x29,
- 0x54, 0xd2, 0x1b, 0xd7, 0xa1, 0xf4, 0x54, 0x1b, 0x36, 0xda, 0x8d, 0x61, 0x43, 0xca, 0xf0, 0x2b,
- 0xc5, 0xee, 0xa7, 0x24, 0x30, 0x99, 0x08, 0x0f, 0x20, 0xaf, 0x6b, 0xcf, 0x34, 0x2c, 0x09, 0xca,
- 0xde, 0x6a, 0xad, 0xee, 0xc4, 0x00, 0x9d, 0x5c, 0x12, 0x0f, 0xd5, 0xa0, 0xd0, 0xe8, 0x7e, 0xd5,
- 0x78, 0x3e, 0x90, 0xb2, 0x0a, 0x5a, 0xad, 0xd5, 0xdd, 0xd8, 0xdd, 0x70, 0x5e, 0x9a, 0x4b, 0xff,
- 0xf8, 0xbf, 0x02, 0x54, 0xd3, 0x0f, 0x2e, 0xaa, 0x81, 0x78, 0xda, 0xe9, 0x6a, 0xf1, 0x71, 0x69,
- 0x5f, 0x38, 0x46, 0x47, 0x50, 0x6e, 0x77, 0xb0, 0xd6, 0x1a, 0xf6, 0xf0, 0xf3, 0x38, 0x96, 0x34,
- 0xa8, 0x6d, 0x7b, 0xac, 0xc0, 0x97, 0xe8, 0x67, 0x50, 0x1d, 0x3c, 0x7f, 0xda, 0xed, 0xe8, 0xbf,
- 0x36, 0xd8, 0x8e, 0x59, 0xe5, 0xd1, 0x6a, 0xad, 0xde, 0xdf, 0x02, 0x93, 0xb9, 0x47, 0xc6, 0x66,
- 0x40, 0xac, 0x01, 0x7f, 0x44, 0x42, 0x67, 0x49, 0x40, 0x2d, 0xd8, 0x8b, 0x97, 0x6e, 0x0e, 0xcb,
- 0x29, 0x9f, 0xac, 0xd6, 0xea, 0x47, 0xdf, 0xbb, 0x3e, 0x39, 0xbd, 0x24, 0xa0, 0x07, 0x50, 0x8c,
- 0x36, 0x89, 0x2b, 0x29, 0xbd, 0x34, 0x5a, 0x70, 0xfc, 0x67, 0x01, 0xca, 0x49, 0xbb, 0x0a, 0x09,
- 0xd7, 0x7b, 0x86, 0x86, 0x71, 0x0f, 0xc7, 0x0c, 0x24, 0x4e, 0x9d, 0xb2, 0x21, 0xba, 0x0f, 0xc5,
- 0x33, 0x4d, 0xd7, 0x70, 0xa7, 0x15, 0x0b, 0x23, 0x81, 0x9c, 0x11, 0x97, 0x78, 0xf6, 0x18, 0x7d,
- 0x0c, 0x55, 0xbd, 0x67, 0x0c, 0xce, 0x5b, 0x4f, 0xe2, 0xd0, 0xd9, 0xf9, 0xa9, 0xad, 0x06, 0x8b,
- 0xf1, 0x05, 0xe3, 0xf3, 0x38, 0xd4, 0xd0, 0xb3, 0x46, 0xb7, 0xd3, 0xe6, 0xd0, 0x9c, 0x22, 0xaf,
- 0xd6, 0xea, 0xdd, 0x04, 0xda, 0xe1, 0x7f, 0x1e, 0x21, 0xf6, 0xd8, 0x82, 0xda, 0xf7, 0x37, 0x26,
- 0xa4, 0x42, 0xa1, 0xd1, 0xef, 0x6b, 0x7a, 0x3b, 0xbe, 0xfd, 0xc6, 0xd7, 0x98, 0xcf, 0x89, 0x6b,
- 0x85, 0x88, 0xd3, 0x1e, 0x3e, 0xd3, 0x86, 0xf1, 0xe5, 0x37, 0x88, 0x53, 0x1a, 0xbe, 0xe0, 0xcd,
- 0x83, 0xd7, 0xdf, 0xd5, 0x32, 0xdf, 0x7e, 0x57, 0xcb, 0xbc, 0xbe, 0xaa, 0x09, 0xdf, 0x5e, 0xd5,
- 0x84, 0x7f, 0x5c, 0xd5, 0x32, 0xff, 0xba, 0xaa, 0x09, 0xdf, 0xfc, 0xb3, 0x26, 0x8c, 0x0a, 0xac,
- 0x91, 0x7d, 0xf1, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0xad, 0x8a, 0xef, 0x7f, 0x8f, 0x0e, 0x00,
- 0x00,
+ 0x15, 0x17, 0x25, 0xea, 0xdf, 0x93, 0xb4, 0xe1, 0x8e, 0xed, 0x2d, 0xcb, 0x6c, 0x24, 0x5a, 0xb1,
+ 0xe3, 0xcd, 0x22, 0x59, 0xbb, 0x49, 0xda, 0xa2, 0x45, 0x5b, 0x40, 0x7f, 0xb8, 0x6b, 0xa1, 0x32,
+ 0xa5, 0x8e, 0xb4, 0x4e, 0x9d, 0x43, 0x09, 0x4a, 0x1c, 0x69, 0x09, 0x53, 0x1c, 0x95, 0xa4, 0xd6,
+ 0x56, 0x3e, 0x82, 0x3e, 0x41, 0x2f, 0x02, 0x02, 0xf4, 0x54, 0xa0, 0xc7, 0x7e, 0x08, 0x1f, 0x83,
+ 0x1e, 0x7a, 0xe8, 0xc1, 0x68, 0xb6, 0x97, 0x1e, 0xfb, 0x09, 0x8a, 0x82, 0x33, 0xa4, 0x44, 0xed,
+ 0xda, 0x81, 0x0f, 0x39, 0x71, 0xe6, 0xbd, 0xdf, 0xbc, 0x99, 0xf9, 0xcd, 0xef, 0xbd, 0x47, 0x28,
+ 0x8e, 0xc8, 0xfc, 0x64, 0xee, 0xd1, 0x80, 0xa2, 0x02, 0xfb, 0x8c, 0xa9, 0xa3, 0x7c, 0x3a, 0xb5,
+ 0x83, 0x8b, 0xc5, 0xe8, 0x64, 0x4c, 0x67, 0x0f, 0xa7, 0x74, 0x4a, 0x1f, 0x32, 0xcf, 0x68, 0x31,
+ 0x61, 0x33, 0x36, 0x61, 0x23, 0xbe, 0xb0, 0x3e, 0x87, 0xec, 0x63, 0xe2, 0x38, 0x14, 0xd5, 0xa0,
+ 0x64, 0x91, 0x4b, 0x7b, 0x4c, 0x0c, 0xd7, 0x9c, 0x11, 0x59, 0x50, 0x85, 0xa3, 0x22, 0x06, 0x6e,
+ 0xd2, 0xcd, 0x19, 0x09, 0x01, 0x63, 0xc7, 0x26, 0x6e, 0xc0, 0x01, 0x69, 0x0e, 0xe0, 0x26, 0x06,
+ 0xb8, 0x0f, 0x7b, 0x11, 0xe0, 0x92, 0x78, 0xbe, 0x4d, 0x5d, 0x39, 0xc3, 0x30, 0x15, 0x6e, 0x7d,
+ 0xca, 0x8d, 0x75, 0x1f, 0x72, 0x8f, 0x89, 0x69, 0x11, 0x0f, 0x7d, 0x0c, 0x62, 0xb0, 0x9c, 0xf3,
+ 0xbd, 0xf6, 0x3e, 0xbb, 0x73, 0x12, 0xdf, 0xe1, 0xe4, 0x09, 0xf1, 0x7d, 0x73, 0x4a, 0x86, 0xcb,
+ 0x39, 0xc1, 0x0c, 0x82, 0x7e, 0x03, 0xa5, 0x31, 0x9d, 0xcd, 0x3d, 0xe2, 0xb3, 0xc0, 0x69, 0xb6,
+ 0xe2, 0xf0, 0xc6, 0x8a, 0xd6, 0x16, 0x83, 0x93, 0x0b, 0xea, 0x0d, 0xa8, 0xb4, 0x9c, 0x85, 0x1f,
+ 0x10, 0xaf, 0x45, 0xdd, 0x89, 0x3d, 0x45, 0x8f, 0x20, 0x3f, 0xa1, 0x8e, 0x45, 0x3c, 0x5f, 0x16,
+ 0xd4, 0xcc, 0x51, 0xe9, 0x33, 0x69, 0x1b, 0xec, 0x94, 0x39, 0x9a, 0xe2, 0xab, 0xd7, 0xb5, 0x14,
+ 0x8e, 0x61, 0xf5, 0x3f, 0xa7, 0x21, 0xc7, 0x3d, 0xe8, 0x00, 0xd2, 0xb6, 0xc5, 0x29, 0x6a, 0xe6,
+ 0xae, 0x5e, 0xd7, 0xd2, 0x9d, 0x36, 0x4e, 0xdb, 0x16, 0xba, 0x0d, 0x59, 0xc7, 0x1c, 0x11, 0x27,
+ 0x22, 0x87, 0x4f, 0xd0, 0xfb, 0x50, 0xf4, 0x88, 0x69, 0x19, 0xd4, 0x75, 0x96, 0x8c, 0x92, 0x02,
+ 0x2e, 0x84, 0x86, 0x9e, 0xeb, 0x2c, 0xd1, 0xa7, 0x80, 0xec, 0xa9, 0x4b, 0x3d, 0x62, 0xcc, 0x89,
+ 0x37, 0xb3, 0xd9, 0x69, 0x7d, 0x59, 0x64, 0xa8, 0x7d, 0xee, 0xe9, 0x6f, 0x1d, 0xe8, 0x43, 0xa8,
+ 0x44, 0x70, 0x8b, 0x38, 0x24, 0x20, 0x72, 0x96, 0x21, 0xcb, 0xdc, 0xd8, 0x66, 0x36, 0xf4, 0x08,
+ 0x6e, 0x5b, 0xb6, 0x6f, 0x8e, 0x1c, 0x62, 0x04, 0x64, 0x36, 0x37, 0x6c, 0xd7, 0x22, 0x2f, 0x89,
+ 0x2f, 0xe7, 0x18, 0x16, 0x45, 0xbe, 0x21, 0x99, 0xcd, 0x3b, 0xdc, 0x83, 0x0e, 0x20, 0x37, 0x37,
+ 0x17, 0x3e, 0xb1, 0xe4, 0x3c, 0xc3, 0x44, 0xb3, 0x90, 0x25, 0xae, 0x00, 0x5f, 0x96, 0xae, 0xb3,
+ 0xd4, 0x66, 0x8e, 0x98, 0xa5, 0x08, 0x56, 0xff, 0x6f, 0x1a, 0x72, 0xdc, 0x83, 0x3e, 0xda, 0xb0,
+ 0x54, 0x6e, 0x1e, 0x84, 0xa8, 0x7f, 0xbe, 0xae, 0x15, 0xb8, 0xaf, 0xd3, 0x4e, 0xb0, 0x86, 0x40,
+ 0x4c, 0x28, 0x8a, 0x8d, 0xd1, 0x21, 0x14, 0x4d, 0xcb, 0x0a, 0x5f, 0x8f, 0xf8, 0x72, 0x46, 0xcd,
+ 0x1c, 0x15, 0xf1, 0xd6, 0x80, 0x7e, 0xbe, 0xab, 0x06, 0xf1, 0xba, 0x7e, 0xde, 0x26, 0x83, 0xf0,
+ 0x29, 0xc6, 0xc4, 0x8b, 0x14, 0x9c, 0x65, 0xfb, 0x15, 0x42, 0x03, 0xd3, 0xef, 0x5d, 0x28, 0xcf,
+ 0xcc, 0x97, 0x86, 0x4f, 0xfe, 0xb8, 0x20, 0xee, 0x98, 0x30, 0xba, 0x32, 0xb8, 0x34, 0x33, 0x5f,
+ 0x0e, 0x22, 0x13, 0xaa, 0x02, 0xd8, 0x6e, 0xe0, 0x51, 0x6b, 0x31, 0x26, 0x5e, 0xc4, 0x55, 0xc2,
+ 0x82, 0x7e, 0x0a, 0x05, 0x46, 0xb6, 0x61, 0x5b, 0x72, 0x41, 0x15, 0x8e, 0xc4, 0xa6, 0x12, 0x5d,
+ 0x3c, 0xcf, 0xa8, 0x66, 0xf7, 0x8e, 0x87, 0x38, 0xcf, 0xb0, 0x1d, 0x0b, 0xfd, 0x0a, 0x14, 0xff,
+ 0xb9, 0x1d, 0x3e, 0x14, 0x8f, 0x14, 0xd8, 0xd4, 0x35, 0x3c, 0x32, 0xa3, 0x97, 0xa6, 0xe3, 0xcb,
+ 0x45, 0xb6, 0x8d, 0x1c, 0x22, 0x3a, 0x09, 0x00, 0x8e, 0xfc, 0xf5, 0x1e, 0x64, 0x59, 0xc4, 0xf0,
+ 0x15, 0xb9, 0x58, 0xa3, 0xec, 0x8d, 0x66, 0xe8, 0x04, 0xb2, 0x13, 0xdb, 0x21, 0xbe, 0x9c, 0x66,
+ 0x6f, 0x88, 0x12, 0x4a, 0xb7, 0x1d, 0xd2, 0x71, 0x27, 0x34, 0x7a, 0x45, 0x0e, 0xab, 0x9f, 0x43,
+ 0x89, 0x05, 0x3c, 0x9f, 0x5b, 0x66, 0x40, 0x7e, 0xb0, 0xb0, 0x7f, 0x15, 0xa1, 0x10, 0x7b, 0x36,
+ 0x8f, 0x2e, 0x24, 0x1e, 0xfd, 0x38, 0xaa, 0x07, 0x3c, 0xbb, 0x0f, 0x6e, 0xc6, 0x4b, 0x14, 0x04,
+ 0x04, 0xa2, 0x6f, 0x7f, 0x4d, 0x58, 0x3e, 0x65, 0x30, 0x1b, 0x23, 0x15, 0x4a, 0xd7, 0x93, 0xa8,
+ 0x82, 0x93, 0x26, 0xf4, 0x01, 0xc0, 0x8c, 0x5a, 0xf6, 0xc4, 0x26, 0x96, 0xe1, 0x33, 0x01, 0x64,
+ 0x70, 0x31, 0xb6, 0x0c, 0x90, 0x1c, 0xca, 0x3d, 0x4c, 0x21, 0x2b, 0xca, 0x95, 0x78, 0x1a, 0x7a,
+ 0x6c, 0xf7, 0xd2, 0x74, 0xec, 0x38, 0x43, 0xe2, 0x69, 0x58, 0xf5, 0x5c, 0xba, 0x93, 0xbc, 0x05,
+ 0x06, 0xa8, 0xb8, 0x34, 0x99, 0xb8, 0x8f, 0x20, 0x1f, 0x57, 0xc5, 0xf0, 0x3d, 0x77, 0x32, 0xe9,
+ 0x29, 0x19, 0x07, 0x74, 0x53, 0x6f, 0x22, 0x18, 0x52, 0xa0, 0xb0, 0x91, 0x22, 0xb0, 0x93, 0x6e,
+ 0xe6, 0x61, 0x2d, 0xde, 0xdc, 0xc3, 0xf5, 0xe5, 0x92, 0x2a, 0x1c, 0x65, 0xf1, 0xe6, 0x6a, 0x7a,
+ 0xb8, 0xdd, 0x16, 0x30, 0x5a, 0xca, 0x65, 0xa6, 0xc5, 0xf7, 0x62, 0x2d, 0x0e, 0x2e, 0xa8, 0x17,
+ 0x74, 0xda, 0xdb, 0x15, 0xcd, 0x25, 0x7a, 0x08, 0x30, 0x72, 0xe8, 0xf8, 0xb9, 0xc1, 0x68, 0xad,
+ 0x84, 0x11, 0x9b, 0xd2, 0xd5, 0xeb, 0x5a, 0x19, 0x9b, 0x2f, 0x9a, 0xa1, 0x63, 0x60, 0x7f, 0x4d,
+ 0x70, 0x71, 0x14, 0x0f, 0xd1, 0x4f, 0x20, 0xc7, 0xec, 0x71, 0x69, 0xb8, 0xb5, 0xbd, 0x10, 0xb3,
+ 0x27, 0x04, 0x10, 0x01, 0x43, 0xae, 0xfc, 0xe5, 0xcc, 0xb1, 0xdd, 0xe7, 0x46, 0x60, 0x7a, 0x53,
+ 0x12, 0xc8, 0xfb, 0xbc, 0x43, 0x44, 0xd6, 0x21, 0x33, 0xfe, 0x52, 0xfc, 0xd3, 0x37, 0xb5, 0x54,
+ 0xdd, 0x85, 0xe2, 0x26, 0x4e, 0xa8, 0x41, 0x3a, 0x99, 0xf8, 0x24, 0x60, 0x82, 0xc9, 0xe0, 0x68,
+ 0xb6, 0x91, 0x41, 0x9a, 0x31, 0xc0, 0x65, 0x80, 0x40, 0xbc, 0x30, 0xfd, 0x0b, 0x26, 0x8d, 0x32,
+ 0x66, 0xe3, 0x30, 0xf1, 0x5f, 0x10, 0xf3, 0xb9, 0xc1, 0x1c, 0x5c, 0x18, 0x85, 0xd0, 0xf0, 0xd8,
+ 0xf4, 0x2f, 0xa2, 0xfd, 0x7e, 0x0d, 0x39, 0xfe, 0x10, 0xe8, 0x73, 0x28, 0x8c, 0xe9, 0xc2, 0x0d,
+ 0xb6, 0xcd, 0x61, 0x3f, 0x59, 0x5b, 0x98, 0x27, 0xba, 0xd9, 0x06, 0x58, 0x3f, 0x85, 0x7c, 0xe4,
+ 0x42, 0xf7, 0x37, 0x85, 0x4f, 0x6c, 0xde, 0xb9, 0xc6, 0xf9, 0x6e, 0xb7, 0xb8, 0x34, 0x9d, 0x05,
+ 0x3f, 0xbc, 0x88, 0xf9, 0xa4, 0xfe, 0x37, 0x01, 0xf2, 0x38, 0x7c, 0x67, 0x3f, 0x48, 0xf4, 0x99,
+ 0xec, 0x4e, 0x9f, 0xd9, 0x66, 0x64, 0x7a, 0x27, 0x23, 0xe3, 0xa4, 0xca, 0x24, 0x92, 0x6a, 0xcb,
+ 0x9c, 0xf8, 0x46, 0xe6, 0xb2, 0x6f, 0x60, 0x2e, 0x97, 0x60, 0xee, 0x3e, 0xec, 0x4d, 0x3c, 0x3a,
+ 0x63, 0x9d, 0x84, 0x7a, 0xa6, 0xb7, 0x8c, 0x12, 0xa0, 0x12, 0x5a, 0x87, 0xb1, 0xb1, 0x6e, 0x40,
+ 0x01, 0x13, 0x7f, 0x4e, 0x5d, 0x9f, 0xbc, 0xf5, 0xd8, 0x08, 0x44, 0xcb, 0x0c, 0x4c, 0x76, 0xe8,
+ 0x32, 0x66, 0x63, 0xf4, 0x00, 0xc4, 0x31, 0xb5, 0xf8, 0x91, 0xf7, 0x92, 0x1a, 0xd2, 0x3c, 0x8f,
+ 0x7a, 0x2d, 0x6a, 0x11, 0xcc, 0x00, 0xf5, 0x39, 0x48, 0x6d, 0xfa, 0xc2, 0x75, 0xa8, 0x69, 0xf5,
+ 0x3d, 0x3a, 0x0d, 0x2b, 0xfa, 0x5b, 0x2b, 0x53, 0x1b, 0xf2, 0x0b, 0x56, 0xbb, 0xe2, 0xda, 0x74,
+ 0x6f, 0xb7, 0x96, 0x5c, 0x0f, 0xc4, 0x0b, 0x5d, 0x9c, 0x80, 0xd1, 0xd2, 0xfa, 0x3f, 0x04, 0x50,
+ 0xde, 0x8e, 0x46, 0x1d, 0x28, 0x71, 0xa4, 0x91, 0xf8, 0x89, 0x39, 0x7a, 0x97, 0x8d, 0x58, 0x19,
+ 0x83, 0xc5, 0x66, 0xfc, 0xc6, 0x0e, 0x98, 0x28, 0x18, 0x99, 0x77, 0x2b, 0x18, 0x0f, 0xa0, 0xc2,
+ 0x33, 0x38, 0xee, 0xf7, 0xa2, 0x9a, 0x39, 0xca, 0x36, 0xd3, 0x52, 0x0a, 0x97, 0x47, 0x3c, 0x93,
+ 0x98, 0xbd, 0x9e, 0x03, 0xb1, 0x6f, 0xbb, 0xd3, 0x7a, 0x0d, 0xb2, 0x2d, 0x87, 0xb2, 0x07, 0xcb,
+ 0x79, 0xc4, 0xf4, 0xa9, 0x1b, 0xf3, 0xc8, 0x67, 0xc7, 0x7f, 0x4f, 0x43, 0x29, 0xf1, 0x2f, 0x86,
+ 0x1e, 0xc1, 0x5e, 0xab, 0x7b, 0x3e, 0x18, 0x6a, 0xd8, 0x68, 0xf5, 0xf4, 0xd3, 0xce, 0x99, 0x94,
+ 0x52, 0x0e, 0x57, 0x6b, 0x55, 0x9e, 0x6d, 0x41, 0xbb, 0xbf, 0x59, 0x35, 0xc8, 0x76, 0xf4, 0xb6,
+ 0xf6, 0x7b, 0x49, 0x50, 0x6e, 0xaf, 0xd6, 0xaa, 0x94, 0x00, 0xf2, 0x9e, 0xf5, 0x09, 0x94, 0x19,
+ 0xc0, 0x38, 0xef, 0xb7, 0x1b, 0x43, 0x4d, 0x4a, 0x2b, 0xca, 0x6a, 0xad, 0x1e, 0x5c, 0xc7, 0x45,
+ 0x9c, 0x7f, 0x08, 0x79, 0xac, 0xfd, 0xee, 0x5c, 0x1b, 0x0c, 0xa5, 0x8c, 0x72, 0xb0, 0x5a, 0xab,
+ 0x28, 0x01, 0x8c, 0xb3, 0xe6, 0x3e, 0x14, 0xb0, 0x36, 0xe8, 0xf7, 0xf4, 0x81, 0x26, 0x89, 0xca,
+ 0x8f, 0x56, 0x6b, 0xf5, 0xd6, 0x0e, 0x2a, 0x52, 0xe9, 0xcf, 0x60, 0xbf, 0xdd, 0xfb, 0x52, 0xef,
+ 0xf6, 0x1a, 0x6d, 0xa3, 0x8f, 0x7b, 0x67, 0x58, 0x1b, 0x0c, 0xa4, 0xac, 0x52, 0x5b, 0xad, 0xd5,
+ 0xf7, 0x13, 0xf8, 0x1b, 0xa2, 0xfb, 0x00, 0xc4, 0x7e, 0x47, 0x3f, 0x93, 0x72, 0xca, 0xad, 0xd5,
+ 0x5a, 0x7d, 0x2f, 0x01, 0x0d, 0x49, 0x0d, 0x6f, 0xdc, 0xea, 0xf6, 0x06, 0x9a, 0x94, 0xbf, 0x71,
+ 0x63, 0x46, 0xf6, 0xf1, 0x1f, 0x00, 0xdd, 0xfc, 0x5b, 0x45, 0xf7, 0x40, 0xd4, 0x7b, 0xba, 0x26,
+ 0xa5, 0xf8, 0xfd, 0x6f, 0x22, 0x74, 0xea, 0x12, 0x54, 0x87, 0x4c, 0xf7, 0xab, 0x2f, 0x24, 0x41,
+ 0xf9, 0xf1, 0x6a, 0xad, 0xde, 0xb9, 0x09, 0xea, 0x7e, 0xf5, 0xc5, 0x31, 0x85, 0x52, 0x32, 0x70,
+ 0x1d, 0x0a, 0x4f, 0xb4, 0x61, 0xa3, 0xdd, 0x18, 0x36, 0xa4, 0x14, 0x3f, 0x52, 0xec, 0x7e, 0x42,
+ 0x02, 0x93, 0x25, 0xe1, 0x21, 0x64, 0x75, 0xed, 0xa9, 0x86, 0x25, 0x41, 0xd9, 0x5f, 0xad, 0xd5,
+ 0x4a, 0x0c, 0xd0, 0xc9, 0x25, 0xf1, 0x50, 0x15, 0x72, 0x8d, 0xee, 0x97, 0x8d, 0x67, 0x03, 0x29,
+ 0xad, 0xa0, 0xd5, 0x5a, 0xdd, 0x8b, 0xdd, 0x0d, 0xe7, 0x85, 0xb9, 0xf4, 0x8f, 0xff, 0x27, 0x40,
+ 0x39, 0xd9, 0xa1, 0x51, 0x15, 0xc4, 0xd3, 0x4e, 0x57, 0x8b, 0xb7, 0x4b, 0xfa, 0xc2, 0x31, 0x3a,
+ 0x82, 0x62, 0xbb, 0x83, 0xb5, 0xd6, 0xb0, 0x87, 0x9f, 0xc5, 0x77, 0x49, 0x82, 0xda, 0xb6, 0xc7,
+ 0x04, 0xbe, 0x44, 0xbf, 0x80, 0xf2, 0xe0, 0xd9, 0x93, 0x6e, 0x47, 0xff, 0xad, 0xc1, 0x22, 0xa6,
+ 0x95, 0x07, 0xab, 0xb5, 0x7a, 0x77, 0x07, 0x4c, 0xe6, 0x1e, 0x19, 0x9b, 0x01, 0xb1, 0x06, 0xbc,
+ 0x89, 0x84, 0xce, 0x82, 0x80, 0x5a, 0xb0, 0x1f, 0x2f, 0xdd, 0x6e, 0x96, 0x51, 0x3e, 0x59, 0xad,
+ 0xd5, 0x8f, 0xbe, 0x77, 0xfd, 0x66, 0xf7, 0x82, 0x80, 0xee, 0x41, 0x3e, 0x0a, 0x12, 0x2b, 0x29,
+ 0xb9, 0x34, 0x5a, 0x70, 0xfc, 0x17, 0x01, 0x8a, 0x9b, 0x72, 0x15, 0x12, 0xae, 0xf7, 0x0c, 0x0d,
+ 0xe3, 0x1e, 0x8e, 0x19, 0xd8, 0x38, 0x75, 0xca, 0x86, 0xe8, 0x2e, 0xe4, 0xcf, 0x34, 0x5d, 0xc3,
+ 0x9d, 0x56, 0x9c, 0x18, 0x1b, 0xc8, 0x19, 0x71, 0x89, 0x67, 0x8f, 0xd1, 0xc7, 0x50, 0xd6, 0x7b,
+ 0xc6, 0xe0, 0xbc, 0xf5, 0x38, 0xbe, 0x3a, 0xdb, 0x3f, 0x11, 0x6a, 0xb0, 0x18, 0x5f, 0x30, 0x3e,
+ 0x8f, 0xc3, 0x1c, 0x7a, 0xda, 0xe8, 0x76, 0xda, 0x1c, 0x9a, 0x51, 0xe4, 0xd5, 0x5a, 0xbd, 0xbd,
+ 0x81, 0x76, 0xf8, 0xaf, 0x4a, 0x88, 0x3d, 0xb6, 0xa0, 0xfa, 0xfd, 0x85, 0x09, 0xa9, 0x90, 0x6b,
+ 0xf4, 0xfb, 0x9a, 0xde, 0x8e, 0x4f, 0xbf, 0xf5, 0x35, 0xe6, 0x73, 0xe2, 0x5a, 0x21, 0xe2, 0xb4,
+ 0x87, 0xcf, 0xb4, 0x61, 0x7c, 0xf8, 0x2d, 0xe2, 0x94, 0x86, 0x1d, 0xbc, 0x79, 0xf8, 0xea, 0xbb,
+ 0x6a, 0xea, 0xdb, 0xef, 0xaa, 0xa9, 0x57, 0x57, 0x55, 0xe1, 0xdb, 0xab, 0xaa, 0xf0, 0xaf, 0xab,
+ 0x6a, 0xea, 0x3f, 0x57, 0x55, 0xe1, 0x9b, 0x7f, 0x57, 0x85, 0x51, 0x8e, 0x15, 0xb2, 0xcf, 0xff,
+ 0x1f, 0x00, 0x00, 0xff, 0xff, 0x0d, 0x36, 0x7c, 0x9b, 0xc0, 0x0e, 0x00, 0x00,
}
diff --git a/lib/protocol/bep.proto b/lib/protocol/bep.proto
index fc6c9d9c7..008801e57 100644
--- a/lib/protocol/bep.proto
+++ b/lib/protocol/bep.proto
@@ -105,6 +105,7 @@ message FileInfo {
bool no_permissions = 8;
Vector version = 9 [(gogoproto.nullable) = false];
int64 sequence = 10;
+ int32 block_size = 13 [(gogoproto.customname) = "RawBlockSize"];
repeated BlockInfo Blocks = 16 [(gogoproto.nullable) = false];
string symlink_target = 17;
diff --git a/lib/protocol/bep_extensions.go b/lib/protocol/bep_extensions.go
index db31f4522..5dde3ea9c 100644
--- a/lib/protocol/bep_extensions.go
+++ b/lib/protocol/bep_extensions.go
@@ -14,16 +14,11 @@ import (
"time"
"github.com/syncthing/syncthing/lib/rand"
- "github.com/syncthing/syncthing/lib/sha256"
)
const (
SyntheticDirectorySize = 128
-)
-
-var (
- sha256OfEmptyBlock = sha256.Sum256(make([]byte, BlockSize))
- HelloMessageMagic = uint32(0x2EA7D90B)
+ HelloMessageMagic = uint32(0x2EA7D90B)
)
func (m Hello) Magic() uint32 {
@@ -36,8 +31,8 @@ func (f FileInfo) String() string {
return fmt.Sprintf("Directory{Name:%q, Sequence:%d, Permissions:0%o, ModTime:%v, Version:%v, Deleted:%v, Invalid:%v, NoPermissions:%v}",
f.Name, f.Sequence, f.Permissions, f.ModTime(), f.Version, f.Deleted, f.Invalid, f.NoPermissions)
case FileInfoTypeFile:
- return fmt.Sprintf("File{Name:%q, Sequence:%d, Permissions:0%o, ModTime:%v, Version:%v, Length:%d, Deleted:%v, Invalid:%v, NoPermissions:%v, Blocks:%v}",
- f.Name, f.Sequence, f.Permissions, f.ModTime(), f.Version, f.Size, f.Deleted, f.Invalid, f.NoPermissions, f.Blocks)
+ return fmt.Sprintf("File{Name:%q, Sequence:%d, Permissions:0%o, ModTime:%v, Version:%v, Length:%d, Deleted:%v, Invalid:%v, NoPermissions:%v, BlockSize:%d, Blocks:%v}",
+ f.Name, f.Sequence, f.Permissions, f.ModTime(), f.Version, f.Size, f.Deleted, f.Invalid, f.NoPermissions, f.RawBlockSize, f.Blocks)
case FileInfoTypeSymlink, FileInfoTypeDeprecatedSymlinkDirectory, FileInfoTypeDeprecatedSymlinkFile:
return fmt.Sprintf("Symlink{Name:%q, Type:%v, Sequence:%d, Version:%v, Deleted:%v, Invalid:%v, NoPermissions:%v, SymlinkTarget:%q}",
f.Name, f.Type, f.Sequence, f.Version, f.Deleted, f.Invalid, f.NoPermissions, f.SymlinkTarget)
@@ -81,6 +76,13 @@ func (f FileInfo) FileSize() int64 {
return f.Size
}
+func (f FileInfo) BlockSize() int {
+ if f.RawBlockSize == 0 {
+ return MinBlockSize
+ }
+ return int(f.RawBlockSize)
+}
+
func (f FileInfo) FileName() string {
return f.Name
}
@@ -204,7 +206,10 @@ func (b BlockInfo) String() string {
// IsEmpty returns true if the block is a full block of zeroes.
func (b BlockInfo) IsEmpty() bool {
- return b.Size == BlockSize && bytes.Equal(b.Hash, sha256OfEmptyBlock[:])
+ if v, ok := sha256OfEmptyBlock[int(b.Size)]; ok {
+ return bytes.Equal(b.Hash, v[:])
+ }
+ return false
}
type IndexID uint64
diff --git a/lib/protocol/protocol.go b/lib/protocol/protocol.go
index 22275a57c..870fe2102 100644
--- a/lib/protocol/protocol.go
+++ b/lib/protocol/protocol.go
@@ -3,6 +3,7 @@
package protocol
import (
+ "crypto/sha256"
"encoding/binary"
"errors"
"fmt"
@@ -16,13 +17,51 @@ import (
)
const (
- // BlockSize is the standard data block size (128 KiB)
- BlockSize = 128 << 10
+ // Shifts
+ KiB = 10
+ MiB = 20
+ GiB = 30
+)
+const (
// MaxMessageLen is the largest message size allowed on the wire. (500 MB)
MaxMessageLen = 500 * 1000 * 1000
+
+ // MinBlockSize is the minimum block size allowed
+ MinBlockSize = 128 << KiB
+
+ // MaxBlockSize is the maximum block size allowed
+ MaxBlockSize = 16 << MiB
+
+ // DesiredPerFileBlocks is the number of blocks we aim for per file
+ DesiredPerFileBlocks = 2000
)
+// BlockSizes is the list of valid block sizes, from min to max
+var BlockSizes []int
+
+// For each block size, the hash of a block of all zeroes
+var sha256OfEmptyBlock = make(map[int][sha256.Size]byte)
+
+func init() {
+ for blockSize := MinBlockSize; blockSize <= MaxBlockSize; blockSize *= 2 {
+ BlockSizes = append(BlockSizes, blockSize)
+ sha256OfEmptyBlock[blockSize] = sha256.Sum256(make([]byte, blockSize))
+ }
+}
+
+// BlockSize returns the block size to use for the given file size
+func BlockSize(fileSize int64) int {
+ var blockSize int
+ for _, blockSize = range BlockSizes {
+ if fileSize < int64(DesiredPerFileBlocks*blockSize) {
+ break
+ }
+ }
+
+ return blockSize
+}
+
const (
stateInitial = iota
stateReady
@@ -158,7 +197,7 @@ func NewConnection(deviceID DeviceID, reader io.Reader, writer io.Writer, receiv
awaiting: make(map[int32]chan asyncResult),
outbox: make(chan asyncMessage),
closed: make(chan struct{}),
- pool: bufferPool{minSize: BlockSize},
+ pool: bufferPool{minSize: MinBlockSize},
compression: compress,
}
@@ -533,7 +572,7 @@ func checkFilename(name string) error {
func (c *rawConnection) handleRequest(req Request) {
size := int(req.Size)
- usePool := size <= BlockSize
+ usePool := size <= MaxBlockSize
var buf []byte
var done chan struct{}
diff --git a/lib/protocol/protocol_test.go b/lib/protocol/protocol_test.go
index f19710ba9..bccff8d52 100644
--- a/lib/protocol/protocol_test.go
+++ b/lib/protocol/protocol_test.go
@@ -11,7 +11,6 @@ import (
"strings"
"testing"
"testing/quick"
- "time"
"encoding/hex"
@@ -234,48 +233,6 @@ func testMarshal(t *testing.T, prefix string, m1, m2 message) bool {
return true
}
-func TestMarshalledIndexMessageSize(t *testing.T) {
- // We should be able to handle a 1 TiB file without
- // blowing the default max message size.
-
- if testing.Short() {
- t.Skip("this test requires a lot of memory")
- return
- }
-
- const (
- maxMessageSize = MaxMessageLen
- fileSize = 1 << 40
- blockSize = BlockSize
- )
-
- f := FileInfo{
- Name: "a normal length file name withoout any weird stuff.txt",
- Type: FileInfoTypeFile,
- Size: fileSize,
- Permissions: 0666,
- ModifiedS: time.Now().Unix(),
- Version: Vector{Counters: []Counter{{ID: 1 << 60, Value: 1}, {ID: 2 << 60, Value: 1}}},
- Blocks: make([]BlockInfo, fileSize/blockSize),
- }
-
- for i := 0; i < fileSize/blockSize; i++ {
- f.Blocks[i].Offset = int64(i) * blockSize
- f.Blocks[i].Size = blockSize
- f.Blocks[i].Hash = []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 20, 1, 2, 3, 4, 5, 6, 7, 8, 9, 30, 1, 2}
- }
-
- idx := Index{
- Folder: "some folder ID",
- Files: []FileInfo{f},
- }
-
- msgSize := idx.ProtoSize()
- if msgSize > maxMessageSize {
- t.Errorf("Message size %d bytes is larger than max %d", msgSize, maxMessageSize)
- }
-}
-
func TestLZ4Compression(t *testing.T) {
c := new(rawConnection)
@@ -400,3 +357,36 @@ func TestCheckConsistency(t *testing.T) {
}
}
}
+
+func TestBlockSize(t *testing.T) {
+ cases := []struct {
+ fileSize int64
+ blockSize int
+ }{
+ {1 << KiB, 128 << KiB},
+ {1 << MiB, 128 << KiB},
+ {499 << MiB, 256 << KiB},
+ {500 << MiB, 512 << KiB},
+ {501 << MiB, 512 << KiB},
+ {1 << GiB, 1 << MiB},
+ {2 << GiB, 2 << MiB},
+ {3 << GiB, 2 << MiB},
+ {500 << GiB, 16 << MiB},
+ {50000 << GiB, 16 << MiB},
+ }
+
+ for _, tc := range cases {
+ size := BlockSize(tc.fileSize)
+ if size != tc.blockSize {
+ t.Errorf("BlockSize(%d), size=%d, expected %d", tc.fileSize, size, tc.blockSize)
+ }
+ }
+}
+
+var blockSize int
+
+func BenchmarkBlockSize(b *testing.B) {
+ for i := 0; i < b.N; i++ {
+ blockSize = BlockSize(16 << 30)
+ }
+}
diff --git a/lib/scanner/blockqueue.go b/lib/scanner/blockqueue.go
index e1ef4be1f..2cfeec84f 100644
--- a/lib/scanner/blockqueue.go
+++ b/lib/scanner/blockqueue.go
@@ -63,7 +63,6 @@ func HashFile(ctx context.Context, fs fs.Filesystem, path string, blockSize int,
// is closed and all items handled.
type parallelHasher struct {
fs fs.Filesystem
- blockSize int
workers int
outbox chan<- protocol.FileInfo
inbox <-chan protocol.FileInfo
@@ -73,10 +72,9 @@ type parallelHasher struct {
wg sync.WaitGroup
}
-func newParallelHasher(ctx context.Context, fs fs.Filesystem, blockSize, workers int, outbox chan<- protocol.FileInfo, inbox <-chan protocol.FileInfo, counter Counter, done chan<- struct{}, useWeakHashes bool) {
+func newParallelHasher(ctx context.Context, fs fs.Filesystem, workers int, outbox chan<- protocol.FileInfo, inbox <-chan protocol.FileInfo, counter Counter, done chan<- struct{}, useWeakHashes bool) {
ph := ¶llelHasher{
fs: fs,
- blockSize: blockSize,
workers: workers,
outbox: outbox,
inbox: inbox,
@@ -108,7 +106,7 @@ func (ph *parallelHasher) hashFiles(ctx context.Context) {
panic("Bug. Asked to hash a directory or a deleted file.")
}
- blocks, err := HashFile(ctx, ph.fs, f.Name, ph.blockSize, ph.counter, ph.useWeakHashes)
+ blocks, err := HashFile(ctx, ph.fs, f.Name, f.BlockSize(), ph.counter, ph.useWeakHashes)
if err != nil {
l.Debugln("hash error:", f.Name, err)
continue
diff --git a/lib/scanner/blocks_test.go b/lib/scanner/blocks_test.go
index 8d706dc00..311dbdd72 100644
--- a/lib/scanner/blocks_test.go
+++ b/lib/scanner/blocks_test.go
@@ -125,7 +125,7 @@ func TestAdler32Variants(t *testing.T) {
}
// protocol block sized data
- data := make([]byte, protocol.BlockSize)
+ data := make([]byte, protocol.MinBlockSize)
for i := 0; i < 5; i++ {
rand.Read(data)
if !checkFn(data) {
diff --git a/lib/scanner/infinitefs_test.go b/lib/scanner/virtualfs_test.go
similarity index 77%
rename from lib/scanner/infinitefs_test.go
rename to lib/scanner/virtualfs_test.go
index 6d850fefe..8445d8d61 100644
--- a/lib/scanner/infinitefs_test.go
+++ b/lib/scanner/virtualfs_test.go
@@ -55,6 +55,48 @@ func (i infiniteFS) Open(name string) (fs.File, error) {
return &fakeFile{name, i.filesize, 0}, nil
}
+type singleFileFS struct {
+ fs.Filesystem
+ name string
+ filesize int64
+}
+
+func (s singleFileFS) Lstat(name string) (fs.FileInfo, error) {
+ switch name {
+ case ".":
+ return fakeInfo{".", 0}, nil
+ case s.name:
+ return fakeInfo{s.name, s.filesize}, nil
+ default:
+ return nil, errors.New("no such file")
+ }
+}
+
+func (s singleFileFS) Stat(name string) (fs.FileInfo, error) {
+ switch name {
+ case ".":
+ return fakeInfo{".", 0}, nil
+ case s.name:
+ return fakeInfo{s.name, s.filesize}, nil
+ default:
+ return nil, errors.New("no such file")
+ }
+}
+
+func (s singleFileFS) DirNames(name string) ([]string, error) {
+ if name != "." {
+ return nil, errors.New("no such file")
+ }
+ return []string{s.name}, nil
+}
+
+func (s singleFileFS) Open(name string) (fs.File, error) {
+ if name != s.name {
+ return nil, errors.New("no such file")
+ }
+ return &fakeFile{s.name, s.filesize, 0}, nil
+}
+
type fakeInfo struct {
name string
size int64
diff --git a/lib/scanner/walk.go b/lib/scanner/walk.go
index 6f958e9dc..0be3fde05 100644
--- a/lib/scanner/walk.go
+++ b/lib/scanner/walk.go
@@ -42,8 +42,6 @@ type Config struct {
Folder string
// Limit walking to these paths within Dir, or no limit if Sub is empty
Subs []string
- // BlockSize controls the size of the block used when hashing.
- BlockSize int
// If Matcher is not nil, it is used to identify files to ignore which were specified by the user.
Matcher *ignore.Matcher
// Number of hours to keep temporary files for
@@ -68,6 +66,8 @@ type Config struct {
ProgressTickIntervalS int
// Whether or not we should also compute weak hashes
UseWeakHashes bool
+ // Whether to use large blocks for large files or the old standard of 128KiB for everything.
+ UseLargeBlocks bool
}
type CurrentFiler interface {
@@ -98,7 +98,7 @@ type walker struct {
// Walk returns the list of files found in the local folder by scanning the
// file system. Files are blockwise hashed.
func (w *walker) walk(ctx context.Context) chan protocol.FileInfo {
- l.Debugln("Walk", w.Subs, w.BlockSize, w.Matcher)
+ l.Debugln("Walk", w.Subs, w.Matcher)
toHashChan := make(chan protocol.FileInfo)
finishedChan := make(chan protocol.FileInfo)
@@ -120,7 +120,7 @@ func (w *walker) walk(ctx context.Context) chan protocol.FileInfo {
// We're not required to emit scan progress events, just kick off hashers,
// and feed inputs directly from the walker.
if w.ProgressTickIntervalS < 0 {
- newParallelHasher(ctx, w.Filesystem, w.BlockSize, w.Hashers, finishedChan, toHashChan, nil, nil, w.UseWeakHashes)
+ newParallelHasher(ctx, w.Filesystem, w.Hashers, finishedChan, toHashChan, nil, nil, w.UseWeakHashes)
return finishedChan
}
@@ -151,7 +151,7 @@ func (w *walker) walk(ctx context.Context) chan protocol.FileInfo {
done := make(chan struct{})
progress := newByteCounter()
- newParallelHasher(ctx, w.Filesystem, w.BlockSize, w.Hashers, finishedChan, realToHashChan, progress, done, w.UseWeakHashes)
+ newParallelHasher(ctx, w.Filesystem, w.Hashers, finishedChan, realToHashChan, progress, done, w.UseWeakHashes)
// A routine which actually emits the FolderScanProgress events
// every w.ProgressTicker ticks, until the hasher routines terminate.
@@ -161,7 +161,7 @@ func (w *walker) walk(ctx context.Context) chan protocol.FileInfo {
for {
select {
case <-done:
- l.Debugln("Walk progress done", w.Folder, w.Subs, w.BlockSize, w.Matcher)
+ l.Debugln("Walk progress done", w.Folder, w.Subs, w.Matcher)
ticker.Stop()
return
case <-ticker.C:
@@ -285,32 +285,52 @@ func (w *walker) walkRegular(ctx context.Context, relPath string, info fs.FileIn
curMode |= 0111
}
- cf, ok := w.CurrentFiler.CurrentFile(relPath)
+ blockSize := protocol.MinBlockSize
+ curFile, hasCurFile := w.CurrentFiler.CurrentFile(relPath)
+
+ if w.UseLargeBlocks {
+ blockSize = protocol.BlockSize(info.Size())
+
+ if hasCurFile {
+ // Check if we should retain current block size.
+ curBlockSize := curFile.BlockSize()
+ if blockSize > curBlockSize && blockSize/curBlockSize <= 2 {
+ // New block size is larger, but not more than twice larger.
+ // Retain.
+ blockSize = curBlockSize
+ } else if curBlockSize > blockSize && curBlockSize/blockSize <= 2 {
+ // Old block size is larger, but not more than twice larger.
+ // Retain.
+ blockSize = curBlockSize
+ }
+ }
+ }
f := protocol.FileInfo{
Name: relPath,
Type: protocol.FileInfoTypeFile,
- Version: cf.Version.Update(w.ShortID),
+ Version: curFile.Version.Update(w.ShortID),
Permissions: curMode & uint32(maskModePerm),
NoPermissions: w.IgnorePerms,
ModifiedS: info.ModTime().Unix(),
ModifiedNs: int32(info.ModTime().Nanosecond()),
ModifiedBy: w.ShortID,
Size: info.Size(),
+ RawBlockSize: int32(blockSize),
}
- if ok {
- if cf.IsEquivalent(f, w.IgnorePerms, true) {
+ if hasCurFile {
+ if curFile.IsEquivalent(f, w.IgnorePerms, true) {
return nil
}
- if cf.Invalid {
+ if curFile.Invalid {
// We do not want to override the global version with the file we
// currently have. Keeping only our local counter makes sure we are in
// conflict with any other existing versions, which will be resolved by
// the normal pulling mechanisms.
f.Version = f.Version.DropOthers(w.ShortID)
}
- l.Debugln("rescan:", cf, info.ModTime().Unix(), info.Mode()&fs.ModePerm)
+ l.Debugln("rescan:", curFile, info.ModTime().Unix(), info.Mode()&fs.ModePerm)
}
l.Debugln("to hash:", relPath, f)
diff --git a/lib/scanner/walk_test.go b/lib/scanner/walk_test.go
index 33e2308f3..80ad3f5d6 100644
--- a/lib/scanner/walk_test.go
+++ b/lib/scanner/walk_test.go
@@ -65,7 +65,6 @@ func TestWalkSub(t *testing.T) {
fchan := Walk(context.TODO(), Config{
Filesystem: fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata"),
Subs: []string{"dir2"},
- BlockSize: 128 * 1024,
Matcher: ignores,
Hashers: 2,
})
@@ -98,7 +97,6 @@ func TestWalk(t *testing.T) {
fchan := Walk(context.TODO(), Config{
Filesystem: fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata"),
- BlockSize: 128 * 1024,
Matcher: ignores,
Hashers: 2,
})
@@ -221,11 +219,11 @@ func TestNormalization(t *testing.T) {
// make sure it all gets done. In production, things will be correct
// eventually...
- _, err := walkDir(fs, "testdata/normalization")
+ _, err := walkDir(fs, "testdata/normalization", nil)
if err != nil {
t.Fatal(err)
}
- tmp, err := walkDir(fs, "testdata/normalization")
+ tmp, err := walkDir(fs, "testdata/normalization", nil)
if err != nil {
t.Fatal(err)
}
@@ -272,7 +270,7 @@ func TestWalkSymlinkUnix(t *testing.T) {
for _, path := range []string{".", "link"} {
// Scan it
- files, _ := walkDir(fs.NewFilesystem(fs.FilesystemTypeBasic, "_symlinks"), path)
+ files, _ := walkDir(fs.NewFilesystem(fs.FilesystemTypeBasic, "_symlinks"), path, nil)
// Verify that we got one symlink and with the correct attributes
if len(files) != 1 {
@@ -303,7 +301,7 @@ func TestWalkSymlinkWindows(t *testing.T) {
for _, path := range []string{".", "link"} {
// Scan it
- files, _ := walkDir(fs.NewFilesystem(fs.FilesystemTypeBasic, "_symlinks"), path)
+ files, _ := walkDir(fs.NewFilesystem(fs.FilesystemTypeBasic, "_symlinks"), path, nil)
// Verify that we got zero symlinks
if len(files) != 0 {
@@ -332,7 +330,7 @@ func TestWalkRootSymlink(t *testing.T) {
}
// Scan it
- files, err := walkDir(fs.NewFilesystem(fs.FilesystemTypeBasic, link), ".")
+ files, err := walkDir(fs.NewFilesystem(fs.FilesystemTypeBasic, link), ".", nil)
if err != nil {
t.Fatal("Expected no error when root folder path is provided via a symlink: " + err.Error())
}
@@ -342,13 +340,83 @@ func TestWalkRootSymlink(t *testing.T) {
}
}
-func walkDir(fs fs.Filesystem, dir string) ([]protocol.FileInfo, error) {
+func TestBlocksizeHysteresis(t *testing.T) {
+ // Verify that we select the right block size in the presence of old
+ // file information.
+
+ sf := fs.NewWalkFilesystem(&singleFileFS{
+ name: "testfile.dat",
+ filesize: 500 << 20, // 500 MiB
+ })
+
+ current := make(fakeCurrentFiler)
+
+ runTest := func(expectedBlockSize int) {
+ files, err := walkDir(sf, ".", current)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(files) != 1 {
+ t.Fatalf("expected one file, not %d", len(files))
+ }
+ if s := files[0].BlockSize(); s != expectedBlockSize {
+ t.Fatalf("incorrect block size %d != expected %d", s, expectedBlockSize)
+ }
+ }
+
+ // Scan with no previous knowledge. We should get a 512 KiB block size.
+
+ runTest(512 << 10)
+
+ // Scan on the assumption that previous size was 256 KiB. Retain 256 KiB
+ // block size.
+
+ current["testfile.dat"] = protocol.FileInfo{
+ Name: "testfile.dat",
+ Size: 500 << 20,
+ RawBlockSize: 256 << 10,
+ }
+ runTest(256 << 10)
+
+ // Scan on the assumption that previous size was 1 MiB. Retain 1 MiB
+ // block size.
+
+ current["testfile.dat"] = protocol.FileInfo{
+ Name: "testfile.dat",
+ Size: 500 << 20,
+ RawBlockSize: 1 << 20,
+ }
+ runTest(1 << 20)
+
+ // Scan on the assumption that previous size was 128 KiB. Move to 512
+ // KiB because the difference is large.
+
+ current["testfile.dat"] = protocol.FileInfo{
+ Name: "testfile.dat",
+ Size: 500 << 20,
+ RawBlockSize: 128 << 10,
+ }
+ runTest(512 << 10)
+
+ // Scan on the assumption that previous size was 2 MiB. Move to 512
+ // KiB because the difference is large.
+
+ current["testfile.dat"] = protocol.FileInfo{
+ Name: "testfile.dat",
+ Size: 500 << 20,
+ RawBlockSize: 2 << 20,
+ }
+ runTest(512 << 10)
+}
+
+func walkDir(fs fs.Filesystem, dir string, cfiler CurrentFiler) ([]protocol.FileInfo, error) {
fchan := Walk(context.TODO(), Config{
- Filesystem: fs,
- Subs: []string{dir},
- BlockSize: 128 * 1024,
- AutoNormalize: true,
- Hashers: 2,
+ Filesystem: fs,
+ Subs: []string{dir},
+ AutoNormalize: true,
+ Hashers: 2,
+ UseLargeBlocks: true,
+ CurrentFiler: cfiler,
})
var tmp []protocol.FileInfo
@@ -410,7 +478,7 @@ func BenchmarkHashFile(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
- if _, err := HashFile(context.TODO(), fs.NewFilesystem(fs.FilesystemTypeBasic, ""), testdataName, protocol.BlockSize, nil, true); err != nil {
+ if _, err := HashFile(context.TODO(), fs.NewFilesystem(fs.FilesystemTypeBasic, ""), testdataName, protocol.MinBlockSize, nil, true); err != nil {
b.Fatal(err)
}
}
@@ -451,7 +519,6 @@ func TestStopWalk(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
fchan := Walk(ctx, Config{
Filesystem: fs,
- BlockSize: 128 * 1024,
Hashers: numHashers,
ProgressTickIntervalS: -1, // Don't attempt to build the full list of files before starting to scan...
})
@@ -513,7 +580,7 @@ func TestIssue4799(t *testing.T) {
}
fd.Close()
- files, err := walkDir(fs, "/foo")
+ files, err := walkDir(fs, "/foo", nil)
if err != nil {
t.Fatal(err)
}
@@ -540,7 +607,6 @@ func TestIssue4841(t *testing.T) {
fchan := Walk(context.TODO(), Config{
Filesystem: fs,
Subs: nil,
- BlockSize: 128 * 1024,
AutoNormalize: true,
Hashers: 2,
CurrentFiler: fakeCurrentFiler{
diff --git a/test/h1/config.xml b/test/h1/config.xml
index 80ccec9c3..418d253a5 100644
--- a/test/h1/config.xml
+++ b/test/h1/config.xml
@@ -19,6 +19,7 @@
false
25
.stfolder
+ true
basic
@@ -38,6 +39,7 @@
false
25
.stfolder
+ true
tcp://127.0.0.1:22004
diff --git a/test/h2/config.xml b/test/h2/config.xml
index cf25c4c7d..a5d21e274 100644
--- a/test/h2/config.xml
+++ b/test/h2/config.xml
@@ -18,6 +18,7 @@
false
25
.stfolder
+ true
basic
@@ -37,6 +38,7 @@
false
25
.stfolder
+ true
basic
@@ -56,6 +58,7 @@
false
25
.stfolder
+ true
tcp://127.0.0.1:22001
diff --git a/test/h3/config.xml b/test/h3/config.xml
index 6eca4321a..a204db7d4 100644
--- a/test/h3/config.xml
+++ b/test/h3/config.xml
@@ -20,6 +20,7 @@
false
25
.stfolder
+ true
basic
@@ -39,6 +40,7 @@
false
25
.stfolder
+ true
tcp://127.0.0.1:22001