all: Implement variable sized blocks (fixes #4807)

cmd/stfileinfo/main.go

@@ -68,8 +68,8 @@ func main() {
 		}
 
 		blockSize := int(fi.Size())
-		if *standardBlocks || blockSize < protocol.BlockSize {
+		if *standardBlocks || blockSize < protocol.MinBlockSize {
-			blockSize = protocol.BlockSize
+			blockSize = protocol.BlockSize(fi.Size())
 		}
 		bs, err := scanner.Blocks(context.TODO(), fd, blockSize, fi.Size(), nil, true)
 		if err != nil {

cmd/syncthing/gui.go

@@ -94,7 +94,7 @@ type modelIntf interface {
 	CurrentFolderFile(folder string, file string) (protocol.FileInfo, bool)
 	CurrentGlobalFile(folder string, file string) (protocol.FileInfo, bool)
 	ResetFolder(folder string)
-	Availability(folder, file string, version protocol.Vector, block protocol.BlockInfo) []model.Availability
+	Availability(folder string, file protocol.FileInfo, block protocol.BlockInfo) []model.Availability
 	GetIgnores(folder string) ([]string, []string, error)
 	GetFolderVersions(folder string) (map[string][]versioner.FileVersion, error)
 	RestoreFolderVersions(folder string, versions map[string]time.Time) (map[string]string, error)
@@ -828,7 +828,7 @@ func (s *apiService) getDBFile(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	av := s.model.Availability(folder, file, protocol.Vector{}, protocol.BlockInfo{})
+	av := s.model.Availability(folder, gf, protocol.BlockInfo{})
 	sendJSON(w, map[string]interface{}{
 		"global":       jsonFileInfo(gf),
 		"local":        jsonFileInfo(lf),

cmd/syncthing/mocked_model_test.go

@@ -64,7 +64,7 @@ func (m *mockedModel) CurrentGlobalFile(folder string, file string) (protocol.Fi
 func (m *mockedModel) ResetFolder(folder string) {
 }
 
-func (m *mockedModel) Availability(folder, file string, version protocol.Vector, block protocol.BlockInfo) []model.Availability {
+func (m *mockedModel) Availability(folder string, file protocol.FileInfo, block protocol.BlockInfo) []model.Availability {
 	return nil
 }
 

cmd/syncthing/usage_report.go

@@ -399,7 +399,7 @@ func (usageReportingService) String() string {
 
 // cpuBench returns CPU performance as a measure of single threaded SHA-256 MiB/s
 func cpuBench(iterations int, duration time.Duration, useWeakHash bool) float64 {
-	dataSize := 16 * protocol.BlockSize
+	dataSize := 16 * protocol.MinBlockSize
 	bs := make([]byte, dataSize)
 	rand.Reader.Read(bs)
 
@@ -420,7 +420,7 @@ func cpuBenchOnce(duration time.Duration, useWeakHash bool, bs []byte) float64 {
 	b := 0
 	for time.Since(t0) < duration {
 		r := bytes.NewReader(bs)
-		blocksResult, _ = scanner.Blocks(context.TODO(), r, protocol.BlockSize, int64(len(bs)), nil, useWeakHash)
+		blocksResult, _ = scanner.Blocks(context.TODO(), r, protocol.MinBlockSize, int64(len(bs)), nil, useWeakHash)
 		b += len(bs)
 	}
 	d := time.Since(t0)

lib/config/folderconfiguration.go

@@ -52,6 +52,7 @@ type FolderConfiguration struct {
 	Paused                bool                        `xml:"paused" json:"paused"`
 	WeakHashThresholdPct  int                         `xml:"weakHashThresholdPct" json:"weakHashThresholdPct"` // Use weak hash if more than X percent of the file has changed. Set to -1 to always use weak hash.
 	MarkerName            string                      `xml:"markerName" json:"markerName"`
+	UseLargeBlocks        bool                        `xml:"useLargeBlocks" json:"useLargeBlocks"`
 
 	cachedFilesystem fs.Filesystem
 

lib/db/set.go

@@ -43,6 +43,7 @@ type FileIntf interface {
 	IsSymlink() bool
 	HasPermissionBits() bool
 	SequenceNo() int64
+	BlockSize() int
 }
 
 // The Iterator is called with either a protocol.FileInfo or a

lib/db/structs.go

@@ -17,8 +17,8 @@ import (
 )
 
 func (f FileInfoTruncated) String() string {
-	return fmt.Sprintf("File{Name:%q, Permissions:0%o, Modified:%v, Version:%v, Length:%d, Deleted:%v, Invalid:%v, NoPermissions:%v}",
+	return fmt.Sprintf("File{Name:%q, Permissions:0%o, Modified:%v, Version:%v, Length:%d, Deleted:%v, Invalid:%v, NoPermissions:%v, BlockSize:%d}",
-		f.Name, f.Permissions, f.ModTime(), f.Version, f.Size, f.Deleted, f.Invalid, f.NoPermissions)
+		f.Name, f.Permissions, f.ModTime(), f.Version, f.Size, f.Deleted, f.Invalid, f.NoPermissions, f.RawBlockSize)
 }
 
 func (f FileInfoTruncated) IsDeleted() bool {
@@ -56,6 +56,13 @@ func (f FileInfoTruncated) FileSize() int64 {
 	return f.Size
 }
 
+func (f FileInfoTruncated) BlockSize() int {
+	if f.RawBlockSize == 0 {
+		return protocol.MinBlockSize
+	}
+	return int(f.RawBlockSize)
+}
+
 func (f FileInfoTruncated) FileName() string {
 	return f.Name
 }
@@ -70,12 +77,13 @@ func (f FileInfoTruncated) SequenceNo() int64 {
 
 func (f FileInfoTruncated) ConvertToInvalidFileInfo(invalidatedBy protocol.ShortID) protocol.FileInfo {
 	return protocol.FileInfo{
-		Name:       f.Name,
+		Name:         f.Name,
-		Type:       f.Type,
+		Type:         f.Type,
-		ModifiedS:  f.ModifiedS,
+		ModifiedS:    f.ModifiedS,
-		ModifiedNs: f.ModifiedNs,
+		ModifiedNs:   f.ModifiedNs,
-		ModifiedBy: invalidatedBy,
+		ModifiedBy:   invalidatedBy,
-		Invalid:    true,
+		Invalid:      true,
-		Version:    f.Version,
+		Version:      f.Version,
+		RawBlockSize: f.RawBlockSize,
 	}
 }

lib/db/structs.pb.go

@@ -70,7 +70,9 @@ type FileInfoTruncated struct {
 	NoPermissions bool                                                `protobuf:"varint,8,opt,name=no_permissions,json=noPermissions,proto3" json:"no_permissions,omitempty"`
 	Version       protocol.Vector                                     `protobuf:"bytes,9,opt,name=version" json:"version"`
 	Sequence      int64                                               `protobuf:"varint,10,opt,name=sequence,proto3" json:"sequence,omitempty"`
-	SymlinkTarget string                                              `protobuf:"bytes,17,opt,name=symlink_target,json=symlinkTarget,proto3" json:"symlink_target,omitempty"`
+	RawBlockSize  int32                                               `protobuf:"varint,13,opt,name=block_size,json=blockSize,proto3" json:"block_size,omitempty"`
+	// repeated BlockInfo Blocks         = 16
+	SymlinkTarget string `protobuf:"bytes,17,opt,name=symlink_target,json=symlinkTarget,proto3" json:"symlink_target,omitempty"`
 }
 
 func (m *FileInfoTruncated) Reset()                    { *m = FileInfoTruncated{} }
@@ -277,6 +279,11 @@ func (m *FileInfoTruncated) MarshalTo(dAtA []byte) (int, error) {
 		i++
 		i = encodeVarintStructs(dAtA, i, uint64(m.ModifiedBy))
 	}
+	if m.RawBlockSize != 0 {
+		dAtA[i] = 0x68
+		i++
+		i = encodeVarintStructs(dAtA, i, uint64(m.RawBlockSize))
+	}
 	if len(m.SymlinkTarget) > 0 {
 		dAtA[i] = 0x8a
 		i++
@@ -472,6 +479,9 @@ func (m *FileInfoTruncated) ProtoSize() (n int) {
 	if m.ModifiedBy != 0 {
 		n += 1 + sovStructs(uint64(m.ModifiedBy))
 	}
+	if m.RawBlockSize != 0 {
+		n += 1 + sovStructs(uint64(m.RawBlockSize))
+	}
 	l = len(m.SymlinkTarget)
 	if l > 0 {
 		n += 2 + l + sovStructs(uint64(l))
@@ -1028,6 +1038,25 @@ func (m *FileInfoTruncated) Unmarshal(dAtA []byte) error {
 					break
 				}
 			}
+		case 13:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field RawBlockSize", wireType)
+			}
+			m.RawBlockSize = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.RawBlockSize |= (int32(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
 		case 17:
 			if wireType != 2 {
 				return fmt.Errorf("proto: wrong wireType = %d for field SymlinkTarget", wireType)
@@ -1481,43 +1510,45 @@ var (
 func init() { proto.RegisterFile("structs.proto", fileDescriptorStructs) }
 
 var fileDescriptorStructs = []byte{
-	// 598 bytes of a gzipped FileDescriptorProto
+	// 630 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x53, 0xcd, 0x6a, 0xdb, 0x4c,
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x53, 0xcf, 0x6a, 0xdb, 0x4c,
-	0x14, 0xb5, 0x62, 0xc9, 0xb1, 0x47, 0x71, 0xbe, 0x2f, 0x43, 0x08, 0xc2, 0x50, 0x5b, 0x18, 0x0a,
+	0x10, 0xb7, 0x62, 0xc9, 0xb1, 0x47, 0x76, 0xbe, 0x64, 0x09, 0x41, 0x18, 0x3e, 0x5b, 0x18, 0x0a,
-	0xa2, 0x50, 0xa5, 0x4d, 0xe8, 0xa6, 0xdd, 0xb9, 0x21, 0x10, 0x28, 0x6d, 0x99, 0x84, 0xac, 0x0a,
+	0xa2, 0x50, 0xb9, 0x4d, 0xe8, 0xa5, 0xbd, 0xb9, 0x21, 0x10, 0x28, 0x6d, 0x59, 0x87, 0x9c, 0x0a,
-	0xc1, 0x92, 0xae, 0x9d, 0xa1, 0xf2, 0x8c, 0xa3, 0x19, 0x07, 0xd4, 0x27, 0xe9, 0x32, 0x0f, 0xd3,
+	0xc6, 0x92, 0xd6, 0xce, 0x12, 0x79, 0xd7, 0xd1, 0xae, 0x52, 0x94, 0x27, 0xe9, 0x31, 0x4f, 0xd0,
-	0x45, 0x96, 0x5d, 0x77, 0x11, 0x5a, 0xf7, 0x39, 0x0a, 0x45, 0x77, 0x24, 0x45, 0xe9, 0xaa, 0xdd,
+	0xa7, 0xe8, 0x21, 0xc7, 0x9e, 0x7b, 0x08, 0xad, 0xfb, 0x22, 0x45, 0xbb, 0x92, 0xa2, 0xf4, 0xd4,
-	0xdd, 0x73, 0xff, 0xef, 0x39, 0x33, 0xa4, 0xaf, 0x74, 0xb6, 0x8a, 0xb5, 0x0a, 0x97, 0x99, 0xd4,
+	0xde, 0xe6, 0x37, 0xff, 0x67, 0x7e, 0x33, 0xd0, 0x13, 0x32, 0x49, 0x43, 0x29, 0xfc, 0x75, 0xc2,
-	0x92, 0x6e, 0x24, 0xd1, 0xe0, 0xe9, 0x9c, 0xeb, 0xcb, 0x55, 0x14, 0xc6, 0x72, 0xb1, 0x3f, 0x97,
+	0x25, 0x47, 0x5b, 0x51, 0xd0, 0x7f, 0xb6, 0xa4, 0xf2, 0x22, 0x0d, 0xfc, 0x90, 0xaf, 0xc6, 0x4b,
-	0x73, 0xb9, 0x8f, 0xa1, 0x68, 0x35, 0x43, 0x84, 0x00, 0x2d, 0x53, 0x32, 0x78, 0xd1, 0x48, 0x57,
+	0xbe, 0xe4, 0x63, 0x65, 0x0a, 0xd2, 0x85, 0x42, 0x0a, 0x28, 0x49, 0x87, 0xf4, 0x5f, 0xd6, 0xdc,
-	0xb9, 0x88, 0xf5, 0x25, 0x17, 0xf3, 0x86, 0x95, 0xf2, 0xc8, 0x74, 0x88, 0x65, 0xba, 0x1f, 0xc1,
+	0x45, 0xc6, 0x42, 0x79, 0x41, 0xd9, 0xb2, 0x26, 0xc5, 0x34, 0xd0, 0x19, 0x42, 0x1e, 0x8f, 0x03,
-	0xd2, 0x94, 0x8d, 0xaf, 0x88, 0x7b, 0xcc, 0x53, 0x38, 0x87, 0x4c, 0x71, 0x29, 0xe8, 0x33, 0xb2,
+	0xb2, 0xd6, 0x61, 0xa3, 0x2b, 0xb0, 0x4f, 0x68, 0x4c, 0xce, 0x49, 0x22, 0x28, 0x67, 0xe8, 0x39,
-	0x79, 0x6d, 0x4c, 0xcf, 0xf2, 0xad, 0xc0, 0x3d, 0xf8, 0x3f, 0xac, 0x8a, 0xc2, 0x73, 0x88, 0xb5,
+	0x6c, 0x5f, 0x6b, 0xd1, 0x31, 0x5c, 0xc3, 0xb3, 0x0f, 0x77, 0xfd, 0x32, 0xc8, 0x3f, 0x27, 0xa1,
-	0xcc, 0x26, 0xf6, 0xed, 0xdd, 0xa8, 0xc5, 0xaa, 0x34, 0xba, 0x47, 0x3a, 0x09, 0x5c, 0xf3, 0x18,
+	0xe4, 0xc9, 0xc4, 0xbc, 0xbb, 0x1f, 0x36, 0x70, 0xe9, 0x86, 0x0e, 0xa0, 0x15, 0x91, 0x6b, 0x1a,
-	0xbc, 0x0d, 0xdf, 0x0a, 0xb6, 0x58, 0x89, 0xa8, 0x47, 0x36, 0xb9, 0xb8, 0x9e, 0xa6, 0x3c, 0xf1,
+	0x12, 0x67, 0xcb, 0x35, 0xbc, 0x2e, 0x2e, 0x10, 0x72, 0x60, 0x9b, 0xb2, 0xeb, 0x79, 0x4c, 0x23,
-	0xda, 0xbe, 0x15, 0x74, 0x59, 0x05, 0xc7, 0xc7, 0xc4, 0x2d, 0xc7, 0xbd, 0xe1, 0x4a, 0xd3, 0xe7,
+	0xa7, 0xe9, 0x1a, 0x5e, 0x1b, 0x97, 0x70, 0x74, 0x02, 0x76, 0x51, 0xee, 0x2d, 0x15, 0x12, 0xbd,
-	0xa4, 0x5b, 0xf6, 0x52, 0x9e, 0xe5, 0xb7, 0x03, 0xf7, 0xe0, 0xbf, 0x30, 0x89, 0xc2, 0xc6, 0x56,
+	0x80, 0x76, 0x91, 0x4b, 0x38, 0x86, 0xdb, 0xf4, 0xec, 0xc3, 0xff, 0xfc, 0x28, 0xf0, 0x6b, 0x5d,
-	0xe5, 0xc8, 0x3a, 0xed, 0xa5, 0xfd, 0xf9, 0x66, 0xd4, 0x1a, 0xff, 0x6a, 0x93, 0x9d, 0x22, 0xeb,
+	0x15, 0x25, 0x2b, 0xb7, 0x57, 0xe6, 0xe7, 0xdb, 0x61, 0x63, 0xf4, 0xc5, 0x84, 0xbd, 0xdc, 0xeb,
-	0x44, 0xcc, 0xe4, 0x59, 0xb6, 0x12, 0xf1, 0x54, 0x43, 0x42, 0x29, 0xb1, 0xc5, 0x74, 0x01, 0xb8,
+	0x94, 0x2d, 0xf8, 0x59, 0x92, 0xb2, 0x70, 0x2e, 0x49, 0x84, 0x10, 0x98, 0x6c, 0xbe, 0x22, 0xaa,
-	0x7e, 0x8f, 0xa1, 0x4d, 0x9f, 0x10, 0x5b, 0xe7, 0x4b, 0xb3, 0xe1, 0xf6, 0xc1, 0xde, 0xfd, 0x49,
+	0xfd, 0x0e, 0x56, 0x32, 0x7a, 0x0a, 0xa6, 0xcc, 0xd6, 0xba, 0xc3, 0x9d, 0xc3, 0x83, 0x87, 0x91,
-	0x75, 0x79, 0xbe, 0x04, 0x86, 0x39, 0x45, 0xbd, 0xe2, 0x9f, 0x00, 0x97, 0x6e, 0x33, 0xb4, 0xa9,
+	0xaa, 0xf0, 0x6c, 0x4d, 0xb0, 0xf2, 0xc9, 0xe3, 0x05, 0xbd, 0x21, 0xaa, 0xe9, 0x26, 0x56, 0x32,
-	0x4f, 0xdc, 0x25, 0x64, 0x0b, 0xae, 0xcc, 0x96, 0xb6, 0x6f, 0x05, 0x7d, 0xd6, 0x74, 0xd1, 0x47,
+	0x72, 0xc1, 0x5e, 0x93, 0x64, 0x45, 0x85, 0xee, 0xd2, 0x74, 0x0d, 0xaf, 0x87, 0xeb, 0x2a, 0xf4,
-	0x84, 0x2c, 0x64, 0xc2, 0x67, 0x1c, 0x92, 0x0b, 0xe5, 0x39, 0x58, 0xdb, 0xab, 0x3c, 0xa7, 0x05,
+	0x3f, 0xc0, 0x8a, 0x47, 0x74, 0x41, 0x49, 0x34, 0x13, 0x8e, 0xa5, 0x62, 0x3b, 0xa5, 0x66, 0x9a,
-	0x19, 0x09, 0xa4, 0xa0, 0x21, 0xf1, 0x3a, 0x86, 0x8c, 0x12, 0x36, 0x69, 0xda, 0x7c, 0x40, 0x13,
+	0x2f, 0x23, 0x22, 0x31, 0x91, 0x24, 0x72, 0x5a, 0x7a, 0x19, 0x05, 0xac, 0xaf, 0x69, 0xfb, 0xd1,
-	0x7d, 0x4c, 0xb6, 0x85, 0xbc, 0x68, 0xce, 0xed, 0x62, 0x42, 0x5f, 0xc8, 0xf7, 0x8d, 0xc9, 0x0d,
+	0x9a, 0xd0, 0x13, 0xd8, 0x61, 0x7c, 0x56, 0xaf, 0xdb, 0x56, 0x0e, 0x3d, 0xc6, 0x3f, 0xd4, 0x2a,
-	0xc5, 0x7a, 0x7f, 0xa7, 0xd8, 0x80, 0x74, 0x15, 0x5c, 0xad, 0x40, 0xc4, 0xe0, 0x11, 0xdc, 0xb4,
+	0xd7, 0x18, 0xeb, 0xfc, 0x1d, 0x63, 0x7d, 0x68, 0x0b, 0x72, 0x95, 0x12, 0x16, 0x12, 0x07, 0x54,
-	0xc6, 0x74, 0x44, 0xdc, 0xfa, 0x0e, 0xa1, 0x3c, 0xd7, 0xb7, 0x02, 0x87, 0xd5, 0xa7, 0xbd, 0x55,
+	0xa7, 0x15, 0x46, 0x43, 0xb0, 0xab, 0x39, 0x98, 0x70, 0x6c, 0xd7, 0xf0, 0x2c, 0x5c, 0x8d, 0xf6,
-	0xf4, 0x43, 0x23, 0x21, 0xca, 0xbd, 0x2d, 0xdf, 0x0a, 0xec, 0xc9, 0xab, 0x62, 0xc0, 0xb7, 0xbb,
+	0x4e, 0xa0, 0x8f, 0x35, 0x87, 0x20, 0x73, 0xba, 0xae, 0xe1, 0x99, 0x93, 0xd7, 0x79, 0x81, 0xef,
-	0xd1, 0xe1, 0x3f, 0xbc, 0xc1, 0xf0, 0xf4, 0x52, 0x66, 0xfa, 0xe4, 0xe8, 0xbe, 0xfb, 0x24, 0x2f,
+	0xf7, 0xc3, 0xa3, 0x7f, 0xb8, 0x41, 0x7f, 0x7a, 0xc1, 0x13, 0x79, 0x7a, 0xfc, 0x90, 0x7d, 0x92,
-	0x6e, 0x56, 0xf9, 0x22, 0xe5, 0xe2, 0xe3, 0x85, 0x9e, 0x66, 0x73, 0xd0, 0xde, 0x0e, 0xca, 0xd8,
+	0xa1, 0x31, 0x40, 0x10, 0xf3, 0xf0, 0x72, 0xa6, 0x28, 0xe8, 0xe5, 0xd5, 0x27, 0xbb, 0x9b, 0xfb,
-	0x2f, 0xbd, 0x67, 0xe8, 0x2c, 0xf5, 0xff, 0x62, 0x91, 0xce, 0x6b, 0xb9, 0x12, 0x5a, 0xd1, 0x5d,
+	0x61, 0x17, 0xcf, 0x3f, 0x4d, 0x72, 0xc3, 0x94, 0xde, 0x10, 0xdc, 0x09, 0x4a, 0x31, 0x5f, 0x92,
-	0xe2, 0xcc, 0x78, 0x0a, 0x0a, 0x55, 0x77, 0x98, 0x01, 0x85, 0x6c, 0x09, 0xcf, 0x90, 0x03, 0x0e,
+	0xc8, 0x56, 0x31, 0x65, 0x97, 0x33, 0x39, 0x4f, 0x96, 0x44, 0x3a, 0x7b, 0x8a, 0xf7, 0x5e, 0xa1,
-	0x0a, 0xd5, 0x77, 0x58, 0xd3, 0x85, 0x54, 0x98, 0xce, 0x0a, 0x05, 0x77, 0x58, 0x8d, 0x9b, 0x9a,
+	0x3d, 0x53, 0xca, 0xe2, 0x60, 0xbe, 0x1a, 0xd0, 0x7a, 0xc3, 0x53, 0x26, 0x05, 0xda, 0x07, 0x6b,
-	0xd9, 0x18, 0xaa, 0x35, 0xdb, 0x25, 0x4e, 0x94, 0x6b, 0xa8, 0x74, 0x36, 0xe0, 0x01, 0xad, 0x9d,
+	0x41, 0x63, 0x22, 0xd4, 0x99, 0x58, 0x58, 0x83, 0x9c, 0xe7, 0x88, 0x26, 0x6a, 0x69, 0x94, 0x08,
-	0x3f, 0x68, 0x1d, 0x90, 0xae, 0xf9, 0x16, 0x27, 0x47, 0x78, 0xd1, 0x16, 0xab, 0xf1, 0xf8, 0x1d,
+	0x75, 0x2e, 0x16, 0xae, 0xab, 0xd4, 0xee, 0x74, 0x66, 0xa1, 0x2e, 0xc4, 0xc2, 0x15, 0xae, 0x93,
-	0xe9, 0x99, 0x2b, 0x4e, 0x41, 0xd3, 0x80, 0x74, 0x62, 0x04, 0xe5, 0x57, 0x20, 0xc5, 0x57, 0x30,
+	0x6c, 0x2a, 0x53, 0x45, 0xf2, 0x3e, 0x58, 0x41, 0x26, 0x49, 0x79, 0x18, 0x1a, 0x3c, 0xe2, 0xa1,
-	0xe1, 0x52, 0xc6, 0x32, 0x5e, 0xac, 0x17, 0x67, 0x50, 0x3c, 0x79, 0x3c, 0xac, 0xcd, 0x2a, 0x38,
+	0xf5, 0x07, 0x0f, 0x7d, 0x68, 0xeb, 0x3f, 0x3a, 0x3d, 0x56, 0x13, 0x75, 0x71, 0x85, 0x47, 0xef,
-	0xd9, 0xbd, 0xfd, 0x31, 0x6c, 0xdd, 0xae, 0x87, 0xd6, 0xd7, 0xf5, 0xd0, 0xfa, 0xbe, 0x1e, 0xb6,
+	0xa1, 0xa3, 0xa7, 0x98, 0x12, 0x89, 0x3c, 0x68, 0x85, 0x0a, 0x14, 0xbf, 0x03, 0xf9, 0xef, 0x68,
-	0x6e, 0x7e, 0x0e, 0xad, 0xa8, 0x83, 0xc4, 0x1f, 0xfe, 0x0e, 0x00, 0x00, 0xff, 0xff, 0x6e, 0x86,
+	0x73, 0xc1, 0x7b, 0x61, 0xcf, 0xdb, 0x0b, 0x13, 0x92, 0xff, 0x88, 0x1a, 0xac, 0x89, 0x4b, 0x38,
-	0x97, 0x21, 0x6a, 0x04, 0x00, 0x00,
+	0xd9, 0xbf, 0xfb, 0x39, 0x68, 0xdc, 0x6d, 0x06, 0xc6, 0xb7, 0xcd, 0xc0, 0xf8, 0xb1, 0x19, 0x34,
+	0x6e, 0x7f, 0x0d, 0x8c, 0xa0, 0xa5, 0x98, 0x3a, 0xfa, 0x1d, 0x00, 0x00, 0xff, 0xff, 0x28, 0xfa,
+	0x1e, 0x63, 0x9b, 0x04, 0x00, 0x00,
 }

lib/db/structs.proto

@@ -35,6 +35,9 @@ message FileInfoTruncated {
     bool                  no_permissions = 8;
     protocol.Vector       version        = 9 [(gogoproto.nullable) = false];
     int64                 sequence       = 10;
+    int32                 block_size     = 13 [(gogoproto.customname) = "RawBlockSize"];
+
+    // repeated BlockInfo Blocks         = 16
     string                symlink_target = 17;
 }
 

lib/model/model.go

@@ -626,7 +626,7 @@ func (m *Model) Completion(device protocol.DeviceID, folder string) FolderComple
 		}
 
 		// This might might be more than it really is, because some blocks can be of a smaller size.
-		downloaded = int64(counts[ft.Name] * protocol.BlockSize)
+		downloaded = int64(counts[ft.Name] * int(ft.BlockSize()))
 
 		fileNeed = ft.FileSize() - downloaded
 		if fileNeed < 0 {
@@ -1968,7 +1968,6 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su
 		Folder:                folderCfg.ID,
 		Subs:                  subDirs,
 		Matcher:               ignores,
-		BlockSize:             protocol.BlockSize,
 		TempLifetime:          time.Duration(m.cfg.Options().KeepTemporariesH) * time.Hour,
 		CurrentFiler:          cFiler{m, folder},
 		Filesystem:            mtimefs,
@@ -1978,6 +1977,7 @@ func (m *Model) internalScanFolderSubdirs(ctx context.Context, folder string, su
 		ShortID:               m.shortID,
 		ProgressTickIntervalS: folderCfg.ScanProgressIntervalS,
 		UseWeakHashes:         weakhash.Enabled,
+		UseLargeBlocks:        folderCfg.UseLargeBlocks,
 	})
 
 	if err := runner.CheckHealth(); err != nil {
@@ -2513,7 +2513,7 @@ func (m *Model) RestoreFolderVersions(folder string, versions map[string]time.Ti
 	return errors, nil
 }
 
-func (m *Model) Availability(folder, file string, version protocol.Vector, block protocol.BlockInfo) []Availability {
+func (m *Model) Availability(folder string, file protocol.FileInfo, block protocol.BlockInfo) []Availability {
 	// The slightly unusual locking sequence here is because we need to hold
 	// pmut for the duration (as the value returned from foldersFiles can
 	// get heavily modified on Close()), but also must acquire fmut before
@@ -2532,7 +2532,7 @@ func (m *Model) Availability(folder, file string, version protocol.Vector, block
 
 	var availabilities []Availability
 next:
-	for _, device := range fs.Availability(file) {
+	for _, device := range fs.Availability(file.Name) {
 		for _, pausedFolder := range m.remotePausedFolders[device] {
 			if pausedFolder == folder {
 				continue next
@@ -2545,7 +2545,7 @@ next:
 	}
 
 	for device := range devices {
-		if m.deviceDownloads[device].Has(folder, file, version, int32(block.Offset/protocol.BlockSize)) {
+		if m.deviceDownloads[device].Has(folder, file.Name, file.Version, int32(block.Offset/int64(file.BlockSize()))) {
 			availabilities = append(availabilities, Availability{ID: device, FromTemporary: true})
 		}
 	}

lib/model/model_test.go

@@ -177,7 +177,7 @@ func TestRequest(t *testing.T) {
 	defer m.Stop()
 	m.ScanFolder("default")
 
-	bs := make([]byte, protocol.BlockSize)
+	bs := make([]byte, protocol.MinBlockSize)
 
 	// Existing, shared file
 	bs = bs[:6]
@@ -409,20 +409,22 @@ func (f *fakeConnection) addFile(name string, flags uint32, ftype protocol.FileI
 	f.mut.Lock()
 	defer f.mut.Unlock()
 
-	blocks, _ := scanner.Blocks(context.TODO(), bytes.NewReader(data), protocol.BlockSize, int64(len(data)), nil, true)
+	blockSize := protocol.BlockSize(int64(len(data)))
+	blocks, _ := scanner.Blocks(context.TODO(), bytes.NewReader(data), blockSize, int64(len(data)), nil, true)
 	var version protocol.Vector
 	version = version.Update(f.id.Short())
 
 	if ftype == protocol.FileInfoTypeFile || ftype == protocol.FileInfoTypeDirectory {
 		f.files = append(f.files, protocol.FileInfo{
-			Name:        name,
+			Name:         name,
-			Type:        ftype,
+			Type:         ftype,
-			Size:        int64(len(data)),
+			Size:         int64(len(data)),
-			ModifiedS:   time.Now().Unix(),
+			ModifiedS:    time.Now().Unix(),
-			Permissions: flags,
+			Permissions:  flags,
-			Version:     version,
+			Version:      version,
-			Sequence:    time.Now().UnixNano(),
+			Sequence:     time.Now().UnixNano(),
-			Blocks:      blocks,
+			RawBlockSize: int32(blockSize),
+			Blocks:       blocks,
 		})
 	} else {
 		// Symlink
@@ -2803,7 +2805,7 @@ func TestNoRequestsFromPausedDevices(t *testing.T) {
 	files.Update(device1, []protocol.FileInfo{file})
 	files.Update(device2, []protocol.FileInfo{file})
 
-	avail := m.Availability("default", file.Name, file.Version, file.Blocks[0])
+	avail := m.Availability("default", file, file.Blocks[0])
 	if len(avail) != 0 {
 		t.Errorf("should not be available, no connections")
 	}
@@ -2813,7 +2815,7 @@ func TestNoRequestsFromPausedDevices(t *testing.T) {
 
 	// !!! This is not what I'd expect to happen, as we don't even know if the peer has the original index !!!
 
-	avail = m.Availability("default", file.Name, file.Version, file.Blocks[0])
+	avail = m.Availability("default", file, file.Blocks[0])
 	if len(avail) != 2 {
 		t.Errorf("should have two available")
 	}
@@ -2833,7 +2835,7 @@ func TestNoRequestsFromPausedDevices(t *testing.T) {
 	m.ClusterConfig(device1, cc)
 	m.ClusterConfig(device2, cc)
 
-	avail = m.Availability("default", file.Name, file.Version, file.Blocks[0])
+	avail = m.Availability("default", file, file.Blocks[0])
 	if len(avail) != 2 {
 		t.Errorf("should have two available")
 	}
@@ -2841,7 +2843,7 @@ func TestNoRequestsFromPausedDevices(t *testing.T) {
 	m.Closed(&fakeConnection{id: device1}, errDeviceUnknown)
 	m.Closed(&fakeConnection{id: device2}, errDeviceUnknown)
 
-	avail = m.Availability("default", file.Name, file.Version, file.Blocks[0])
+	avail = m.Availability("default", file, file.Blocks[0])
 	if len(avail) != 0 {
 		t.Errorf("should have no available")
 	}
@@ -2856,7 +2858,7 @@ func TestNoRequestsFromPausedDevices(t *testing.T) {
 	ccp.Folders[0].Paused = true
 	m.ClusterConfig(device1, ccp)
 
-	avail = m.Availability("default", file.Name, file.Version, file.Blocks[0])
+	avail = m.Availability("default", file, file.Blocks[0])
 	if len(avail) != 1 {
 		t.Errorf("should have one available")
 	}

lib/model/rwfolder.go

@@ -126,7 +126,7 @@ func newSendReceiveFolder(model *Model, cfg config.FolderConfiguration, ver vers
 	if f.PullerMaxPendingKiB == 0 {
 		f.PullerMaxPendingKiB = defaultPullerPendingKiB
 	}
-	if blockSizeKiB := protocol.BlockSize / 1024; f.PullerMaxPendingKiB < blockSizeKiB {
+	if blockSizeKiB := protocol.MaxBlockSize / 1024; f.PullerMaxPendingKiB < blockSizeKiB {
 		f.PullerMaxPendingKiB = blockSizeKiB
 	}
 
@@ -1010,7 +1010,7 @@ func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, copyChan chan<- c
 
 	// Check for an old temporary file which might have some blocks we could
 	// reuse.
-	tempBlocks, err := scanner.HashFile(f.ctx, f.fs, tempName, protocol.BlockSize, nil, false)
+	tempBlocks, err := scanner.HashFile(f.ctx, f.fs, tempName, file.BlockSize(), nil, false)
 	if err == nil {
 		// Check for any reusable blocks in the temp file
 		tempCopyBlocks, _ := blockDiff(tempBlocks, file.Blocks)
@@ -1160,7 +1160,7 @@ func (f *sendReceiveFolder) shortcutFile(file protocol.FileInfo) error {
 // copierRoutine reads copierStates until the in channel closes and performs
 // the relevant copies when possible, or passes it to the puller routine.
 func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan chan<- pullBlockState, out chan<- *sharedPullerState) {
-	buf := make([]byte, protocol.BlockSize)
+	buf := make([]byte, protocol.MinBlockSize)
 
 	for state := range in {
 		dstFd, err := state.tempFile()
@@ -1203,7 +1203,7 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
 				if len(hashesToFind) > 0 {
 					file, err = f.fs.Open(state.file.Name)
 					if err == nil {
-						weakHashFinder, err = weakhash.NewFinder(file, protocol.BlockSize, hashesToFind)
+						weakHashFinder, err = weakhash.NewFinder(file, int(state.file.BlockSize()), hashesToFind)
 						if err != nil {
 							l.Debugln("weak hasher", err)
 						}
@@ -1268,7 +1268,7 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
 						return false
 					}
 
-					_, err = fd.ReadAt(buf, protocol.BlockSize*int64(index))
+					_, err = fd.ReadAt(buf, int64(state.file.BlockSize())*int64(index))
 					fd.Close()
 					if err != nil {
 						return false
@@ -1391,7 +1391,7 @@ func (f *sendReceiveFolder) pullBlock(state pullBlockState, out chan<- *sharedPu
 	}
 
 	var lastError error
-	candidates := f.model.Availability(f.folderID, state.file.Name, state.file.Version, state.block)
+	candidates := f.model.Availability(f.folderID, state.file, state.block)
 	for {
 		// Select the least busy device to pull the block from. If we found no
 		// feasible device at all, fail the block (and in the long run, the

lib/model/rwfolder_test.go

@@ -256,7 +256,7 @@ func TestCopierFinder(t *testing.T) {
 	}
 
 	// Verify that the fetched blocks have actually been written to the temp file
-	blks, err := scanner.HashFile(context.TODO(), fs.NewFilesystem(fs.FilesystemTypeBasic, "."), tempFile, protocol.BlockSize, nil, false)
+	blks, err := scanner.HashFile(context.TODO(), fs.NewFilesystem(fs.FilesystemTypeBasic, "."), tempFile, protocol.MinBlockSize, nil, false)
 	if err != nil {
 		t.Log(err)
 	}
@@ -273,8 +273,8 @@ func TestWeakHash(t *testing.T) {
 	tempFile := filepath.Join("testdata", fs.TempName("weakhash"))
 	var shift int64 = 10
 	var size int64 = 1 << 20
-	expectBlocks := int(size / protocol.BlockSize)
+	expectBlocks := int(size / protocol.MinBlockSize)
-	expectPulls := int(shift / protocol.BlockSize)
+	expectPulls := int(shift / protocol.MinBlockSize)
 	if shift > 0 {
 		expectPulls++
 	}
@@ -307,7 +307,7 @@ func TestWeakHash(t *testing.T) {
 	// File 1: abcdefgh
 	// File 2: xyabcdef
 	f.Seek(0, os.SEEK_SET)
-	existing, err := scanner.Blocks(context.TODO(), f, protocol.BlockSize, size, nil, true)
+	existing, err := scanner.Blocks(context.TODO(), f, protocol.MinBlockSize, size, nil, true)
 	if err != nil {
 		t.Error(err)
 	}
@@ -316,7 +316,7 @@ func TestWeakHash(t *testing.T) {
 	remainder := io.LimitReader(f, size-shift)
 	prefix := io.LimitReader(rand.Reader, shift)
 	nf := io.MultiReader(prefix, remainder)
-	desired, err := scanner.Blocks(context.TODO(), nf, protocol.BlockSize, size, nil, true)
+	desired, err := scanner.Blocks(context.TODO(), nf, protocol.MinBlockSize, size, nil, true)
 	if err != nil {
 		t.Error(err)
 	}

lib/model/sharedpullerstate.go

@@ -217,7 +217,7 @@ func (s *sharedPullerState) copyDone(block protocol.BlockInfo) {
 	s.mut.Lock()
 	s.copyNeeded--
 	s.updated = time.Now()
-	s.available = append(s.available, int32(block.Offset/protocol.BlockSize))
+	s.available = append(s.available, int32(block.Offset/int64(s.file.BlockSize())))
 	s.availableUpdated = time.Now()
 	l.Debugln("sharedPullerState", s.folder, s.file.Name, "copyNeeded ->", s.copyNeeded)
 	s.mut.Unlock()
@@ -253,7 +253,7 @@ func (s *sharedPullerState) pullDone(block protocol.BlockInfo) {
 	s.mut.Lock()
 	s.pullNeeded--
 	s.updated = time.Now()
-	s.available = append(s.available, int32(block.Offset/protocol.BlockSize))
+	s.available = append(s.available, int32(block.Offset/int64(s.file.BlockSize())))
 	s.availableUpdated = time.Now()
 	l.Debugln("sharedPullerState", s.folder, s.file.Name, "pullNeeded done ->", s.pullNeeded)
 	s.mut.Unlock()
@@ -314,8 +314,8 @@ func (s *sharedPullerState) Progress() *pullerProgress {
 		CopiedFromElsewhere: s.copyTotal - s.copyNeeded - s.copyOrigin,
 		Pulled:              s.pullTotal - s.pullNeeded,
 		Pulling:             s.pullNeeded,
-		BytesTotal:          blocksToSize(total),
+		BytesTotal:          blocksToSize(s.file.BlockSize(), total),
-		BytesDone:           blocksToSize(done),
+		BytesDone:           blocksToSize(s.file.BlockSize(), done),
 	}
 }
 
@@ -343,9 +343,9 @@ func (s *sharedPullerState) Available() []int32 {
 	return blocks
 }
 
-func blocksToSize(num int) int64 {
+func blocksToSize(size int, num int) int64 {
 	if num < 2 {
-		return protocol.BlockSize / 2
+		return int64(size / 2)
 	}
-	return int64(num-1)*protocol.BlockSize + protocol.BlockSize/2
+	return int64(num-1)*int64(size) + int64(size/2)
 }

lib/protocol/bep.pb.go

@@ -306,6 +306,7 @@ type FileInfo struct {
 	NoPermissions bool         `protobuf:"varint,8,opt,name=no_permissions,json=noPermissions,proto3" json:"no_permissions,omitempty"`
 	Version       Vector       `protobuf:"bytes,9,opt,name=version" json:"version"`
 	Sequence      int64        `protobuf:"varint,10,opt,name=sequence,proto3" json:"sequence,omitempty"`
+	RawBlockSize  int32        `protobuf:"varint,13,opt,name=block_size,json=blockSize,proto3" json:"block_size,omitempty"`
 	Blocks        []BlockInfo  `protobuf:"bytes,16,rep,name=Blocks" json:"Blocks"`
 	SymlinkTarget string       `protobuf:"bytes,17,opt,name=symlink_target,json=symlinkTarget,proto3" json:"symlink_target,omitempty"`
 }
@@ -876,6 +877,11 @@ func (m *FileInfo) MarshalTo(dAtA []byte) (int, error) {
 		i++
 		i = encodeVarintBep(dAtA, i, uint64(m.ModifiedBy))
 	}
+	if m.RawBlockSize != 0 {
+		dAtA[i] = 0x68
+		i++
+		i = encodeVarintBep(dAtA, i, uint64(m.RawBlockSize))
+	}
 	if len(m.Blocks) > 0 {
 		for _, msg := range m.Blocks {
 			dAtA[i] = 0x82
@@ -1427,6 +1433,9 @@ func (m *FileInfo) ProtoSize() (n int) {
 	if m.ModifiedBy != 0 {
 		n += 1 + sovBep(uint64(m.ModifiedBy))
 	}
+	if m.RawBlockSize != 0 {
+		n += 1 + sovBep(uint64(m.RawBlockSize))
+	}
 	if len(m.Blocks) > 0 {
 		for _, e := range m.Blocks {
 			l = e.ProtoSize()
@@ -2904,6 +2913,25 @@ func (m *FileInfo) Unmarshal(dAtA []byte) error {
 					break
 				}
 			}
+		case 13:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field RawBlockSize", wireType)
+			}
+			m.RawBlockSize = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowBep
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.RawBlockSize |= (int32(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
 		case 16:
 			if wireType != 2 {
 				return fmt.Errorf("proto: wrong wireType = %d for field Blocks", wireType)
@@ -4164,114 +4192,115 @@ var (
 func init() { proto.RegisterFile("bep.proto", fileDescriptorBep) }
 
 var fileDescriptorBep = []byte{
-	// 1729 bytes of a gzipped FileDescriptorProto
+	// 1757 bytes of a gzipped FileDescriptorProto
 	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0x4f, 0x8f, 0xdb, 0xc6,
-	0x15, 0x17, 0x25, 0xea, 0xdf, 0x93, 0x76, 0xc3, 0x1d, 0xdb, 0x5b, 0x96, 0xd9, 0x68, 0x69, 0xc5,
+	0x15, 0x17, 0x25, 0xea, 0xdf, 0x93, 0xb4, 0xe1, 0x8e, 0xed, 0x2d, 0xcb, 0x6c, 0x24, 0x5a, 0xb1,
-	0x8e, 0x37, 0x8b, 0x64, 0xe3, 0x26, 0x69, 0x8b, 0x16, 0x6d, 0x01, 0xfd, 0xe1, 0xae, 0x85, 0xca,
+	0xe3, 0xcd, 0x22, 0x59, 0xbb, 0x49, 0xda, 0xa2, 0x45, 0x5b, 0x40, 0x7f, 0xb8, 0x6b, 0xa1, 0x32,
-	0x94, 0x3a, 0xd2, 0x3a, 0x75, 0x0e, 0x25, 0x28, 0x71, 0xa4, 0x25, 0x4c, 0x71, 0x54, 0x92, 0x5a,
+	0xa5, 0x8e, 0xb4, 0x4e, 0x9d, 0x43, 0x09, 0x4a, 0x1c, 0x69, 0x09, 0x53, 0x1c, 0x95, 0xa4, 0xd6,
-	0x5b, 0xfd, 0x08, 0xfa, 0x04, 0xbd, 0x08, 0x08, 0xd0, 0x53, 0xef, 0xfd, 0x10, 0x3e, 0x06, 0x3d,
+	0x56, 0x3e, 0x82, 0x3e, 0x41, 0x2f, 0x02, 0x02, 0xf4, 0x54, 0xa0, 0xc7, 0x7e, 0x08, 0x1f, 0x83,
-	0xf4, 0xd0, 0x83, 0xd1, 0x6c, 0x2f, 0x3d, 0xf6, 0x5e, 0xa0, 0x28, 0x38, 0x43, 0x52, 0xd4, 0xae,
+	0x1e, 0x7a, 0xe8, 0xc1, 0x68, 0xb6, 0x97, 0x1e, 0xfb, 0x09, 0x8a, 0x82, 0x33, 0xa4, 0x44, 0xed,
-	0x37, 0xf0, 0xa1, 0x27, 0xce, 0xbc, 0xf7, 0x9b, 0x3f, 0xef, 0xf7, 0xde, 0xef, 0x0d, 0xa1, 0x3c,
+	0xda, 0x81, 0x0f, 0x39, 0x71, 0xe6, 0xbd, 0xdf, 0xbc, 0x99, 0xf9, 0xcd, 0xef, 0xbd, 0x47, 0x28,
-	0x22, 0xf3, 0x93, 0xb9, 0x47, 0x03, 0x8a, 0x4a, 0xec, 0x33, 0xa6, 0x8e, 0xf2, 0xe9, 0xd4, 0x0e,
+	0x8e, 0xc8, 0xfc, 0x64, 0xee, 0xd1, 0x80, 0xa2, 0x02, 0xfb, 0x8c, 0xa9, 0xa3, 0x7c, 0x3a, 0xb5,
-	0x2e, 0x16, 0xa3, 0x93, 0x31, 0x9d, 0x7d, 0x36, 0xa5, 0x53, 0xfa, 0x19, 0xf3, 0x8c, 0x16, 0x13,
+	0x83, 0x8b, 0xc5, 0xe8, 0x64, 0x4c, 0x67, 0x0f, 0xa7, 0x74, 0x4a, 0x1f, 0x32, 0xcf, 0x68, 0x31,
-	0x36, 0x63, 0x13, 0x36, 0xe2, 0x0b, 0xeb, 0x73, 0xc8, 0x3f, 0x21, 0x8e, 0x43, 0xd1, 0x21, 0x54,
+	0x61, 0x33, 0x36, 0x61, 0x23, 0xbe, 0xb0, 0x3e, 0x87, 0xec, 0x63, 0xe2, 0x38, 0x14, 0xd5, 0xa0,
-	0x2c, 0x72, 0x69, 0x8f, 0x89, 0xe1, 0x9a, 0x33, 0x22, 0x0b, 0xaa, 0x70, 0x54, 0xc6, 0xc0, 0x4d,
+	0x64, 0x91, 0x4b, 0x7b, 0x4c, 0x0c, 0xd7, 0x9c, 0x11, 0x59, 0x50, 0x85, 0xa3, 0x22, 0x06, 0x6e,
-	0xba, 0x39, 0x23, 0x21, 0x60, 0xec, 0xd8, 0xc4, 0x0d, 0x38, 0x20, 0xcb, 0x01, 0xdc, 0xc4, 0x00,
+	0xd2, 0xcd, 0x19, 0x09, 0x01, 0x63, 0xc7, 0x26, 0x6e, 0xc0, 0x01, 0x69, 0x0e, 0xe0, 0x26, 0x06,
-	0x0f, 0x61, 0x37, 0x02, 0x5c, 0x12, 0xcf, 0xb7, 0xa9, 0x2b, 0xe7, 0x18, 0x66, 0x87, 0x5b, 0x9f,
+	0xb8, 0x0f, 0x7b, 0x11, 0xe0, 0x92, 0x78, 0xbe, 0x4d, 0x5d, 0x39, 0xc3, 0x30, 0x15, 0x6e, 0x7d,
-	0x71, 0x63, 0xdd, 0x87, 0xc2, 0x13, 0x62, 0x5a, 0xc4, 0x43, 0x1f, 0x83, 0x18, 0x2c, 0xe7, 0xfc,
+	0xca, 0x8d, 0x75, 0x1f, 0x72, 0x8f, 0x89, 0x69, 0x11, 0x0f, 0x7d, 0x0c, 0x62, 0xb0, 0x9c, 0xf3,
-	0xac, 0xdd, 0xcf, 0xef, 0x9d, 0xc4, 0x31, 0x9c, 0x3c, 0x25, 0xbe, 0x6f, 0x4e, 0xc9, 0x70, 0x39,
+	0xbd, 0xf6, 0x3e, 0xbb, 0x73, 0x12, 0xdf, 0xe1, 0xe4, 0x09, 0xf1, 0x7d, 0x73, 0x4a, 0x86, 0xcb,
-	0x27, 0x98, 0x41, 0xd0, 0xaf, 0xa0, 0x32, 0xa6, 0xb3, 0xb9, 0x47, 0x7c, 0xb6, 0x71, 0x96, 0xad,
+	0x39, 0xc1, 0x0c, 0x82, 0x7e, 0x03, 0xa5, 0x31, 0x9d, 0xcd, 0x3d, 0xe2, 0xb3, 0xc0, 0x69, 0xb6,
-	0x38, 0xb8, 0xb1, 0xa2, 0xb5, 0xc1, 0xe0, 0xf4, 0x82, 0x7a, 0x03, 0x76, 0x5a, 0xce, 0xc2, 0x0f,
+	0xe2, 0xf0, 0xc6, 0x8a, 0xd6, 0x16, 0x83, 0x93, 0x0b, 0xea, 0x0d, 0xa8, 0xb4, 0x9c, 0x85, 0x1f,
-	0x88, 0xd7, 0xa2, 0xee, 0xc4, 0x9e, 0xa2, 0xc7, 0x50, 0x9c, 0x50, 0xc7, 0x22, 0x9e, 0x2f, 0x0b,
+	0x10, 0xaf, 0x45, 0xdd, 0x89, 0x3d, 0x45, 0x8f, 0x20, 0x3f, 0xa1, 0x8e, 0x45, 0x3c, 0x5f, 0x16,
-	0x6a, 0xee, 0xa8, 0xf2, 0xb9, 0xb4, 0xd9, 0xec, 0x94, 0x39, 0x9a, 0xe2, 0xeb, 0x37, 0x87, 0x19,
+	0xd4, 0xcc, 0x51, 0xe9, 0x33, 0x69, 0x1b, 0xec, 0x94, 0x39, 0x9a, 0xe2, 0xab, 0xd7, 0xb5, 0x14,
-	0x1c, 0xc3, 0xea, 0x7f, 0xca, 0x42, 0x81, 0x7b, 0xd0, 0x3e, 0x64, 0x6d, 0x8b, 0x53, 0xd4, 0x2c,
+	0x8e, 0x61, 0xf5, 0x3f, 0xa7, 0x21, 0xc7, 0x3d, 0xe8, 0x00, 0xd2, 0xb6, 0xc5, 0x29, 0x6a, 0xe6,
-	0x5c, 0xbd, 0x39, 0xcc, 0x76, 0xda, 0x38, 0x6b, 0x5b, 0xe8, 0x2e, 0xe4, 0x1d, 0x73, 0x44, 0x9c,
+	0xae, 0x5e, 0xd7, 0xd2, 0x9d, 0x36, 0x4e, 0xdb, 0x16, 0xba, 0x0d, 0x59, 0xc7, 0x1c, 0x11, 0x27,
-	0x88, 0x1c, 0x3e, 0x41, 0xef, 0x43, 0xd9, 0x23, 0xa6, 0x65, 0x50, 0xd7, 0x59, 0x32, 0x4a, 0x4a,
+	0x22, 0x87, 0x4f, 0xd0, 0xfb, 0x50, 0xf4, 0x88, 0x69, 0x19, 0xd4, 0x75, 0x96, 0x8c, 0x92, 0x02,
-	0xb8, 0x14, 0x1a, 0x7a, 0xae, 0xb3, 0x44, 0x9f, 0x02, 0xb2, 0xa7, 0x2e, 0xf5, 0x88, 0x31, 0x27,
+	0x2e, 0x84, 0x86, 0x9e, 0xeb, 0x2c, 0xd1, 0xa7, 0x80, 0xec, 0xa9, 0x4b, 0x3d, 0x62, 0xcc, 0x89,
-	0xde, 0xcc, 0x66, 0xb7, 0xf5, 0x65, 0x91, 0xa1, 0xf6, 0xb8, 0xa7, 0xbf, 0x71, 0xa0, 0x0f, 0x61,
+	0x37, 0xb3, 0xd9, 0x69, 0x7d, 0x59, 0x64, 0xa8, 0x7d, 0xee, 0xe9, 0x6f, 0x1d, 0xe8, 0x43, 0xa8,
-	0x27, 0x82, 0x5b, 0xc4, 0x21, 0x01, 0x91, 0xf3, 0x0c, 0x59, 0xe5, 0xc6, 0x36, 0xb3, 0xa1, 0xc7,
+	0x44, 0x70, 0x8b, 0x38, 0x24, 0x20, 0x72, 0x96, 0x21, 0xcb, 0xdc, 0xd8, 0x66, 0x36, 0xf4, 0x08,
-	0x70, 0xd7, 0xb2, 0x7d, 0x73, 0xe4, 0x10, 0x23, 0x20, 0xb3, 0xb9, 0x61, 0xbb, 0x16, 0x79, 0x45,
+	0x6e, 0x5b, 0xb6, 0x6f, 0x8e, 0x1c, 0x62, 0x04, 0x64, 0x36, 0x37, 0x6c, 0xd7, 0x22, 0x2f, 0x89,
-	0x7c, 0xb9, 0xc0, 0xb0, 0x28, 0xf2, 0x0d, 0xc9, 0x6c, 0xde, 0xe1, 0x1e, 0xb4, 0x0f, 0x85, 0xb9,
+	0x2f, 0xe7, 0x18, 0x16, 0x45, 0xbe, 0x21, 0x99, 0xcd, 0x3b, 0xdc, 0x83, 0x0e, 0x20, 0x37, 0x37,
-	0xb9, 0xf0, 0x89, 0x25, 0x17, 0x19, 0x26, 0x9a, 0x85, 0x2c, 0xf1, 0x0a, 0xf0, 0x65, 0xe9, 0x3a,
+	0x17, 0x3e, 0xb1, 0xe4, 0x3c, 0xc3, 0x44, 0xb3, 0x90, 0x25, 0xae, 0x00, 0x5f, 0x96, 0xae, 0xb3,
-	0x4b, 0x6d, 0xe6, 0x88, 0x59, 0x8a, 0x60, 0xf5, 0x7f, 0x67, 0xa1, 0xc0, 0x3d, 0xe8, 0xa3, 0x84,
+	0xd4, 0x66, 0x8e, 0x98, 0xa5, 0x08, 0x56, 0xff, 0x6f, 0x1a, 0x72, 0xdc, 0x83, 0x3e, 0xda, 0xb0,
-	0xa5, 0x6a, 0x73, 0x3f, 0x44, 0xfd, 0xfd, 0xcd, 0x61, 0x89, 0xfb, 0x3a, 0xed, 0x14, 0x6b, 0x08,
+	0x54, 0x6e, 0x1e, 0x84, 0xa8, 0x7f, 0xbe, 0xae, 0x15, 0xb8, 0xaf, 0xd3, 0x4e, 0xb0, 0x86, 0x40,
-	0xc4, 0x54, 0x45, 0xb1, 0x31, 0x3a, 0x80, 0xb2, 0x69, 0x59, 0x61, 0xf6, 0x88, 0x2f, 0xe7, 0xd4,
+	0x4c, 0x28, 0x8a, 0x8d, 0xd1, 0x21, 0x14, 0x4d, 0xcb, 0x0a, 0x5f, 0x8f, 0xf8, 0x72, 0x46, 0xcd,
-	0xdc, 0x51, 0x19, 0x6f, 0x0c, 0xe8, 0xa7, 0xdb, 0xd5, 0x20, 0x5e, 0xaf, 0x9f, 0xdb, 0xca, 0x20,
+	0x1c, 0x15, 0xf1, 0xd6, 0x80, 0x7e, 0xbe, 0xab, 0x06, 0xf1, 0xba, 0x7e, 0xde, 0x26, 0x83, 0xf0,
-	0x4c, 0xc5, 0x98, 0x78, 0x51, 0x05, 0xe7, 0xd9, 0x79, 0xa5, 0xd0, 0xc0, 0xea, 0xf7, 0x3e, 0x54,
+	0x29, 0xc6, 0xc4, 0x8b, 0x14, 0x9c, 0x65, 0xfb, 0x15, 0x42, 0x03, 0xd3, 0xef, 0x5d, 0x28, 0xcf,
-	0x67, 0xe6, 0x2b, 0xc3, 0x27, 0xbf, 0x5f, 0x10, 0x77, 0x4c, 0x18, 0x5d, 0x39, 0x5c, 0x99, 0x99,
+	0xcc, 0x97, 0x86, 0x4f, 0xfe, 0xb8, 0x20, 0xee, 0x98, 0x30, 0xba, 0x32, 0xb8, 0x34, 0x33, 0x5f,
-	0xaf, 0x06, 0x91, 0x09, 0xd5, 0x00, 0x6c, 0x37, 0xf0, 0xa8, 0xb5, 0x18, 0x13, 0x2f, 0xe2, 0x2a,
+	0x0e, 0x22, 0x13, 0xaa, 0x02, 0xd8, 0x6e, 0xe0, 0x51, 0x6b, 0x31, 0x26, 0x5e, 0xc4, 0x55, 0xc2,
-	0x65, 0x41, 0x3f, 0x86, 0x12, 0x23, 0xdb, 0xb0, 0x2d, 0xb9, 0xa4, 0x0a, 0x47, 0x62, 0x53, 0x89,
+	0x82, 0x7e, 0x0a, 0x05, 0x46, 0xb6, 0x61, 0x5b, 0x72, 0x41, 0x15, 0x8e, 0xc4, 0xa6, 0x12, 0x5d,
-	0x02, 0x2f, 0x32, 0xaa, 0x59, 0xdc, 0xf1, 0x10, 0x17, 0x19, 0xb6, 0x63, 0xa1, 0x5f, 0x80, 0xe2,
+	0x3c, 0xcf, 0xa8, 0x66, 0xf7, 0x8e, 0x87, 0x38, 0xcf, 0xb0, 0x1d, 0x0b, 0xfd, 0x0a, 0x14, 0xff,
-	0xbf, 0xb0, 0xc3, 0x44, 0xf1, 0x9d, 0x02, 0x9b, 0xba, 0x86, 0x47, 0x66, 0xf4, 0xd2, 0x74, 0x7c,
+	0xb9, 0x1d, 0x3e, 0x14, 0x8f, 0x14, 0xd8, 0xd4, 0x35, 0x3c, 0x32, 0xa3, 0x97, 0xa6, 0xe3, 0xcb,
-	0xb9, 0xcc, 0x8e, 0x91, 0x43, 0x44, 0x27, 0x05, 0xc0, 0x91, 0xbf, 0xde, 0x83, 0x3c, 0xdb, 0x31,
+	0x45, 0xb6, 0x8d, 0x1c, 0x22, 0x3a, 0x09, 0x00, 0x8e, 0xfc, 0xf5, 0x1e, 0x64, 0x59, 0xc4, 0xf0,
-	0xcc, 0x22, 0x2f, 0xd6, 0x48, 0xbd, 0xd1, 0x0c, 0x9d, 0x40, 0x7e, 0x62, 0x3b, 0xc4, 0x97, 0xb3,
+	0x15, 0xb9, 0x58, 0xa3, 0xec, 0x8d, 0x66, 0xe8, 0x04, 0xb2, 0x13, 0xdb, 0x21, 0xbe, 0x9c, 0x66,
-	0x2c, 0x87, 0x28, 0x55, 0xe9, 0xb6, 0x43, 0x3a, 0xee, 0x84, 0x46, 0x59, 0xe4, 0xb0, 0xfa, 0x39,
+	0x6f, 0x88, 0x12, 0x4a, 0xb7, 0x1d, 0xd2, 0x71, 0x27, 0x34, 0x7a, 0x45, 0x0e, 0xab, 0x9f, 0x43,
-	0x54, 0xd8, 0x86, 0xe7, 0x73, 0xcb, 0x0c, 0xc8, 0xff, 0x6d, 0xdb, 0xff, 0xe4, 0xa0, 0x14, 0x7b,
+	0x89, 0x05, 0x3c, 0x9f, 0x5b, 0x66, 0x40, 0x7e, 0xb0, 0xb0, 0x7f, 0x15, 0xa1, 0x10, 0x7b, 0x36,
-	0x92, 0xa4, 0x0b, 0xa9, 0xa4, 0x1f, 0x47, 0xfd, 0x80, 0xab, 0x7b, 0xff, 0xe6, 0x7e, 0xa9, 0x86,
+	0x8f, 0x2e, 0x24, 0x1e, 0xfd, 0x38, 0xaa, 0x07, 0x3c, 0xbb, 0x0f, 0x6e, 0xc6, 0x4b, 0x14, 0x04,
-	0x80, 0x40, 0xf4, 0xed, 0x3f, 0x10, 0xa6, 0xa7, 0x1c, 0x66, 0x63, 0xa4, 0x42, 0xe5, 0xba, 0x88,
+	0x04, 0xa2, 0x6f, 0x7f, 0x4d, 0x58, 0x3e, 0x65, 0x30, 0x1b, 0x23, 0x15, 0x4a, 0xd7, 0x93, 0xa8,
-	0x76, 0x70, 0xda, 0x84, 0x3e, 0x00, 0x98, 0x51, 0xcb, 0x9e, 0xd8, 0xc4, 0x32, 0x7c, 0x56, 0x00,
+	0x82, 0x93, 0x26, 0xf4, 0x01, 0xc0, 0x8c, 0x5a, 0xf6, 0xc4, 0x26, 0x96, 0xe1, 0x33, 0x01, 0x64,
-	0x39, 0x5c, 0x8e, 0x2d, 0x03, 0x24, 0x87, 0xe5, 0x1e, 0x4a, 0xc8, 0x8a, 0xb4, 0x12, 0x4f, 0x43,
+	0x70, 0x31, 0xb6, 0x0c, 0x90, 0x1c, 0xca, 0x3d, 0x4c, 0x21, 0x2b, 0xca, 0x95, 0x78, 0x1a, 0x7a,
-	0x8f, 0xed, 0x5e, 0x9a, 0x8e, 0x1d, 0x2b, 0x24, 0x9e, 0x86, 0x5d, 0xcf, 0xa5, 0x5b, 0xe2, 0x2d,
+	0x6c, 0xf7, 0xd2, 0x74, 0xec, 0x38, 0x43, 0xe2, 0x69, 0x58, 0xf5, 0x5c, 0xba, 0x93, 0xbc, 0x05,
-	0x31, 0xc0, 0x8e, 0x4b, 0xd3, 0xc2, 0x7d, 0x0c, 0xc5, 0xb8, 0x2b, 0x86, 0xf9, 0xdc, 0x52, 0xd2,
+	0x06, 0xa8, 0xb8, 0x34, 0x99, 0xb8, 0x8f, 0x20, 0x1f, 0x57, 0xc5, 0xf0, 0x3d, 0x77, 0x32, 0xe9,
-	0x33, 0x32, 0x0e, 0x68, 0xd2, 0x6f, 0x22, 0x18, 0x52, 0xa0, 0x94, 0x94, 0x22, 0xb0, 0x9b, 0x26,
+	0x29, 0x19, 0x07, 0x74, 0x53, 0x6f, 0x22, 0x18, 0x52, 0xa0, 0xb0, 0x91, 0x22, 0xb0, 0x93, 0x6e,
-	0xf3, 0xb0, 0x17, 0x27, 0x71, 0xb8, 0xbe, 0x5c, 0x51, 0x85, 0xa3, 0x3c, 0x4e, 0x42, 0xd3, 0xc3,
+	0xe6, 0x61, 0x2d, 0xde, 0xdc, 0xc3, 0xf5, 0xe5, 0x92, 0x2a, 0x1c, 0x65, 0xf1, 0xe6, 0x6a, 0x7a,
-	0xe3, 0x36, 0x80, 0xd1, 0x52, 0xae, 0xb2, 0x5a, 0x7c, 0x2f, 0xae, 0xc5, 0xc1, 0x05, 0xf5, 0x82,
+	0xb8, 0xdd, 0x16, 0x30, 0x5a, 0xca, 0x65, 0xa6, 0xc5, 0xf7, 0x62, 0x2d, 0x0e, 0x2e, 0xa8, 0x17,
-	0x4e, 0x7b, 0xb3, 0xa2, 0xb9, 0x44, 0x3f, 0x82, 0x42, 0xd3, 0xa1, 0xe3, 0x17, 0xb1, 0xd2, 0xef,
+	0x74, 0xda, 0xdb, 0x15, 0xcd, 0x25, 0x7a, 0x08, 0x30, 0x72, 0xe8, 0xf8, 0xb9, 0xc1, 0x68, 0xad,
-	0x6c, 0xee, 0xc7, 0xec, 0xa9, 0x7c, 0x46, 0xc0, 0x30, 0x74, 0x7f, 0x39, 0x73, 0x6c, 0xf7, 0x85,
+	0x84, 0x11, 0x9b, 0xd2, 0xd5, 0xeb, 0x5a, 0x19, 0x9b, 0x2f, 0x9a, 0xa1, 0x63, 0x60, 0x7f, 0x4d,
-	0x11, 0x98, 0xde, 0x94, 0x04, 0xf2, 0x1e, 0x6f, 0xf8, 0x91, 0x75, 0xc8, 0x8c, 0x3f, 0x17, 0xff,
+	0x70, 0x71, 0x14, 0x0f, 0xd1, 0x4f, 0x20, 0xc7, 0xec, 0x71, 0x69, 0xb8, 0xb5, 0xbd, 0x10, 0xb3,
-	0xf8, 0xcd, 0x61, 0xa6, 0xee, 0x42, 0x39, 0xd9, 0x27, 0x2c, 0x29, 0x3a, 0x99, 0xf8, 0x24, 0x60,
+	0x27, 0x04, 0x10, 0x01, 0x43, 0xae, 0xfc, 0xe5, 0xcc, 0xb1, 0xdd, 0xe7, 0x46, 0x60, 0x7a, 0x53,
-	0xf9, 0xcf, 0xe1, 0x68, 0x96, 0x64, 0x35, 0xcb, 0x02, 0xe2, 0x59, 0x45, 0x20, 0x5e, 0x98, 0xfe,
+	0x12, 0xc8, 0xfb, 0xbc, 0x43, 0x44, 0xd6, 0x21, 0x33, 0xfe, 0x52, 0xfc, 0xd3, 0x37, 0xb5, 0x54,
-	0x05, 0xcb, 0x74, 0x15, 0xb3, 0x71, 0xa8, 0xe3, 0x97, 0xc4, 0x7c, 0x61, 0x30, 0x07, 0xcf, 0x73,
+	0xdd, 0x85, 0xe2, 0x26, 0x4e, 0xa8, 0x41, 0x3a, 0x99, 0xf8, 0x24, 0x60, 0x82, 0xc9, 0xe0, 0x68,
-	0x29, 0x34, 0x3c, 0x31, 0xfd, 0x8b, 0xe8, 0xbc, 0x5f, 0x42, 0x81, 0xf3, 0x8a, 0xbe, 0x80, 0xd2,
+	0xb6, 0x91, 0x41, 0x9a, 0x31, 0xc0, 0x65, 0x80, 0x40, 0xbc, 0x30, 0xfd, 0x0b, 0x26, 0x8d, 0x32,
-	0x98, 0x2e, 0xdc, 0x60, 0xd3, 0xeb, 0xf7, 0xd2, 0xad, 0x82, 0x79, 0xa2, 0xc8, 0x12, 0x60, 0xfd,
+	0x66, 0xe3, 0x30, 0xf1, 0x5f, 0x10, 0xf3, 0xb9, 0xc1, 0x1c, 0x5c, 0x18, 0x85, 0xd0, 0xf0, 0xd8,
-	0x14, 0x8a, 0x91, 0x0b, 0x3d, 0x4c, 0xfa, 0x98, 0xd8, 0xbc, 0x77, 0x8d, 0xc2, 0xed, 0xe6, 0x7f,
+	0xf4, 0x2f, 0xa2, 0xfd, 0x7e, 0x0d, 0x39, 0xfe, 0x10, 0xe8, 0x73, 0x28, 0x8c, 0xe9, 0xc2, 0x0d,
-	0x69, 0x3a, 0x0b, 0x7e, 0x79, 0x11, 0xf3, 0x49, 0xfd, 0x2f, 0x02, 0x14, 0x71, 0x98, 0x36, 0x3f,
+	0xb6, 0xcd, 0x61, 0x3f, 0x59, 0x5b, 0x98, 0x27, 0xba, 0xd9, 0x06, 0x58, 0x3f, 0x85, 0x7c, 0xe4,
-	0x48, 0x3d, 0x1b, 0xf9, 0xad, 0x67, 0x63, 0x23, 0xb0, 0xec, 0x96, 0xc0, 0x62, 0x8d, 0xe4, 0x52,
+	0x42, 0xf7, 0x37, 0x85, 0x4f, 0x6c, 0xde, 0xb9, 0xc6, 0xf9, 0x6e, 0xb7, 0xb8, 0x34, 0x9d, 0x05,
-	0x1a, 0xd9, 0x30, 0x27, 0xbe, 0x95, 0xb9, 0xfc, 0x5b, 0x98, 0x2b, 0xa4, 0x98, 0x7b, 0x08, 0xbb,
+	0x3f, 0xbc, 0x88, 0xf9, 0xa4, 0xfe, 0x37, 0x01, 0xf2, 0x38, 0x7c, 0x67, 0x3f, 0x48, 0xf4, 0x99,
-	0x13, 0x8f, 0xce, 0xd8, 0xc3, 0x40, 0x3d, 0xd3, 0x5b, 0x46, 0xf5, 0xbc, 0x13, 0x5a, 0x87, 0xb1,
+	0xec, 0x4e, 0x9f, 0xd9, 0x66, 0x64, 0x7a, 0x27, 0x23, 0xe3, 0xa4, 0xca, 0x24, 0x92, 0x6a, 0xcb,
-	0xb1, 0x6e, 0x40, 0x09, 0x13, 0x7f, 0x4e, 0x5d, 0x9f, 0xdc, 0x7a, 0x6d, 0x04, 0xa2, 0x65, 0x06,
+	0x9c, 0xf8, 0x46, 0xe6, 0xb2, 0x6f, 0x60, 0x2e, 0x97, 0x60, 0xee, 0x3e, 0xec, 0x4d, 0x3c, 0x3a,
-	0x26, 0xbb, 0x74, 0x15, 0xb3, 0x31, 0x7a, 0x04, 0xe2, 0x98, 0x5a, 0xfc, 0xca, 0xbb, 0xe9, 0x1a,
+	0x63, 0x9d, 0x84, 0x7a, 0xa6, 0xb7, 0x8c, 0x12, 0xa0, 0x12, 0x5a, 0x87, 0xb1, 0xb1, 0x6e, 0x40,
-	0xd2, 0x3c, 0x8f, 0x7a, 0x2d, 0x6a, 0x11, 0xcc, 0x00, 0xf5, 0x39, 0x48, 0x6d, 0xfa, 0xd2, 0x75,
+	0x01, 0x13, 0x7f, 0x4e, 0x5d, 0x9f, 0xbc, 0xf5, 0xd8, 0x08, 0x44, 0xcb, 0x0c, 0x4c, 0x76, 0xe8,
-	0xa8, 0x69, 0xf5, 0x3d, 0x3a, 0x0d, 0x1b, 0xf4, 0xad, 0x8d, 0xa6, 0x0d, 0xc5, 0x05, 0x6b, 0x45,
+	0x32, 0x66, 0x63, 0xf4, 0x00, 0xc4, 0x31, 0xb5, 0xf8, 0x91, 0xf7, 0x92, 0x1a, 0xd2, 0x3c, 0x8f,
-	0x71, 0xab, 0x79, 0xb0, 0xdd, 0x1a, 0xae, 0x6f, 0xc4, 0xfb, 0x56, 0xac, 0xa7, 0x68, 0x69, 0xfd,
+	0x7a, 0x2d, 0x6a, 0x11, 0xcc, 0x00, 0xf5, 0x39, 0x48, 0x6d, 0xfa, 0xc2, 0x75, 0xa8, 0x69, 0xf5,
-	0x6f, 0x02, 0x28, 0xb7, 0xa3, 0x51, 0x07, 0x2a, 0x1c, 0x69, 0xa4, 0xfe, 0x49, 0x8e, 0xde, 0xe5,
+	0x3d, 0x3a, 0x0d, 0x2b, 0xfa, 0x5b, 0x2b, 0x53, 0x1b, 0xf2, 0x0b, 0x56, 0xbb, 0xe2, 0xda, 0x74,
-	0x20, 0xd6, 0x95, 0x60, 0x91, 0x8c, 0xdf, 0xfa, 0xa0, 0xa5, 0xf4, 0x9f, 0x7b, 0x37, 0xfd, 0x3f,
+	0x6f, 0xb7, 0x96, 0x5c, 0x0f, 0xc4, 0x0b, 0x5d, 0x9c, 0x80, 0xd1, 0xd2, 0xfa, 0x3f, 0x04, 0x50,
-	0x82, 0x9d, 0x51, 0x28, 0x98, 0xe4, 0xf9, 0x16, 0xd5, 0xdc, 0x51, 0xbe, 0x99, 0x95, 0x32, 0xb8,
+	0xde, 0x8e, 0x46, 0x1d, 0x28, 0x71, 0xa4, 0x91, 0xf8, 0x89, 0x39, 0x7a, 0x97, 0x8d, 0x58, 0x19,
-	0x3a, 0xe2, 0x4a, 0x62, 0xf6, 0x7a, 0x01, 0xc4, 0xbe, 0xed, 0x4e, 0xeb, 0x87, 0x90, 0x6f, 0x39,
+	0x83, 0xc5, 0x66, 0xfc, 0xc6, 0x0e, 0x98, 0x28, 0x18, 0x99, 0x77, 0x2b, 0x18, 0x0f, 0xa0, 0xc2,
-	0x94, 0x25, 0xac, 0xe0, 0x11, 0xd3, 0xa7, 0x6e, 0xcc, 0x23, 0x9f, 0x1d, 0xff, 0x35, 0x0b, 0x95,
+	0x33, 0x38, 0xee, 0xf7, 0xa2, 0x9a, 0x39, 0xca, 0x36, 0xd3, 0x52, 0x0a, 0x97, 0x47, 0x3c, 0x93,
-	0xd4, 0xaf, 0x15, 0x7a, 0x0c, 0xbb, 0xad, 0xee, 0xf9, 0x60, 0xa8, 0x61, 0xa3, 0xd5, 0xd3, 0x4f,
+	0x98, 0xbd, 0x9e, 0x03, 0xb1, 0x6f, 0xbb, 0xd3, 0x7a, 0x0d, 0xb2, 0x2d, 0x87, 0xb2, 0x07, 0xcb,
-	0x3b, 0x67, 0x52, 0x46, 0x39, 0x58, 0xad, 0x55, 0x79, 0xb6, 0x01, 0x6d, 0xff, 0x35, 0x1d, 0x42,
+	0x79, 0xc4, 0xf4, 0xa9, 0x1b, 0xf3, 0xc8, 0x67, 0xc7, 0x7f, 0x4f, 0x43, 0x29, 0xf1, 0x2f, 0x86,
-	0xbe, 0xa3, 0xb7, 0xb5, 0xdf, 0x4a, 0x82, 0x72, 0x77, 0xb5, 0x56, 0xa5, 0x14, 0x90, 0x3f, 0x41,
+	0x1e, 0xc1, 0x5e, 0xab, 0x7b, 0x3e, 0x18, 0x6a, 0xd8, 0x68, 0xf5, 0xf4, 0xd3, 0xce, 0x99, 0x94,
-	0x9f, 0x40, 0x95, 0x01, 0x8c, 0xf3, 0x7e, 0xbb, 0x31, 0xd4, 0xa4, 0xac, 0xa2, 0xac, 0xd6, 0xea,
+	0x52, 0x0e, 0x57, 0x6b, 0x55, 0x9e, 0x6d, 0x41, 0xbb, 0xbf, 0x59, 0x35, 0xc8, 0x76, 0xf4, 0xb6,
-	0xfe, 0x75, 0x5c, 0xc4, 0xf9, 0x87, 0x50, 0xc4, 0xda, 0x6f, 0xce, 0xb5, 0xc1, 0x50, 0xca, 0x29,
+	0xf6, 0x7b, 0x49, 0x50, 0x6e, 0xaf, 0xd6, 0xaa, 0x94, 0x00, 0xf2, 0x9e, 0xf5, 0x09, 0x94, 0x19,
-	0xfb, 0xab, 0xb5, 0x8a, 0x52, 0xc0, 0x58, 0x35, 0x0f, 0xa1, 0x84, 0xb5, 0x41, 0xbf, 0xa7, 0x0f,
+	0xc0, 0x38, 0xef, 0xb7, 0x1b, 0x43, 0x4d, 0x4a, 0x2b, 0xca, 0x6a, 0xad, 0x1e, 0x5c, 0xc7, 0x45,
-	0x34, 0x49, 0x54, 0x7e, 0xb0, 0x5a, 0xab, 0x77, 0xb6, 0x50, 0x51, 0x95, 0xfe, 0x04, 0xf6, 0xda,
+	0x9c, 0x7f, 0x08, 0x79, 0xac, 0xfd, 0xee, 0x5c, 0x1b, 0x0c, 0xa5, 0x8c, 0x72, 0xb0, 0x5a, 0xab,
-	0xbd, 0xaf, 0xf4, 0x6e, 0xaf, 0xd1, 0x36, 0xfa, 0xb8, 0x77, 0x86, 0xb5, 0xc1, 0x40, 0xca, 0x2b,
+	0x28, 0x01, 0x8c, 0xb3, 0xe6, 0x3e, 0x14, 0xb0, 0x36, 0xe8, 0xf7, 0xf4, 0x81, 0x26, 0x89, 0xca,
-	0x87, 0xab, 0xb5, 0xfa, 0x7e, 0x0a, 0x7f, 0xa3, 0xe8, 0x3e, 0x00, 0xb1, 0xdf, 0xd1, 0xcf, 0xa4,
+	0x8f, 0x56, 0x6b, 0xf5, 0xd6, 0x0e, 0x2a, 0x52, 0xe9, 0xcf, 0x60, 0xbf, 0xdd, 0xfb, 0x52, 0xef,
-	0x82, 0x72, 0x67, 0xb5, 0x56, 0xdf, 0x4b, 0x41, 0x43, 0x52, 0xc3, 0x88, 0x5b, 0xdd, 0xde, 0x40,
+	0xf6, 0x1a, 0x6d, 0xa3, 0x8f, 0x7b, 0x67, 0x58, 0x1b, 0x0c, 0xa4, 0xac, 0x52, 0x5b, 0xad, 0xd5,
-	0x93, 0x8a, 0x37, 0x22, 0x66, 0x64, 0x1f, 0xff, 0x0e, 0xd0, 0xcd, 0x9f, 0x4f, 0xf4, 0x00, 0x44,
+	0xf7, 0x13, 0xf8, 0x1b, 0xa2, 0xfb, 0x00, 0xc4, 0x7e, 0x47, 0x3f, 0x93, 0x72, 0xca, 0xad, 0xd5,
-	0xbd, 0xa7, 0x6b, 0x52, 0x86, 0xc7, 0x7f, 0x13, 0xa1, 0x53, 0x97, 0xa0, 0x3a, 0xe4, 0xba, 0x5f,
+	0x5a, 0x7d, 0x2f, 0x01, 0x0d, 0x49, 0x0d, 0x6f, 0xdc, 0xea, 0xf6, 0x06, 0x9a, 0x94, 0xbf, 0x71,
-	0x7f, 0x29, 0x09, 0xca, 0x0f, 0x57, 0x6b, 0xf5, 0xde, 0x4d, 0x50, 0xf7, 0xeb, 0x2f, 0x8f, 0x29,
+	0x63, 0x46, 0xf6, 0xf1, 0x1f, 0x00, 0xdd, 0xfc, 0x5b, 0x45, 0xf7, 0x40, 0xd4, 0x7b, 0xba, 0x26,
-	0x54, 0xd2, 0x1b, 0xd7, 0xa1, 0xf4, 0x54, 0x1b, 0x36, 0xda, 0x8d, 0x61, 0x43, 0xca, 0xf0, 0x2b,
+	0xa5, 0xf8, 0xfd, 0x6f, 0x22, 0x74, 0xea, 0x12, 0x54, 0x87, 0x4c, 0xf7, 0xab, 0x2f, 0x24, 0x41,
-	0xc5, 0xee, 0xa7, 0x24, 0x30, 0x99, 0x08, 0x0f, 0x20, 0xaf, 0x6b, 0xcf, 0x34, 0x2c, 0x09, 0xca,
+	0xf9, 0xf1, 0x6a, 0xad, 0xde, 0xb9, 0x09, 0xea, 0x7e, 0xf5, 0xc5, 0x31, 0x85, 0x52, 0x32, 0x70,
-	0xde, 0x6a, 0xad, 0xee, 0xc4, 0x00, 0x9d, 0x5c, 0x12, 0x0f, 0xd5, 0xa0, 0xd0, 0xe8, 0x7e, 0xd5,
+	0x1d, 0x0a, 0x4f, 0xb4, 0x61, 0xa3, 0xdd, 0x18, 0x36, 0xa4, 0x14, 0x3f, 0x52, 0xec, 0x7e, 0x42,
-	0x78, 0x3e, 0x90, 0xb2, 0x0a, 0x5a, 0xad, 0xd5, 0xdd, 0xd8, 0xdd, 0x70, 0x5e, 0x9a, 0x4b, 0xff,
+	0x02, 0x93, 0x25, 0xe1, 0x21, 0x64, 0x75, 0xed, 0xa9, 0x86, 0x25, 0x41, 0xd9, 0x5f, 0xad, 0xd5,
-	0xf8, 0xbf, 0x02, 0x54, 0xd3, 0x0f, 0x2e, 0xaa, 0x81, 0x78, 0xda, 0xe9, 0x6a, 0xf1, 0x71, 0x69,
+	0x4a, 0x0c, 0xd0, 0xc9, 0x25, 0xf1, 0x50, 0x15, 0x72, 0x8d, 0xee, 0x97, 0x8d, 0x67, 0x03, 0x29,
-	0x5f, 0x38, 0x46, 0x47, 0x50, 0x6e, 0x77, 0xb0, 0xd6, 0x1a, 0xf6, 0xf0, 0xf3, 0x38, 0x96, 0x34,
+	0xad, 0xa0, 0xd5, 0x5a, 0xdd, 0x8b, 0xdd, 0x0d, 0xe7, 0x85, 0xb9, 0xf4, 0x8f, 0xff, 0x27, 0x40,
-	0xa8, 0x6d, 0x7b, 0xac, 0xc0, 0x97, 0xe8, 0x67, 0x50, 0x1d, 0x3c, 0x7f, 0xda, 0xed, 0xe8, 0xbf,
+	0x39, 0xd9, 0xa1, 0x51, 0x15, 0xc4, 0xd3, 0x4e, 0x57, 0x8b, 0xb7, 0x4b, 0xfa, 0xc2, 0x31, 0x3a,
-	0x36, 0xd8, 0x8e, 0x59, 0xe5, 0xd1, 0x6a, 0xad, 0xde, 0xdf, 0x02, 0x93, 0xb9, 0x47, 0xc6, 0x66,
+	0x82, 0x62, 0xbb, 0x83, 0xb5, 0xd6, 0xb0, 0x87, 0x9f, 0xc5, 0x77, 0x49, 0x82, 0xda, 0xb6, 0xc7,
-	0x40, 0xac, 0x01, 0x7f, 0x44, 0x42, 0x67, 0x49, 0x40, 0x2d, 0xd8, 0x8b, 0x97, 0x6e, 0x0e, 0xcb,
+	0x04, 0xbe, 0x44, 0xbf, 0x80, 0xf2, 0xe0, 0xd9, 0x93, 0x6e, 0x47, 0xff, 0xad, 0xc1, 0x22, 0xa6,
-	0x29, 0x9f, 0xac, 0xd6, 0xea, 0x47, 0xdf, 0xbb, 0x3e, 0x39, 0xbd, 0x24, 0xa0, 0x07, 0x50, 0x8c,
+	0x95, 0x07, 0xab, 0xb5, 0x7a, 0x77, 0x07, 0x4c, 0xe6, 0x1e, 0x19, 0x9b, 0x01, 0xb1, 0x06, 0xbc,
-	0x36, 0x89, 0x2b, 0x29, 0xbd, 0x34, 0x5a, 0x70, 0xfc, 0x67, 0x01, 0xca, 0x49, 0xbb, 0x0a, 0x09,
+	0x89, 0x84, 0xce, 0x82, 0x80, 0x5a, 0xb0, 0x1f, 0x2f, 0xdd, 0x6e, 0x96, 0x51, 0x3e, 0x59, 0xad,
-	0xd7, 0x7b, 0x86, 0x86, 0x71, 0x0f, 0xc7, 0x0c, 0x24, 0x4e, 0x9d, 0xb2, 0x21, 0xba, 0x0f, 0xc5,
+	0xd5, 0x8f, 0xbe, 0x77, 0xfd, 0x66, 0xf7, 0x82, 0x80, 0xee, 0x41, 0x3e, 0x0a, 0x12, 0x2b, 0x29,
-	0x33, 0x4d, 0xd7, 0x70, 0xa7, 0x15, 0x0b, 0x23, 0x81, 0x9c, 0x11, 0x97, 0x78, 0xf6, 0x18, 0x7d,
+	0xb9, 0x34, 0x5a, 0x70, 0xfc, 0x17, 0x01, 0x8a, 0x9b, 0x72, 0x15, 0x12, 0xae, 0xf7, 0x0c, 0x0d,
-	0x0c, 0x55, 0xbd, 0x67, 0x0c, 0xce, 0x5b, 0x4f, 0xe2, 0xd0, 0xd9, 0xf9, 0xa9, 0xad, 0x06, 0x8b,
+	0xe3, 0x1e, 0x8e, 0x19, 0xd8, 0x38, 0x75, 0xca, 0x86, 0xe8, 0x2e, 0xe4, 0xcf, 0x34, 0x5d, 0xc3,
-	0xf1, 0x05, 0xe3, 0xf3, 0x38, 0xd4, 0xd0, 0xb3, 0x46, 0xb7, 0xd3, 0xe6, 0xd0, 0x9c, 0x22, 0xaf,
+	0x9d, 0x56, 0x9c, 0x18, 0x1b, 0xc8, 0x19, 0x71, 0x89, 0x67, 0x8f, 0xd1, 0xc7, 0x50, 0xd6, 0x7b,
-	0xd6, 0xea, 0xdd, 0x04, 0xda, 0xe1, 0x7f, 0x1e, 0x21, 0xf6, 0xd8, 0x82, 0xda, 0xf7, 0x37, 0x26,
+	0xc6, 0xe0, 0xbc, 0xf5, 0x38, 0xbe, 0x3a, 0xdb, 0x3f, 0x11, 0x6a, 0xb0, 0x18, 0x5f, 0x30, 0x3e,
-	0xa4, 0x42, 0xa1, 0xd1, 0xef, 0x6b, 0x7a, 0x3b, 0xbe, 0xfd, 0xc6, 0xd7, 0x98, 0xcf, 0x89, 0x6b,
+	0x8f, 0xc3, 0x1c, 0x7a, 0xda, 0xe8, 0x76, 0xda, 0x1c, 0x9a, 0x51, 0xe4, 0xd5, 0x5a, 0xbd, 0xbd,
-	0x85, 0x88, 0xd3, 0x1e, 0x3e, 0xd3, 0x86, 0xf1, 0xe5, 0x37, 0x88, 0x53, 0x1a, 0xbe, 0xe0, 0xcd,
+	0x81, 0x76, 0xf8, 0xaf, 0x4a, 0x88, 0x3d, 0xb6, 0xa0, 0xfa, 0xfd, 0x85, 0x09, 0xa9, 0x90, 0x6b,
-	0x83, 0xd7, 0xdf, 0xd5, 0x32, 0xdf, 0x7e, 0x57, 0xcb, 0xbc, 0xbe, 0xaa, 0x09, 0xdf, 0x5e, 0xd5,
+	0xf4, 0xfb, 0x9a, 0xde, 0x8e, 0x4f, 0xbf, 0xf5, 0x35, 0xe6, 0x73, 0xe2, 0x5a, 0x21, 0xe2, 0xb4,
-	0x84, 0x7f, 0x5c, 0xd5, 0x32, 0xff, 0xba, 0xaa, 0x09, 0xdf, 0xfc, 0xb3, 0x26, 0x8c, 0x0a, 0xac,
+	0x87, 0xcf, 0xb4, 0x61, 0x7c, 0xf8, 0x2d, 0xe2, 0x94, 0x86, 0x1d, 0xbc, 0x79, 0xf8, 0xea, 0xbb,
-	0x91, 0x7d, 0xf1, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0xad, 0x8a, 0xef, 0x7f, 0x8f, 0x0e, 0x00,
+	0x6a, 0xea, 0xdb, 0xef, 0xaa, 0xa9, 0x57, 0x57, 0x55, 0xe1, 0xdb, 0xab, 0xaa, 0xf0, 0xaf, 0xab,
-	0x00,
+	0x6a, 0xea, 0x3f, 0x57, 0x55, 0xe1, 0x9b, 0x7f, 0x57, 0x85, 0x51, 0x8e, 0x15, 0xb2, 0xcf, 0xff,
+	0x1f, 0x00, 0x00, 0xff, 0xff, 0x0d, 0x36, 0x7c, 0x9b, 0xc0, 0x0e, 0x00, 0x00,
 }

lib/protocol/bep.proto

@@ -105,6 +105,7 @@ message FileInfo {
     bool         no_permissions = 8;
     Vector       version        = 9 [(gogoproto.nullable) = false];
     int64        sequence       = 10;
+    int32        block_size     = 13 [(gogoproto.customname) = "RawBlockSize"];
 
     repeated BlockInfo Blocks         = 16 [(gogoproto.nullable) = false];
     string             symlink_target = 17;

lib/protocol/bep_extensions.go

@@ -14,16 +14,11 @@ import (
 	"time"
 
 	"github.com/syncthing/syncthing/lib/rand"
-	"github.com/syncthing/syncthing/lib/sha256"
 )
 
 const (
 	SyntheticDirectorySize = 128
-)
+	HelloMessageMagic      = uint32(0x2EA7D90B)
-
-var (
-	sha256OfEmptyBlock = sha256.Sum256(make([]byte, BlockSize))
-	HelloMessageMagic  = uint32(0x2EA7D90B)
 )
 
 func (m Hello) Magic() uint32 {
@@ -36,8 +31,8 @@ func (f FileInfo) String() string {
 		return fmt.Sprintf("Directory{Name:%q, Sequence:%d, Permissions:0%o, ModTime:%v, Version:%v, Deleted:%v, Invalid:%v, NoPermissions:%v}",
 			f.Name, f.Sequence, f.Permissions, f.ModTime(), f.Version, f.Deleted, f.Invalid, f.NoPermissions)
 	case FileInfoTypeFile:
-		return fmt.Sprintf("File{Name:%q, Sequence:%d, Permissions:0%o, ModTime:%v, Version:%v, Length:%d, Deleted:%v, Invalid:%v, NoPermissions:%v, Blocks:%v}",
+		return fmt.Sprintf("File{Name:%q, Sequence:%d, Permissions:0%o, ModTime:%v, Version:%v, Length:%d, Deleted:%v, Invalid:%v, NoPermissions:%v, BlockSize:%d, Blocks:%v}",
-			f.Name, f.Sequence, f.Permissions, f.ModTime(), f.Version, f.Size, f.Deleted, f.Invalid, f.NoPermissions, f.Blocks)
+			f.Name, f.Sequence, f.Permissions, f.ModTime(), f.Version, f.Size, f.Deleted, f.Invalid, f.NoPermissions, f.RawBlockSize, f.Blocks)
 	case FileInfoTypeSymlink, FileInfoTypeDeprecatedSymlinkDirectory, FileInfoTypeDeprecatedSymlinkFile:
 		return fmt.Sprintf("Symlink{Name:%q, Type:%v, Sequence:%d, Version:%v, Deleted:%v, Invalid:%v, NoPermissions:%v, SymlinkTarget:%q}",
 			f.Name, f.Type, f.Sequence, f.Version, f.Deleted, f.Invalid, f.NoPermissions, f.SymlinkTarget)
@@ -81,6 +76,13 @@ func (f FileInfo) FileSize() int64 {
 	return f.Size
 }
 
+func (f FileInfo) BlockSize() int {
+	if f.RawBlockSize == 0 {
+		return MinBlockSize
+	}
+	return int(f.RawBlockSize)
+}
+
 func (f FileInfo) FileName() string {
 	return f.Name
 }
@@ -204,7 +206,10 @@ func (b BlockInfo) String() string {
 
 // IsEmpty returns true if the block is a full block of zeroes.
 func (b BlockInfo) IsEmpty() bool {
-	return b.Size == BlockSize && bytes.Equal(b.Hash, sha256OfEmptyBlock[:])
+	if v, ok := sha256OfEmptyBlock[int(b.Size)]; ok {
+		return bytes.Equal(b.Hash, v[:])
+	}
+	return false
 }
 
 type IndexID uint64

lib/protocol/protocol.go

@@ -3,6 +3,7 @@
 package protocol
 
 import (
+	"crypto/sha256"
 	"encoding/binary"
 	"errors"
 	"fmt"
@@ -16,13 +17,51 @@ import (
 )
 
 const (
-	// BlockSize is the standard data block size (128 KiB)
+	// Shifts
-	BlockSize = 128 << 10
+	KiB = 10
+	MiB = 20
+	GiB = 30
+)
 
+const (
 	// MaxMessageLen is the largest message size allowed on the wire. (500 MB)
 	MaxMessageLen = 500 * 1000 * 1000
+
+	// MinBlockSize is the minimum block size allowed
+	MinBlockSize = 128 << KiB
+
+	// MaxBlockSize is the maximum block size allowed
+	MaxBlockSize = 16 << MiB
+
+	// DesiredPerFileBlocks is the number of blocks we aim for per file
+	DesiredPerFileBlocks = 2000
 )
 
+// BlockSizes is the list of valid block sizes, from min to max
+var BlockSizes []int
+
+// For each block size, the hash of a block of all zeroes
+var sha256OfEmptyBlock = make(map[int][sha256.Size]byte)
+
+func init() {
+	for blockSize := MinBlockSize; blockSize <= MaxBlockSize; blockSize *= 2 {
+		BlockSizes = append(BlockSizes, blockSize)
+		sha256OfEmptyBlock[blockSize] = sha256.Sum256(make([]byte, blockSize))
+	}
+}
+
+// BlockSize returns the block size to use for the given file size
+func BlockSize(fileSize int64) int {
+	var blockSize int
+	for _, blockSize = range BlockSizes {
+		if fileSize < int64(DesiredPerFileBlocks*blockSize) {
+			break
+		}
+	}
+
+	return blockSize
+}
+
 const (
 	stateInitial = iota
 	stateReady
@@ -158,7 +197,7 @@ func NewConnection(deviceID DeviceID, reader io.Reader, writer io.Writer, receiv
 		awaiting:    make(map[int32]chan asyncResult),
 		outbox:      make(chan asyncMessage),
 		closed:      make(chan struct{}),
-		pool:        bufferPool{minSize: BlockSize},
+		pool:        bufferPool{minSize: MinBlockSize},
 		compression: compress,
 	}
 
@@ -533,7 +572,7 @@ func checkFilename(name string) error {
 
 func (c *rawConnection) handleRequest(req Request) {
 	size := int(req.Size)
-	usePool := size <= BlockSize
+	usePool := size <= MaxBlockSize
 
 	var buf []byte
 	var done chan struct{}

lib/protocol/protocol_test.go

@@ -11,7 +11,6 @@ import (
 	"strings"
 	"testing"
 	"testing/quick"
-	"time"
 
 	"encoding/hex"
 
@@ -234,48 +233,6 @@ func testMarshal(t *testing.T, prefix string, m1, m2 message) bool {
 	return true
 }
 
-func TestMarshalledIndexMessageSize(t *testing.T) {
-	// We should be able to handle a 1 TiB file without
-	// blowing the default max message size.
-
-	if testing.Short() {
-		t.Skip("this test requires a lot of memory")
-		return
-	}
-
-	const (
-		maxMessageSize = MaxMessageLen
-		fileSize       = 1 << 40
-		blockSize      = BlockSize
-	)
-
-	f := FileInfo{
-		Name:        "a normal length file name withoout any weird stuff.txt",
-		Type:        FileInfoTypeFile,
-		Size:        fileSize,
-		Permissions: 0666,
-		ModifiedS:   time.Now().Unix(),
-		Version:     Vector{Counters: []Counter{{ID: 1 << 60, Value: 1}, {ID: 2 << 60, Value: 1}}},
-		Blocks:      make([]BlockInfo, fileSize/blockSize),
-	}
-
-	for i := 0; i < fileSize/blockSize; i++ {
-		f.Blocks[i].Offset = int64(i) * blockSize
-		f.Blocks[i].Size = blockSize
-		f.Blocks[i].Hash = []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 20, 1, 2, 3, 4, 5, 6, 7, 8, 9, 30, 1, 2}
-	}
-
-	idx := Index{
-		Folder: "some folder ID",
-		Files:  []FileInfo{f},
-	}
-
-	msgSize := idx.ProtoSize()
-	if msgSize > maxMessageSize {
-		t.Errorf("Message size %d bytes is larger than max %d", msgSize, maxMessageSize)
-	}
-}
-
 func TestLZ4Compression(t *testing.T) {
 	c := new(rawConnection)
 
@@ -400,3 +357,36 @@ func TestCheckConsistency(t *testing.T) {
 		}
 	}
 }
+
+func TestBlockSize(t *testing.T) {
+	cases := []struct {
+		fileSize  int64
+		blockSize int
+	}{
+		{1 << KiB, 128 << KiB},
+		{1 << MiB, 128 << KiB},
+		{499 << MiB, 256 << KiB},
+		{500 << MiB, 512 << KiB},
+		{501 << MiB, 512 << KiB},
+		{1 << GiB, 1 << MiB},
+		{2 << GiB, 2 << MiB},
+		{3 << GiB, 2 << MiB},
+		{500 << GiB, 16 << MiB},
+		{50000 << GiB, 16 << MiB},
+	}
+
+	for _, tc := range cases {
+		size := BlockSize(tc.fileSize)
+		if size != tc.blockSize {
+			t.Errorf("BlockSize(%d), size=%d, expected %d", tc.fileSize, size, tc.blockSize)
+		}
+	}
+}
+
+var blockSize int
+
+func BenchmarkBlockSize(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		blockSize = BlockSize(16 << 30)
+	}
+}

lib/scanner/blockqueue.go

@@ -63,7 +63,6 @@ func HashFile(ctx context.Context, fs fs.Filesystem, path string, blockSize int,
 // is closed and all items handled.
 type parallelHasher struct {
 	fs            fs.Filesystem
-	blockSize     int
 	workers       int
 	outbox        chan<- protocol.FileInfo
 	inbox         <-chan protocol.FileInfo
@@ -73,10 +72,9 @@ type parallelHasher struct {
 	wg            sync.WaitGroup
 }
 
-func newParallelHasher(ctx context.Context, fs fs.Filesystem, blockSize, workers int, outbox chan<- protocol.FileInfo, inbox <-chan protocol.FileInfo, counter Counter, done chan<- struct{}, useWeakHashes bool) {
+func newParallelHasher(ctx context.Context, fs fs.Filesystem, workers int, outbox chan<- protocol.FileInfo, inbox <-chan protocol.FileInfo, counter Counter, done chan<- struct{}, useWeakHashes bool) {
 	ph := &parallelHasher{
 		fs:            fs,
-		blockSize:     blockSize,
 		workers:       workers,
 		outbox:        outbox,
 		inbox:         inbox,
@@ -108,7 +106,7 @@ func (ph *parallelHasher) hashFiles(ctx context.Context) {
 				panic("Bug. Asked to hash a directory or a deleted file.")
 			}
 
-			blocks, err := HashFile(ctx, ph.fs, f.Name, ph.blockSize, ph.counter, ph.useWeakHashes)
+			blocks, err := HashFile(ctx, ph.fs, f.Name, f.BlockSize(), ph.counter, ph.useWeakHashes)
 			if err != nil {
 				l.Debugln("hash error:", f.Name, err)
 				continue

lib/scanner/blocks_test.go

@@ -125,7 +125,7 @@ func TestAdler32Variants(t *testing.T) {
 	}
 
 	// protocol block sized data
-	data := make([]byte, protocol.BlockSize)
+	data := make([]byte, protocol.MinBlockSize)
 	for i := 0; i < 5; i++ {
 		rand.Read(data)
 		if !checkFn(data) {

lib/scanner/virtualfs_test.go

@@ -55,6 +55,48 @@ func (i infiniteFS) Open(name string) (fs.File, error) {
 	return &fakeFile{name, i.filesize, 0}, nil
 }
 
+type singleFileFS struct {
+	fs.Filesystem
+	name     string
+	filesize int64
+}
+
+func (s singleFileFS) Lstat(name string) (fs.FileInfo, error) {
+	switch name {
+	case ".":
+		return fakeInfo{".", 0}, nil
+	case s.name:
+		return fakeInfo{s.name, s.filesize}, nil
+	default:
+		return nil, errors.New("no such file")
+	}
+}
+
+func (s singleFileFS) Stat(name string) (fs.FileInfo, error) {
+	switch name {
+	case ".":
+		return fakeInfo{".", 0}, nil
+	case s.name:
+		return fakeInfo{s.name, s.filesize}, nil
+	default:
+		return nil, errors.New("no such file")
+	}
+}
+
+func (s singleFileFS) DirNames(name string) ([]string, error) {
+	if name != "." {
+		return nil, errors.New("no such file")
+	}
+	return []string{s.name}, nil
+}
+
+func (s singleFileFS) Open(name string) (fs.File, error) {
+	if name != s.name {
+		return nil, errors.New("no such file")
+	}
+	return &fakeFile{s.name, s.filesize, 0}, nil
+}
+
 type fakeInfo struct {
 	name string
 	size int64

lib/scanner/walk.go

@@ -42,8 +42,6 @@ type Config struct {
 	Folder string
 	// Limit walking to these paths within Dir, or no limit if Sub is empty
 	Subs []string
-	// BlockSize controls the size of the block used when hashing.
-	BlockSize int
 	// If Matcher is not nil, it is used to identify files to ignore which were specified by the user.
 	Matcher *ignore.Matcher
 	// Number of hours to keep temporary files for
@@ -68,6 +66,8 @@ type Config struct {
 	ProgressTickIntervalS int
 	// Whether or not we should also compute weak hashes
 	UseWeakHashes bool
+	// Whether to use large blocks for large files or the old standard of 128KiB for everything.
+	UseLargeBlocks bool
 }
 
 type CurrentFiler interface {
@@ -98,7 +98,7 @@ type walker struct {
 // Walk returns the list of files found in the local folder by scanning the
 // file system. Files are blockwise hashed.
 func (w *walker) walk(ctx context.Context) chan protocol.FileInfo {
-	l.Debugln("Walk", w.Subs, w.BlockSize, w.Matcher)
+	l.Debugln("Walk", w.Subs, w.Matcher)
 
 	toHashChan := make(chan protocol.FileInfo)
 	finishedChan := make(chan protocol.FileInfo)
@@ -120,7 +120,7 @@ func (w *walker) walk(ctx context.Context) chan protocol.FileInfo {
 	// We're not required to emit scan progress events, just kick off hashers,
 	// and feed inputs directly from the walker.
 	if w.ProgressTickIntervalS < 0 {
-		newParallelHasher(ctx, w.Filesystem, w.BlockSize, w.Hashers, finishedChan, toHashChan, nil, nil, w.UseWeakHashes)
+		newParallelHasher(ctx, w.Filesystem, w.Hashers, finishedChan, toHashChan, nil, nil, w.UseWeakHashes)
 		return finishedChan
 	}
 
@@ -151,7 +151,7 @@ func (w *walker) walk(ctx context.Context) chan protocol.FileInfo {
 		done := make(chan struct{})
 		progress := newByteCounter()
 
-		newParallelHasher(ctx, w.Filesystem, w.BlockSize, w.Hashers, finishedChan, realToHashChan, progress, done, w.UseWeakHashes)
+		newParallelHasher(ctx, w.Filesystem, w.Hashers, finishedChan, realToHashChan, progress, done, w.UseWeakHashes)
 
 		// A routine which actually emits the FolderScanProgress events
 		// every w.ProgressTicker ticks, until the hasher routines terminate.
@@ -161,7 +161,7 @@ func (w *walker) walk(ctx context.Context) chan protocol.FileInfo {
 			for {
 				select {
 				case <-done:
-					l.Debugln("Walk progress done", w.Folder, w.Subs, w.BlockSize, w.Matcher)
+					l.Debugln("Walk progress done", w.Folder, w.Subs, w.Matcher)
 					ticker.Stop()
 					return
 				case <-ticker.C:
@@ -285,32 +285,52 @@ func (w *walker) walkRegular(ctx context.Context, relPath string, info fs.FileIn
 		curMode |= 0111
 	}
 
-	cf, ok := w.CurrentFiler.CurrentFile(relPath)
+	blockSize := protocol.MinBlockSize
+	curFile, hasCurFile := w.CurrentFiler.CurrentFile(relPath)
+
+	if w.UseLargeBlocks {
+		blockSize = protocol.BlockSize(info.Size())
+
+		if hasCurFile {
+			// Check if we should retain current block size.
+			curBlockSize := curFile.BlockSize()
+			if blockSize > curBlockSize && blockSize/curBlockSize <= 2 {
+				// New block size is larger, but not more than twice larger.
+				// Retain.
+				blockSize = curBlockSize
+			} else if curBlockSize > blockSize && curBlockSize/blockSize <= 2 {
+				// Old block size is larger, but not more than twice larger.
+				// Retain.
+				blockSize = curBlockSize
+			}
+		}
+	}
 
 	f := protocol.FileInfo{
 		Name:          relPath,
 		Type:          protocol.FileInfoTypeFile,
-		Version:       cf.Version.Update(w.ShortID),
+		Version:       curFile.Version.Update(w.ShortID),
 		Permissions:   curMode & uint32(maskModePerm),
 		NoPermissions: w.IgnorePerms,
 		ModifiedS:     info.ModTime().Unix(),
 		ModifiedNs:    int32(info.ModTime().Nanosecond()),
 		ModifiedBy:    w.ShortID,
 		Size:          info.Size(),
+		RawBlockSize:  int32(blockSize),
 	}
 
-	if ok {
+	if hasCurFile {
-		if cf.IsEquivalent(f, w.IgnorePerms, true) {
+		if curFile.IsEquivalent(f, w.IgnorePerms, true) {
 			return nil
 		}
-		if cf.Invalid {
+		if curFile.Invalid {
 			// We do not want to override the global version with the file we
 			// currently have. Keeping only our local counter makes sure we are in
 			// conflict with any other existing versions, which will be resolved by
 			// the normal pulling mechanisms.
 			f.Version = f.Version.DropOthers(w.ShortID)
 		}
-		l.Debugln("rescan:", cf, info.ModTime().Unix(), info.Mode()&fs.ModePerm)
+		l.Debugln("rescan:", curFile, info.ModTime().Unix(), info.Mode()&fs.ModePerm)
 	}
 
 	l.Debugln("to hash:", relPath, f)

lib/scanner/walk_test.go

@@ -65,7 +65,6 @@ func TestWalkSub(t *testing.T) {
 	fchan := Walk(context.TODO(), Config{
 		Filesystem: fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata"),
 		Subs:       []string{"dir2"},
-		BlockSize:  128 * 1024,
 		Matcher:    ignores,
 		Hashers:    2,
 	})
@@ -98,7 +97,6 @@ func TestWalk(t *testing.T) {
 
 	fchan := Walk(context.TODO(), Config{
 		Filesystem: fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata"),
-		BlockSize:  128 * 1024,
 		Matcher:    ignores,
 		Hashers:    2,
 	})
@@ -221,11 +219,11 @@ func TestNormalization(t *testing.T) {
 	// make sure it all gets done. In production, things will be correct
 	// eventually...
 
-	_, err := walkDir(fs, "testdata/normalization")
+	_, err := walkDir(fs, "testdata/normalization", nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	tmp, err := walkDir(fs, "testdata/normalization")
+	tmp, err := walkDir(fs, "testdata/normalization", nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -272,7 +270,7 @@ func TestWalkSymlinkUnix(t *testing.T) {
 
 	for _, path := range []string{".", "link"} {
 		// Scan it
-		files, _ := walkDir(fs.NewFilesystem(fs.FilesystemTypeBasic, "_symlinks"), path)
+		files, _ := walkDir(fs.NewFilesystem(fs.FilesystemTypeBasic, "_symlinks"), path, nil)
 
 		// Verify that we got one symlink and with the correct attributes
 		if len(files) != 1 {
@@ -303,7 +301,7 @@ func TestWalkSymlinkWindows(t *testing.T) {
 
 	for _, path := range []string{".", "link"} {
 		// Scan it
-		files, _ := walkDir(fs.NewFilesystem(fs.FilesystemTypeBasic, "_symlinks"), path)
+		files, _ := walkDir(fs.NewFilesystem(fs.FilesystemTypeBasic, "_symlinks"), path, nil)
 
 		// Verify that we got zero symlinks
 		if len(files) != 0 {
@@ -332,7 +330,7 @@ func TestWalkRootSymlink(t *testing.T) {
 	}
 
 	// Scan it
-	files, err := walkDir(fs.NewFilesystem(fs.FilesystemTypeBasic, link), ".")
+	files, err := walkDir(fs.NewFilesystem(fs.FilesystemTypeBasic, link), ".", nil)
 	if err != nil {
 		t.Fatal("Expected no error when root folder path is provided via a symlink: " + err.Error())
 	}
@@ -342,13 +340,83 @@ func TestWalkRootSymlink(t *testing.T) {
 	}
 }
 
-func walkDir(fs fs.Filesystem, dir string) ([]protocol.FileInfo, error) {
+func TestBlocksizeHysteresis(t *testing.T) {
+	// Verify that we select the right block size in the presence of old
+	// file information.
+
+	sf := fs.NewWalkFilesystem(&singleFileFS{
+		name:     "testfile.dat",
+		filesize: 500 << 20, // 500 MiB
+	})
+
+	current := make(fakeCurrentFiler)
+
+	runTest := func(expectedBlockSize int) {
+		files, err := walkDir(sf, ".", current)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(files) != 1 {
+			t.Fatalf("expected one file, not %d", len(files))
+		}
+		if s := files[0].BlockSize(); s != expectedBlockSize {
+			t.Fatalf("incorrect block size %d != expected %d", s, expectedBlockSize)
+		}
+	}
+
+	// Scan with no previous knowledge. We should get a 512 KiB block size.
+
+	runTest(512 << 10)
+
+	// Scan on the assumption that previous size was 256 KiB. Retain 256 KiB
+	// block size.
+
+	current["testfile.dat"] = protocol.FileInfo{
+		Name:         "testfile.dat",
+		Size:         500 << 20,
+		RawBlockSize: 256 << 10,
+	}
+	runTest(256 << 10)
+
+	// Scan on the assumption that previous size was 1 MiB. Retain 1 MiB
+	// block size.
+
+	current["testfile.dat"] = protocol.FileInfo{
+		Name:         "testfile.dat",
+		Size:         500 << 20,
+		RawBlockSize: 1 << 20,
+	}
+	runTest(1 << 20)
+
+	// Scan on the assumption that previous size was 128 KiB. Move to 512
+	// KiB because the difference is large.
+
+	current["testfile.dat"] = protocol.FileInfo{
+		Name:         "testfile.dat",
+		Size:         500 << 20,
+		RawBlockSize: 128 << 10,
+	}
+	runTest(512 << 10)
+
+	// Scan on the assumption that previous size was 2 MiB. Move to 512
+	// KiB because the difference is large.
+
+	current["testfile.dat"] = protocol.FileInfo{
+		Name:         "testfile.dat",
+		Size:         500 << 20,
+		RawBlockSize: 2 << 20,
+	}
+	runTest(512 << 10)
+}
+
+func walkDir(fs fs.Filesystem, dir string, cfiler CurrentFiler) ([]protocol.FileInfo, error) {
 	fchan := Walk(context.TODO(), Config{
-		Filesystem:    fs,
+		Filesystem:     fs,
-		Subs:          []string{dir},
+		Subs:           []string{dir},
-		BlockSize:     128 * 1024,
+		AutoNormalize:  true,
-		AutoNormalize: true,
+		Hashers:        2,
-		Hashers:       2,
+		UseLargeBlocks: true,
+		CurrentFiler:   cfiler,
 	})
 
 	var tmp []protocol.FileInfo
@@ -410,7 +478,7 @@ func BenchmarkHashFile(b *testing.B) {
 	b.ResetTimer()
 
 	for i := 0; i < b.N; i++ {
-		if _, err := HashFile(context.TODO(), fs.NewFilesystem(fs.FilesystemTypeBasic, ""), testdataName, protocol.BlockSize, nil, true); err != nil {
+		if _, err := HashFile(context.TODO(), fs.NewFilesystem(fs.FilesystemTypeBasic, ""), testdataName, protocol.MinBlockSize, nil, true); err != nil {
 			b.Fatal(err)
 		}
 	}
@@ -451,7 +519,6 @@ func TestStopWalk(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	fchan := Walk(ctx, Config{
 		Filesystem:            fs,
-		BlockSize:             128 * 1024,
 		Hashers:               numHashers,
 		ProgressTickIntervalS: -1, // Don't attempt to build the full list of files before starting to scan...
 	})
@@ -513,7 +580,7 @@ func TestIssue4799(t *testing.T) {
 	}
 	fd.Close()
 
-	files, err := walkDir(fs, "/foo")
+	files, err := walkDir(fs, "/foo", nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -540,7 +607,6 @@ func TestIssue4841(t *testing.T) {
 	fchan := Walk(context.TODO(), Config{
 		Filesystem:    fs,
 		Subs:          nil,
-		BlockSize:     128 * 1024,
 		AutoNormalize: true,
 		Hashers:       2,
 		CurrentFiler: fakeCurrentFiler{

test/h1/config.xml

@@ -19,6 +19,7 @@
         <paused>false</paused>
         <weakHashThresholdPct>25</weakHashThresholdPct>
         <markerName>.stfolder</markerName>
+        <useLargeBlocks>true</useLargeBlocks>
     </folder>
     <folder id="¯\_(ツ)_/¯ Räksmörgås 动作 Адрес" label="" path="s12-1/" type="readwrite" rescanIntervalS="10" fsWatcherEnabled="false" fsWatcherDelayS="10" ignorePerms="false" autoNormalize="true">
         <filesystemType>basic</filesystemType>
@@ -38,6 +39,7 @@
         <paused>false</paused>
         <weakHashThresholdPct>25</weakHashThresholdPct>
         <markerName>.stfolder</markerName>
+        <useLargeBlocks>true</useLargeBlocks>
     </folder>
     <device id="EJHMPAQ-OGCVORE-ISB4IS3-SYYVJXF-TKJGLTU-66DIQPF-GJ5D2GX-GQ3OWQK" name="s4" compression="metadata" introducer="false" skipIntroductionRemovals="false" introducedBy="">
         <address>tcp://127.0.0.1:22004</address>

test/h2/config.xml

@@ -18,6 +18,7 @@
         <paused>false</paused>
         <weakHashThresholdPct>25</weakHashThresholdPct>
         <markerName>.stfolder</markerName>
+        <useLargeBlocks>true</useLargeBlocks>
     </folder>
     <folder id="s23" label="" path="s23-2" type="readwrite" rescanIntervalS="15" fsWatcherEnabled="false" fsWatcherDelayS="10" ignorePerms="false" autoNormalize="true">
         <filesystemType>basic</filesystemType>
@@ -37,6 +38,7 @@
         <paused>false</paused>
         <weakHashThresholdPct>25</weakHashThresholdPct>
         <markerName>.stfolder</markerName>
+        <useLargeBlocks>true</useLargeBlocks>
     </folder>
     <folder id="¯\_(ツ)_/¯ Räksmörgås 动作 Адрес" label="" path="s12-2" type="readwrite" rescanIntervalS="15" fsWatcherEnabled="false" fsWatcherDelayS="10" ignorePerms="false" autoNormalize="true">
         <filesystemType>basic</filesystemType>
@@ -56,6 +58,7 @@
         <paused>false</paused>
         <weakHashThresholdPct>25</weakHashThresholdPct>
         <markerName>.stfolder</markerName>
+        <useLargeBlocks>true</useLargeBlocks>
     </folder>
     <device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU" name="s1" compression="metadata" introducer="false" skipIntroductionRemovals="false" introducedBy="">
         <address>tcp://127.0.0.1:22001</address>

test/h3/config.xml

@@ -20,6 +20,7 @@
         <paused>false</paused>
         <weakHashThresholdPct>25</weakHashThresholdPct>
         <markerName>.stfolder</markerName>
+        <useLargeBlocks>true</useLargeBlocks>
     </folder>
     <folder id="s23" label="" path="s23-3" type="readwrite" rescanIntervalS="20" fsWatcherEnabled="false" fsWatcherDelayS="10" ignorePerms="false" autoNormalize="true">
         <filesystemType>basic</filesystemType>
@@ -39,6 +40,7 @@
         <paused>false</paused>
         <weakHashThresholdPct>25</weakHashThresholdPct>
         <markerName>.stfolder</markerName>
+        <useLargeBlocks>true</useLargeBlocks>
     </folder>
     <device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU" name="s1" compression="metadata" introducer="false" skipIntroductionRemovals="false" introducedBy="">
         <address>tcp://127.0.0.1:22001</address>