octoleo/syncthing
1
0
Fork 0
mirror of https://github.com/octoleo/syncthing.git synced 2024-11-08 22:31:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1219 from syncthing/refactor-truncated

Browse Source 
Refactor stuff around FileInfoTruncated
This commit is contained in:
Audrius Butkevicius 2015-01-09 10:12:48 +00:00
commit a7b75a54bb
12 changed files with 288 additions and 249 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
cmd/stindex/main.go
View File

@ -43,8 +43,8 @@ func main() {
if *device == "" {
log.Printf("*** Global index for folder %q", *folder)
fs.WithGlobalTruncated(func(fi protocol.FileIntf) bool {
f := fi.(protocol.FileInfoTruncated)
fs.WithGlobalTruncated(func(fi files.FileIntf) bool {
f := fi.(files.FileInfoTruncated)
fmt.Println(f)
fmt.Println("\t", fs.Availability(f.Name))
return true
@ -55,8 +55,8 @@ func main() {
log.Fatal(err)
}
log.Printf("*** Have index for folder %q device %q", *folder, n)
fs.WithHaveTruncated(n, func(fi protocol.FileIntf) bool {
f := fi.(protocol.FileInfoTruncated)
fs.WithHaveTruncated(n, func(fi files.FileIntf) bool {
f := fi.(files.FileInfoTruncated)
fmt.Println(f)
return true
})

9
cmd/syncthing/gui.go
View File

@ -36,6 +36,7 @@ import (
"github.com/syncthing/syncthing/internal/config"
"github.com/syncthing/syncthing/internal/discover"
"github.com/syncthing/syncthing/internal/events"
"github.com/syncthing/syncthing/internal/files"
"github.com/syncthing/syncthing/internal/model"
"github.com/syncthing/syncthing/internal/osutil"
"github.com/syncthing/syncthing/internal/protocol"
@ -782,9 +783,9 @@ func mimeTypeForFile(file string) string {
}
}
func toNeedSlice(files []protocol.FileInfoTruncated) []map[string]interface{} {
output := make([]map[string]interface{}, len(files))
for i, file := range files {
func toNeedSlice(fs []files.FileInfoTruncated) []map[string]interface{} {
output := make([]map[string]interface{}, len(fs))
for i, file := range fs {
output[i] = map[string]interface{}{
"Name": file.Name,
"Flags": file.Flags,
@ -792,7 +793,7 @@ func toNeedSlice(files []protocol.FileInfoTruncated) []map[string]interface{} {
"Version": file.Version,
"LocalVersion": file.LocalVersion,
"NumBlocks": file.NumBlocks,
"Size": protocol.BlocksToSize(file.NumBlocks),
"Size": files.BlocksToSize(file.NumBlocks),
}
}
return output

31
internal/files/leveldb.go
View File

@ -166,8 +166,6 @@ func globalKeyFolder(key []byte) []byte {
type deletionHandler func(db dbReader, batch dbWriter, folder, device, name []byte, dbi iterator.Iterator) uint64
type fileIterator func(f protocol.FileIntf) bool
func ldbGenericReplace(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo, deleteFn deletionHandler) uint64 {
runtime.GC()
@ -246,7 +244,7 @@ func ldbGenericReplace(db *leveldb.DB, folder, device []byte, fs []protocol.File
if debugDB {
l.Debugln("generic replace; exists - compare")
}
var ef protocol.FileInfoTruncated
var ef FileInfoTruncated
ef.UnmarshalXDR(dbi.Value())
if fs[fsi].Version > ef.Version ||
(fs[fsi].Version == ef.Version && fs[fsi].Flags != ef.Flags) {
@ -308,7 +306,7 @@ func ldbReplace(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo) u
func ldbReplaceWithDelete(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo) uint64 {
return ldbGenericReplace(db, folder, device, fs, func(db dbReader, batch dbWriter, folder, device, name []byte, dbi iterator.Iterator) uint64 {
var tf protocol.FileInfoTruncated
var tf FileInfoTruncated
err := tf.UnmarshalXDR(dbi.Value())
if err != nil {
panic(err)
@ -378,7 +376,7 @@ func ldbUpdate(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo) ui
continue
}
var ef protocol.FileInfoTruncated
var ef FileInfoTruncated
err = ef.UnmarshalXDR(bs)
if err != nil {
panic(err)
@ -528,7 +526,7 @@ func ldbRemoveFromGlobal(db dbReader, batch dbWriter, folder, device, file []byt
}
}
func ldbWithHave(db *leveldb.DB, folder, device []byte, truncate bool, fn fileIterator) {
func ldbWithHave(db *leveldb.DB, folder, device []byte, truncate bool, fn Iterator) {
start := deviceKey(folder, device, nil) // before all folder/device files
limit := deviceKey(folder, device, []byte{0xff, 0xff, 0xff, 0xff}) // after all folder/device files
snap, err := db.GetSnapshot()
@ -559,7 +557,7 @@ func ldbWithHave(db *leveldb.DB, folder, device []byte, truncate bool, fn fileIt
}
}
func ldbWithAllFolderTruncated(db *leveldb.DB, folder []byte, fn func(device []byte, f protocol.FileInfoTruncated) bool) {
func ldbWithAllFolderTruncated(db *leveldb.DB, folder []byte, fn func(device []byte, f FileInfoTruncated) bool) {
runtime.GC()
start := deviceKey(folder, nil, nil) // before all folder/device files
@ -583,7 +581,7 @@ func ldbWithAllFolderTruncated(db *leveldb.DB, folder []byte, fn func(device []b
for dbi.Next() {
device := deviceKeyDevice(dbi.Key())
var f protocol.FileInfoTruncated
var f FileInfoTruncated
err := f.UnmarshalXDR(dbi.Value())
if err != nil {
panic(err)
@ -612,7 +610,7 @@ func ldbGet(db *leveldb.DB, folder, device, file []byte) (protocol.FileInfo, boo
return f, true
}
func ldbGetGlobal(db *leveldb.DB, folder, file []byte) (protocol.FileInfo, bool) {
func ldbGetGlobal(db *leveldb.DB, folder, file []byte, truncate bool) (FileIntf, bool) {
k := globalKey(folder, file)
snap, err := db.GetSnapshot()
if err != nil {
@ -633,7 +631,7 @@ func ldbGetGlobal(db *leveldb.DB, folder, file []byte) (protocol.FileInfo, bool)
}
bs, err := snap.Get(k, nil)
if err == leveldb.ErrNotFound {
return protocol.FileInfo{}, false
return nil, false
}
if err != nil {
panic(err)
@ -658,15 +656,14 @@ func ldbGetGlobal(db *leveldb.DB, folder, file []byte) (protocol.FileInfo, bool)
panic(err)
}
var f protocol.FileInfo
err = f.UnmarshalXDR(bs)
fi, err := unmarshalTrunc(bs, truncate)
if err != nil {
panic(err)
}
return f, true
return fi, true
}
func ldbWithGlobal(db *leveldb.DB, folder []byte, truncate bool, fn fileIterator) {
func ldbWithGlobal(db *leveldb.DB, folder []byte, truncate bool, fn Iterator) {
runtime.GC()
start := globalKey(folder, nil)
@ -754,7 +751,7 @@ func ldbAvailability(db *leveldb.DB, folder, file []byte) []protocol.DeviceID {
return devices
}
func ldbWithNeed(db *leveldb.DB, folder, device []byte, truncate bool, fn fileIterator) {
func ldbWithNeed(db *leveldb.DB, folder, device []byte, truncate bool, fn Iterator) {
runtime.GC()
start := globalKey(folder, nil)
@ -938,9 +935,9 @@ func ldbDropFolder(db *leveldb.DB, folder []byte) {
dbi.Release()
}
func unmarshalTrunc(bs []byte, truncate bool) (protocol.FileIntf, error) {
func unmarshalTrunc(bs []byte, truncate bool) (FileIntf, error) {
if truncate {
var tf protocol.FileInfoTruncated
var tf FileInfoTruncated
err := tf.UnmarshalXDR(bs)
return tf, err
} else {

62
internal/files/set.go
View File

@ -30,14 +30,6 @@ import (
"github.com/syndtr/goleveldb/leveldb"
)
type fileRecord struct {
File protocol.FileInfo
Usage int
Global bool
}
type bitset uint64
type Set struct {
localVersion map[protocol.DeviceID]uint64
mutex sync.Mutex
@ -46,6 +38,22 @@ type Set struct {
blockmap *BlockMap
}
// FileIntf is the set of methods implemented by both protocol.FileInfo and
// protocol.FileInfoTruncated.
type FileIntf interface {
Size() int64
IsDeleted() bool
IsInvalid() bool
IsDirectory() bool
IsSymlink() bool
HasPermissionBits() bool
}
// The Iterator is called with either a protocol.FileInfo or a
// protocol.FileInfoTruncated (depending on the method) and returns true to
// continue iteration, false to stop.
type Iterator func(f FileIntf) bool
func NewSet(folder string, db *leveldb.DB) *Set {
var s = Set{
localVersion: make(map[protocol.DeviceID]uint64),
@ -57,7 +65,7 @@ func NewSet(folder string, db *leveldb.DB) *Set {
ldbCheckGlobals(db, []byte(folder))
var deviceID protocol.DeviceID
ldbWithAllFolderTruncated(db, []byte(folder), func(device []byte, f protocol.FileInfoTruncated) bool {
ldbWithAllFolderTruncated(db, []byte(folder), func(device []byte, f FileInfoTruncated) bool {
copy(deviceID[:], device)
if f.LocalVersion > s.localVersion[deviceID] {
s.localVersion[deviceID] = f.LocalVersion
@ -132,42 +140,42 @@ func (s *Set) Update(device protocol.DeviceID, fs []protocol.FileInfo) {
}
}
func (s *Set) WithNeed(device protocol.DeviceID, fn fileIterator) {
func (s *Set) WithNeed(device protocol.DeviceID, fn Iterator) {
if debug {
l.Debugf("%s WithNeed(%v)", s.folder, device)
}
ldbWithNeed(s.db, []byte(s.folder), device[:], false, nativeFileIterator(fn))
}
func (s *Set) WithNeedTruncated(device protocol.DeviceID, fn fileIterator) {
func (s *Set) WithNeedTruncated(device protocol.DeviceID, fn Iterator) {
if debug {
l.Debugf("%s WithNeedTruncated(%v)", s.folder, device)
}
ldbWithNeed(s.db, []byte(s.folder), device[:], true, nativeFileIterator(fn))
}
func (s *Set) WithHave(device protocol.DeviceID, fn fileIterator) {
func (s *Set) WithHave(device protocol.DeviceID, fn Iterator) {
if debug {
l.Debugf("%s WithHave(%v)", s.folder, device)
}
ldbWithHave(s.db, []byte(s.folder), device[:], false, nativeFileIterator(fn))
}
func (s *Set) WithHaveTruncated(device protocol.DeviceID, fn fileIterator) {
func (s *Set) WithHaveTruncated(device protocol.DeviceID, fn Iterator) {
if debug {
l.Debugf("%s WithHaveTruncated(%v)", s.folder, device)
}
ldbWithHave(s.db, []byte(s.folder), device[:], true, nativeFileIterator(fn))
}
func (s *Set) WithGlobal(fn fileIterator) {
func (s *Set) WithGlobal(fn Iterator) {
if debug {
l.Debugf("%s WithGlobal()", s.folder)
}
ldbWithGlobal(s.db, []byte(s.folder), false, nativeFileIterator(fn))
}
func (s *Set) WithGlobalTruncated(fn fileIterator) {
func (s *Set) WithGlobalTruncated(fn Iterator) {
if debug {
l.Debugf("%s WithGlobalTruncated()", s.folder)
}
@ -181,9 +189,23 @@ func (s *Set) Get(device protocol.DeviceID, file string) (protocol.FileInfo, boo
}
func (s *Set) GetGlobal(file string) (protocol.FileInfo, bool) {
f, ok := ldbGetGlobal(s.db, []byte(s.folder), []byte(osutil.NormalizedFilename(file)))
fi, ok := ldbGetGlobal(s.db, []byte(s.folder), []byte(osutil.NormalizedFilename(file)), false)
if !ok {
return protocol.FileInfo{}, false
}
f := fi.(protocol.FileInfo)
f.Name = osutil.NativeFilename(f.Name)
return f, ok
return f, true
}
func (s *Set) GetGlobalTruncated(file string) (FileInfoTruncated, bool) {
fi, ok := ldbGetGlobal(s.db, []byte(s.folder), []byte(osutil.NormalizedFilename(file)), true)
if !ok {
return FileInfoTruncated{}, false
}
f := fi.(FileInfoTruncated)
f.Name = osutil.NativeFilename(f.Name)
return f, true
}
func (s *Set) Availability(file string) []protocol.DeviceID {
@ -218,13 +240,13 @@ func normalizeFilenames(fs []protocol.FileInfo) {
}
}
func nativeFileIterator(fn fileIterator) fileIterator {
return func(fi protocol.FileIntf) bool {
func nativeFileIterator(fn Iterator) Iterator {
return func(fi FileIntf) bool {
switch f := fi.(type) {
case protocol.FileInfo:
f.Name = osutil.NativeFilename(f.Name)
return fn(f)
case protocol.FileInfoTruncated:
case FileInfoTruncated:
f.Name = osutil.NativeFilename(f.Name)
return fn(f)
default:

6
internal/files/set_test.go
View File

@ -51,7 +51,7 @@ func genBlocks(n int) []protocol.BlockInfo {
func globalList(s *files.Set) []protocol.FileInfo {
var fs []protocol.FileInfo
s.WithGlobal(func(fi protocol.FileIntf) bool {
s.WithGlobal(func(fi files.FileIntf) bool {
f := fi.(protocol.FileInfo)
fs = append(fs, f)
return true
@ -61,7 +61,7 @@ func globalList(s *files.Set) []protocol.FileInfo {
func haveList(s *files.Set, n protocol.DeviceID) []protocol.FileInfo {
var fs []protocol.FileInfo
s.WithHave(n, func(fi protocol.FileIntf) bool {
s.WithHave(n, func(fi files.FileIntf) bool {
f := fi.(protocol.FileInfo)
fs = append(fs, f)
return true
@ -71,7 +71,7 @@ func haveList(s *files.Set, n protocol.DeviceID) []protocol.FileInfo {
func needList(s *files.Set, n protocol.DeviceID) []protocol.FileInfo {
var fs []protocol.FileInfo
s.WithNeed(n, func(fi protocol.FileIntf) bool {
s.WithNeed(n, func(fi files.FileIntf) bool {
f := fi.(protocol.FileInfo)
fs = append(fs, f)
return true

75
internal/files/truncated.go Normal file
View File

@ -0,0 +1,75 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This program is free software: you can redistribute it and/or modify it
// under the terms of the GNU General Public License as published by the Free
// Software Foundation, either version 3 of the License, or (at your option)
// any later version.
//
// This program is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
// more details.
//
// You should have received a copy of the GNU General Public License along
// with this program. If not, see <http://www.gnu.org/licenses/>.
//go:generate -command genxdr go run ../../Godeps/_workspace/src/github.com/calmh/xdr/cmd/genxdr/main.go
//go:generate genxdr -o truncated_xdr.go truncated.go
package files
import (
"fmt"
"github.com/syncthing/syncthing/internal/protocol"
)
// Used for unmarshalling a FileInfo structure but skipping the block list.
type FileInfoTruncated struct {
Name string // max:8192
Flags uint32
Modified int64
Version uint64
LocalVersion uint64
NumBlocks uint32
}
func (f FileInfoTruncated) String() string {
return fmt.Sprintf("File{Name:%q, Flags:0%o, Modified:%d, Version:%d, Size:%d, NumBlocks:%d}",
f.Name, f.Flags, f.Modified, f.Version, f.Size(), f.NumBlocks)
}
// Returns a statistical guess on the size, not the exact figure
func (f FileInfoTruncated) Size() int64 {
if f.IsDeleted() || f.IsDirectory() {
return 128
}
return BlocksToSize(f.NumBlocks)
}
func (f FileInfoTruncated) IsDeleted() bool {
return f.Flags&protocol.FlagDeleted != 0
}
func (f FileInfoTruncated) IsInvalid() bool {
return f.Flags&protocol.FlagInvalid != 0
}
func (f FileInfoTruncated) IsDirectory() bool {
return f.Flags&protocol.FlagDirectory != 0
}
func (f FileInfoTruncated) IsSymlink() bool {
return f.Flags&protocol.FlagSymlink != 0
}
func (f FileInfoTruncated) HasPermissionBits() bool {
return f.Flags&protocol.FlagNoPermBits == 0
}
func BlocksToSize(num uint32) int64 {
if num < 2 {
return protocol.BlockSize / 2
}
return int64(num-1)*protocol.BlockSize + protocol.BlockSize/2
}

112
internal/files/truncated_xdr.go Normal file
View File

@ -0,0 +1,112 @@
// ************************************************************
// This file is automatically generated by genxdr. Do not edit.
// ************************************************************
package files
import (
"bytes"
"io"
"github.com/calmh/xdr"
)
/*
FileInfoTruncated Structure:
0 1 2 3
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Length of Name |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
/ /
\ Name (variable length) \
/ /
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Flags |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| |
+ Modified (64 bits) +
| |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| |
+ Version (64 bits) +
| |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| |
+ Local Version (64 bits) +
| |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Num Blocks |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
struct FileInfoTruncated {
string Name<8192>;
unsigned int Flags;
hyper Modified;
unsigned hyper Version;
unsigned hyper LocalVersion;
unsigned int NumBlocks;
}
*/
func (o FileInfoTruncated) EncodeXDR(w io.Writer) (int, error) {
var xw = xdr.NewWriter(w)
return o.encodeXDR(xw)
}
func (o FileInfoTruncated) MarshalXDR() ([]byte, error) {
return o.AppendXDR(make([]byte, 0, 128))
}
func (o FileInfoTruncated) MustMarshalXDR() []byte {
bs, err := o.MarshalXDR()
if err != nil {
panic(err)
}
return bs
}
func (o FileInfoTruncated) AppendXDR(bs []byte) ([]byte, error) {
var aw = xdr.AppendWriter(bs)
var xw = xdr.NewWriter(&aw)
_, err := o.encodeXDR(xw)
return []byte(aw), err
}
func (o FileInfoTruncated) encodeXDR(xw *xdr.Writer) (int, error) {
if l := len(o.Name); l > 8192 {
return xw.Tot(), xdr.ElementSizeExceeded("Name", l, 8192)
}
xw.WriteString(o.Name)
xw.WriteUint32(o.Flags)
xw.WriteUint64(uint64(o.Modified))
xw.WriteUint64(o.Version)
xw.WriteUint64(o.LocalVersion)
xw.WriteUint32(o.NumBlocks)
return xw.Tot(), xw.Error()
}
func (o *FileInfoTruncated) DecodeXDR(r io.Reader) error {
xr := xdr.NewReader(r)
return o.decodeXDR(xr)
}
func (o *FileInfoTruncated) UnmarshalXDR(bs []byte) error {
var br = bytes.NewReader(bs)
var xr = xdr.NewReader(br)
return o.decodeXDR(xr)
}
func (o *FileInfoTruncated) decodeXDR(xr *xdr.Reader) error {
o.Name = xr.ReadStringMax(8192)
o.Flags = xr.ReadUint32()
o.Modified = int64(xr.ReadUint64())
o.Version = xr.ReadUint64()
o.LocalVersion = xr.ReadUint64()
o.NumBlocks = xr.ReadUint32()
return xr.Error()
}

54
internal/model/model.go
View File

@ -310,7 +310,7 @@ func (m *Model) Completion(device protocol.DeviceID, folder string) float64 {
return 0 // Folder doesn't exist, so we hardly have any of it
}
rf.WithGlobalTruncated(func(f protocol.FileIntf) bool {
rf.WithGlobalTruncated(func(f files.FileIntf) bool {
if !f.IsDeleted() {
tot += f.Size()
}
@ -322,7 +322,7 @@ func (m *Model) Completion(device protocol.DeviceID, folder string) float64 {
}
var need int64
rf.WithNeedTruncated(device, func(f protocol.FileIntf) bool {
rf.WithNeedTruncated(device, func(f files.FileIntf) bool {
if !f.IsDeleted() {
need += f.Size()
}
@ -347,7 +347,7 @@ func sizeOf(fs []protocol.FileInfo) (files, deleted int, bytes int64) {
return
}
func sizeOfFile(f protocol.FileIntf) (files, deleted int, bytes int64) {
func sizeOfFile(f files.FileIntf) (files, deleted int, bytes int64) {
if !f.IsDeleted() {
files++
} else {
@ -359,15 +359,15 @@ func sizeOfFile(f protocol.FileIntf) (files, deleted int, bytes int64) {
// GlobalSize returns the number of files, deleted files and total bytes for all
// files in the global model.
func (m *Model) GlobalSize(folder string) (files, deleted int, bytes int64) {
func (m *Model) GlobalSize(folder string) (nfiles, deleted int, bytes int64) {
defer m.leveldbPanicWorkaround()
m.fmut.RLock()
defer m.fmut.RUnlock()
if rf, ok := m.folderFiles[folder]; ok {
rf.WithGlobalTruncated(func(f protocol.FileIntf) bool {
rf.WithGlobalTruncated(func(f files.FileIntf) bool {
fs, de, by := sizeOfFile(f)
files += fs
nfiles += fs
deleted += de
bytes += by
return true
@ -378,18 +378,18 @@ func (m *Model) GlobalSize(folder string) (files, deleted int, bytes int64) {
// LocalSize returns the number of files, deleted files and total bytes for all
// files in the local folder.
func (m *Model) LocalSize(folder string) (files, deleted int, bytes int64) {
func (m *Model) LocalSize(folder string) (nfiles, deleted int, bytes int64) {
defer m.leveldbPanicWorkaround()
m.fmut.RLock()
defer m.fmut.RUnlock()
if rf, ok := m.folderFiles[folder]; ok {
rf.WithHaveTruncated(protocol.LocalDeviceID, func(f protocol.FileIntf) bool {
rf.WithHaveTruncated(protocol.LocalDeviceID, func(f files.FileIntf) bool {
if f.IsInvalid() {
return true
}
fs, de, by := sizeOfFile(f)
files += fs
nfiles += fs
deleted += de
bytes += by
return true
@ -399,22 +399,22 @@ func (m *Model) LocalSize(folder string) (files, deleted int, bytes int64) {
}
// NeedSize returns the number and total size of currently needed files.
func (m *Model) NeedSize(folder string) (files int, bytes int64) {
func (m *Model) NeedSize(folder string) (nfiles int, bytes int64) {
defer m.leveldbPanicWorkaround()
m.fmut.RLock()
defer m.fmut.RUnlock()
if rf, ok := m.folderFiles[folder]; ok {
rf.WithNeedTruncated(protocol.LocalDeviceID, func(f protocol.FileIntf) bool {
rf.WithNeedTruncated(protocol.LocalDeviceID, func(f files.FileIntf) bool {
fs, de, by := sizeOfFile(f)
files += fs + de
nfiles += fs + de
bytes += by
return true
})
}
bytes -= m.progressEmitter.BytesCompleted(folder)
if debug {
l.Debugf("%v NeedSize(%q): %d %d", m, folder, files, bytes)
l.Debugf("%v NeedSize(%q): %d %d", m, folder, nfiles, bytes)
}
return
}
@ -422,42 +422,42 @@ func (m *Model) NeedSize(folder string) (files int, bytes int64) {
// NeedFiles returns the list of currently needed files in progress, queued,
// and to be queued on next puller iteration. Also takes a soft cap which is
// only respected when adding files from the model rather than the runner queue.
func (m *Model) NeedFolderFiles(folder string, max int) ([]protocol.FileInfoTruncated, []protocol.FileInfoTruncated, []protocol.FileInfoTruncated) {
func (m *Model) NeedFolderFiles(folder string, max int) ([]files.FileInfoTruncated, []files.FileInfoTruncated, []files.FileInfoTruncated) {
defer m.leveldbPanicWorkaround()
m.fmut.RLock()
defer m.fmut.RUnlock()
if rf, ok := m.folderFiles[folder]; ok {
var progress, queued, rest []protocol.FileInfoTruncated
var progress, queued, rest []files.FileInfoTruncated
var seen map[string]bool
runner, ok := m.folderRunners[folder]
if ok {
progressNames, queuedNames := runner.Jobs()
progress = make([]protocol.FileInfoTruncated, len(progressNames))
queued = make([]protocol.FileInfoTruncated, len(queuedNames))
progress = make([]files.FileInfoTruncated, len(progressNames))
queued = make([]files.FileInfoTruncated, len(queuedNames))
seen = make(map[string]bool, len(progressNames)+len(queuedNames))
for i, name := range progressNames {
if f, ok := rf.GetGlobal(name); ok {
progress[i] = f.ToTruncated() /// XXX: Should implement GetGlobalTruncated directly
if f, ok := rf.GetGlobalTruncated(name); ok {
progress[i] = f
seen[name] = true
}
}
for i, name := range queuedNames {
if f, ok := rf.GetGlobal(name); ok {
queued[i] = f.ToTruncated() /// XXX: Should implement GetGlobalTruncated directly
if f, ok := rf.GetGlobalTruncated(name); ok {
queued[i] = f
seen[name] = true
}
}
}
left := max - len(progress) - len(queued)
if max < 1 || left > 0 {
rf.WithNeedTruncated(protocol.LocalDeviceID, func(f protocol.FileIntf) bool {
rf.WithNeedTruncated(protocol.LocalDeviceID, func(f files.FileIntf) bool {
left--
ft := f.(protocol.FileInfoTruncated)
ft := f.(files.FileInfoTruncated)
if !seen[ft.Name] {
rest = append(rest, ft)
}
@ -970,7 +970,7 @@ func sendIndexTo(initial bool, minLocalVer uint64, conn protocol.Connection, fol
maxLocalVer := uint64(0)
var err error
fs.WithHave(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool {
fs.WithHave(protocol.LocalDeviceID, func(fi files.FileIntf) bool {
f := fi.(protocol.FileInfo)
if f.LocalVersion <= minLocalVer {
return true
@ -1169,8 +1169,8 @@ func (m *Model) ScanFolderSub(folder, sub string) error {
batch = batch[:0]
// TODO: We should limit the Have scanning to start at sub
seenPrefix := false
fs.WithHaveTruncated(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool {
f := fi.(protocol.FileInfoTruncated)
fs.WithHaveTruncated(protocol.LocalDeviceID, func(fi files.FileIntf) bool {
f := fi.(files.FileInfoTruncated)
if !strings.HasPrefix(f.Name, sub) {
// Return true so that we keep iterating, until we get to the part
// of the tree we are interested in. Then return false so we stop
@ -1310,7 +1310,7 @@ func (m *Model) Override(folder string) {
m.setState(folder, FolderScanning)
batch := make([]protocol.FileInfo, 0, indexBatchSize)
fs.WithNeed(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool {
fs.WithNeed(protocol.LocalDeviceID, func(fi files.FileIntf) bool {
need := fi.(protocol.FileInfo)
if len(batch) == indexBatchSize {
fs.Update(protocol.LocalDeviceID, batch)

5
internal/model/puller.go
View File

@ -29,6 +29,7 @@ import (
"github.com/syncthing/syncthing/internal/config"
"github.com/syncthing/syncthing/internal/events"
"github.com/syncthing/syncthing/internal/files"
"github.com/syncthing/syncthing/internal/ignore"
"github.com/syncthing/syncthing/internal/osutil"
"github.com/syncthing/syncthing/internal/protocol"
@ -288,7 +289,7 @@ func (p *Puller) pullerIteration(ignores *ignore.Matcher) int {
}
p.model.fmut.RLock()
files := p.model.folderFiles[p.folder]
folderFiles := p.model.folderFiles[p.folder]
p.model.fmut.RUnlock()
// !!!
@ -301,7 +302,7 @@ func (p *Puller) pullerIteration(ignores *ignore.Matcher) int {
var deletions []protocol.FileInfo
files.WithNeed(protocol.LocalDeviceID, func(intf protocol.FileIntf) bool {
folderFiles.WithNeed(protocol.LocalDeviceID, func(intf files.FileIntf) bool {
// Needed items are delivered sorted lexicographically. This isn't
// really optimal from a performance point of view - it would be

5
internal/model/sharedpullerstate.go
View File

@ -21,6 +21,7 @@ import (
"path/filepath"
"sync"
"github.com/syncthing/syncthing/internal/files"
"github.com/syncthing/syncthing/internal/protocol"
)
@ -259,7 +260,7 @@ func (s *sharedPullerState) Progress() *pullerProgress {
CopiedFromElsewhere: s.copyTotal - s.copyNeeded - s.copyOrigin,
Pulled: s.pullTotal - s.pullNeeded,
Pulling: s.pullNeeded,
BytesTotal: protocol.BlocksToSize(total),
BytesDone: protocol.BlocksToSize(done),
BytesTotal: files.BlocksToSize(total),
BytesDone: files.BlocksToSize(done),
}
}

70
internal/protocol/message.go
View File

@ -71,76 +71,6 @@ func (f FileInfo) HasPermissionBits() bool {
return f.Flags&FlagNoPermBits == 0
}
func (f FileInfo) ToTruncated() FileInfoTruncated {
return FileInfoTruncated{
Name: f.Name,
Flags: f.Flags,
Modified: f.Modified,
Version: f.Version,
LocalVersion: f.LocalVersion,
NumBlocks: uint32(len(f.Blocks)),
}
}
// Used for unmarshalling a FileInfo structure but skipping the actual block list
type FileInfoTruncated struct {
Name string // max:8192
Flags uint32
Modified int64
Version uint64
LocalVersion uint64
NumBlocks uint32
}
func (f FileInfoTruncated) String() string {
return fmt.Sprintf("File{Name:%q, Flags:0%o, Modified:%d, Version:%d, Size:%d, NumBlocks:%d}",
f.Name, f.Flags, f.Modified, f.Version, f.Size(), f.NumBlocks)
}
func BlocksToSize(num uint32) int64 {
if num < 2 {
return BlockSize / 2
}
return int64(num-1)*BlockSize + BlockSize/2
}
// Returns a statistical guess on the size, not the exact figure
func (f FileInfoTruncated) Size() int64 {
if f.IsDeleted() || f.IsDirectory() {
return 128
}
return BlocksToSize(f.NumBlocks)
}
func (f FileInfoTruncated) IsDeleted() bool {
return f.Flags&FlagDeleted != 0
}
func (f FileInfoTruncated) IsInvalid() bool {
return f.Flags&FlagInvalid != 0
}
func (f FileInfoTruncated) IsDirectory() bool {
return f.Flags&FlagDirectory != 0
}
func (f FileInfoTruncated) IsSymlink() bool {
return f.Flags&FlagSymlink != 0
}
func (f FileInfoTruncated) HasPermissionBits() bool {
return f.Flags&FlagNoPermBits == 0
}
type FileIntf interface {
Size() int64
IsDeleted() bool
IsInvalid() bool
IsDirectory() bool
IsSymlink() bool
HasPermissionBits() bool
}
type BlockInfo struct {
Offset int64 // noencode (cache only)
Size uint32

100
internal/protocol/message_xdr.go
View File

@ -245,106 +245,6 @@ func (o *FileInfo) decodeXDR(xr *xdr.Reader) error {
/*
FileInfoTruncated Structure:
0 1 2 3
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Length of Name |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
/ /
\ Name (variable length) \
/ /
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Flags |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| |
+ Modified (64 bits) +
| |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| |
+ Version (64 bits) +
| |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| |
+ Local Version (64 bits) +
| |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Num Blocks |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
struct FileInfoTruncated {
string Name<8192>;
unsigned int Flags;
hyper Modified;
unsigned hyper Version;
unsigned hyper LocalVersion;
unsigned int NumBlocks;
}
*/
func (o FileInfoTruncated) EncodeXDR(w io.Writer) (int, error) {
var xw = xdr.NewWriter(w)
return o.encodeXDR(xw)
}
func (o FileInfoTruncated) MarshalXDR() ([]byte, error) {
return o.AppendXDR(make([]byte, 0, 128))
}
func (o FileInfoTruncated) MustMarshalXDR() []byte {
bs, err := o.MarshalXDR()
if err != nil {
panic(err)
}
return bs
}
func (o FileInfoTruncated) AppendXDR(bs []byte) ([]byte, error) {
var aw = xdr.AppendWriter(bs)
var xw = xdr.NewWriter(&aw)
_, err := o.encodeXDR(xw)
return []byte(aw), err
}
func (o FileInfoTruncated) encodeXDR(xw *xdr.Writer) (int, error) {
if l := len(o.Name); l > 8192 {
return xw.Tot(), xdr.ElementSizeExceeded("Name", l, 8192)
}
xw.WriteString(o.Name)
xw.WriteUint32(o.Flags)
xw.WriteUint64(uint64(o.Modified))
xw.WriteUint64(o.Version)
xw.WriteUint64(o.LocalVersion)
xw.WriteUint32(o.NumBlocks)
return xw.Tot(), xw.Error()
}
func (o *FileInfoTruncated) DecodeXDR(r io.Reader) error {
xr := xdr.NewReader(r)
return o.decodeXDR(xr)
}
func (o *FileInfoTruncated) UnmarshalXDR(bs []byte) error {
var br = bytes.NewReader(bs)
var xr = xdr.NewReader(br)
return o.decodeXDR(xr)
}
func (o *FileInfoTruncated) decodeXDR(xr *xdr.Reader) error {
o.Name = xr.ReadStringMax(8192)
o.Flags = xr.ReadUint32()
o.Modified = int64(xr.ReadUint64())
o.Version = xr.ReadUint64()
o.LocalVersion = xr.ReadUint64()
o.NumBlocks = xr.ReadUint32()
return xr.Error()
}
/*
BlockInfo Structure:
0 1 2 3