octoleo/syncthing
1
0
Fork 0
mirror of https://github.com/octoleo/syncthing.git synced 2024-12-31 14:01:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Move FileIntf to files package, expose Iterator type

Browse Source 
This is where FileIntf is used, so it should be defined here (it's not
a protocol thing, really).
This commit is contained in:
Jakob Borg 2015-01-09 08:18:42 +01:00
parent 2d047fa428
commit 8bc7d259f4
7 changed files with 55 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/stindex/main.go
View File

@ -43,7 +43,7 @@ func main() {
if *device == "" {
log.Printf("*** Global index for folder %q", *folder)
fs.WithGlobalTruncated(func(fi protocol.FileIntf) bool {
fs.WithGlobalTruncated(func(fi files.FileIntf) bool {
f := fi.(protocol.FileInfoTruncated)
fmt.Println(f)
fmt.Println("\t", fs.Availability(f.Name))
@ -55,7 +55,7 @@ func main() {
log.Fatal(err)
}
log.Printf("*** Have index for folder %q device %q", *folder, n)
fs.WithHaveTruncated(n, func(fi protocol.FileIntf) bool {
fs.WithHaveTruncated(n, func(fi files.FileIntf) bool {
f := fi.(protocol.FileInfoTruncated)
fmt.Println(f)
return true

10
internal/files/leveldb.go
View File

@ -166,8 +166,6 @@ func globalKeyFolder(key []byte) []byte {
type deletionHandler func(db dbReader, batch dbWriter, folder, device, name []byte, dbi iterator.Iterator) uint64
type fileIterator func(f protocol.FileIntf) bool
func ldbGenericReplace(db *leveldb.DB, folder, device []byte, fs []protocol.FileInfo, deleteFn deletionHandler) uint64 {
runtime.GC()
@ -528,7 +526,7 @@ func ldbRemoveFromGlobal(db dbReader, batch dbWriter, folder, device, file []byt
}
}
func ldbWithHave(db *leveldb.DB, folder, device []byte, truncate bool, fn fileIterator) {
func ldbWithHave(db *leveldb.DB, folder, device []byte, truncate bool, fn Iterator) {
start := deviceKey(folder, device, nil) // before all folder/device files
limit := deviceKey(folder, device, []byte{0xff, 0xff, 0xff, 0xff}) // after all folder/device files
snap, err := db.GetSnapshot()
@ -666,7 +664,7 @@ func ldbGetGlobal(db *leveldb.DB, folder, file []byte) (protocol.FileInfo, bool)
return f, true
}
func ldbWithGlobal(db *leveldb.DB, folder []byte, truncate bool, fn fileIterator) {
func ldbWithGlobal(db *leveldb.DB, folder []byte, truncate bool, fn Iterator) {
runtime.GC()
start := globalKey(folder, nil)
@ -754,7 +752,7 @@ func ldbAvailability(db *leveldb.DB, folder, file []byte) []protocol.DeviceID {
return devices
}
func ldbWithNeed(db *leveldb.DB, folder, device []byte, truncate bool, fn fileIterator) {
func ldbWithNeed(db *leveldb.DB, folder, device []byte, truncate bool, fn Iterator) {
runtime.GC()
start := globalKey(folder, nil)
@ -938,7 +936,7 @@ func ldbDropFolder(db *leveldb.DB, folder []byte) {
dbi.Release()
}
func unmarshalTrunc(bs []byte, truncate bool) (protocol.FileIntf, error) {
func unmarshalTrunc(bs []byte, truncate bool) (FileIntf, error) {
if truncate {
var tf protocol.FileInfoTruncated
err := tf.UnmarshalXDR(bs)

32
internal/files/set.go
View File

@ -38,6 +38,22 @@ type Set struct {
blockmap *BlockMap
}
// FileIntf is the set of methods implemented by both protocol.FileInfo and
// protocol.FileInfoTruncated.
type FileIntf interface {
Size() int64
IsDeleted() bool
IsInvalid() bool
IsDirectory() bool
IsSymlink() bool
HasPermissionBits() bool
}
// The Iterator is called with either a protocol.FileInfo or a
// protocol.FileInfoTruncated (depending on the method) and returns true to
// continue iteration, false to stop.
type Iterator func(f FileIntf) bool
func NewSet(folder string, db *leveldb.DB) *Set {
var s = Set{
localVersion: make(map[protocol.DeviceID]uint64),
@ -124,42 +140,42 @@ func (s *Set) Update(device protocol.DeviceID, fs []protocol.FileInfo) {
}
}
func (s *Set) WithNeed(device protocol.DeviceID, fn fileIterator) {
func (s *Set) WithNeed(device protocol.DeviceID, fn Iterator) {
if debug {
l.Debugf("%s WithNeed(%v)", s.folder, device)
}
ldbWithNeed(s.db, []byte(s.folder), device[:], false, nativeFileIterator(fn))
}
func (s *Set) WithNeedTruncated(device protocol.DeviceID, fn fileIterator) {
func (s *Set) WithNeedTruncated(device protocol.DeviceID, fn Iterator) {
if debug {
l.Debugf("%s WithNeedTruncated(%v)", s.folder, device)
}
ldbWithNeed(s.db, []byte(s.folder), device[:], true, nativeFileIterator(fn))
}
func (s *Set) WithHave(device protocol.DeviceID, fn fileIterator) {
func (s *Set) WithHave(device protocol.DeviceID, fn Iterator) {
if debug {
l.Debugf("%s WithHave(%v)", s.folder, device)
}
ldbWithHave(s.db, []byte(s.folder), device[:], false, nativeFileIterator(fn))
}
func (s *Set) WithHaveTruncated(device protocol.DeviceID, fn fileIterator) {
func (s *Set) WithHaveTruncated(device protocol.DeviceID, fn Iterator) {
if debug {
l.Debugf("%s WithHaveTruncated(%v)", s.folder, device)
}
ldbWithHave(s.db, []byte(s.folder), device[:], true, nativeFileIterator(fn))
}
func (s *Set) WithGlobal(fn fileIterator) {
func (s *Set) WithGlobal(fn Iterator) {
if debug {
l.Debugf("%s WithGlobal()", s.folder)
}
ldbWithGlobal(s.db, []byte(s.folder), false, nativeFileIterator(fn))
}
func (s *Set) WithGlobalTruncated(fn fileIterator) {
func (s *Set) WithGlobalTruncated(fn Iterator) {
if debug {
l.Debugf("%s WithGlobalTruncated()", s.folder)
}
@ -210,8 +226,8 @@ func normalizeFilenames(fs []protocol.FileInfo) {
}
}
func nativeFileIterator(fn fileIterator) fileIterator {
return func(fi protocol.FileIntf) bool {
func nativeFileIterator(fn Iterator) Iterator {
return func(fi FileIntf) bool {
switch f := fi.(type) {
case protocol.FileInfo:
f.Name = osutil.NativeFilename(f.Name)

6
internal/files/set_test.go
View File

@ -51,7 +51,7 @@ func genBlocks(n int) []protocol.BlockInfo {
func globalList(s *files.Set) []protocol.FileInfo {
var fs []protocol.FileInfo
s.WithGlobal(func(fi protocol.FileIntf) bool {
s.WithGlobal(func(fi FileIntf) bool {
f := fi.(protocol.FileInfo)
fs = append(fs, f)
return true
@ -61,7 +61,7 @@ func globalList(s *files.Set) []protocol.FileInfo {
func haveList(s *files.Set, n protocol.DeviceID) []protocol.FileInfo {
var fs []protocol.FileInfo
s.WithHave(n, func(fi protocol.FileIntf) bool {
s.WithHave(n, func(fi FileIntf) bool {
f := fi.(protocol.FileInfo)
fs = append(fs, f)
return true
@ -71,7 +71,7 @@ func haveList(s *files.Set, n protocol.DeviceID) []protocol.FileInfo {
func needList(s *files.Set, n protocol.DeviceID) []protocol.FileInfo {
var fs []protocol.FileInfo
s.WithNeed(n, func(fi protocol.FileIntf) bool {
s.WithNeed(n, func(fi FileIntf) bool {
f := fi.(protocol.FileInfo)
fs = append(fs, f)
return true

38
internal/model/model.go
View File

@ -310,7 +310,7 @@ func (m *Model) Completion(device protocol.DeviceID, folder string) float64 {
return 0 // Folder doesn't exist, so we hardly have any of it
}
rf.WithGlobalTruncated(func(f protocol.FileIntf) bool {
rf.WithGlobalTruncated(func(f files.FileIntf) bool {
if !f.IsDeleted() {
tot += f.Size()
}
@ -322,7 +322,7 @@ func (m *Model) Completion(device protocol.DeviceID, folder string) float64 {
}
var need int64
rf.WithNeedTruncated(device, func(f protocol.FileIntf) bool {
rf.WithNeedTruncated(device, func(f files.FileIntf) bool {
if !f.IsDeleted() {
need += f.Size()
}
@ -347,7 +347,7 @@ func sizeOf(fs []protocol.FileInfo) (files, deleted int, bytes int64) {
return
}
func sizeOfFile(f protocol.FileIntf) (files, deleted int, bytes int64) {
func sizeOfFile(f files.FileIntf) (files, deleted int, bytes int64) {
if !f.IsDeleted() {
files++
} else {
@ -359,15 +359,15 @@ func sizeOfFile(f protocol.FileIntf) (files, deleted int, bytes int64) {
// GlobalSize returns the number of files, deleted files and total bytes for all
// files in the global model.
func (m *Model) GlobalSize(folder string) (files, deleted int, bytes int64) {
func (m *Model) GlobalSize(folder string) (nfiles, deleted int, bytes int64) {
defer m.leveldbPanicWorkaround()
m.fmut.RLock()
defer m.fmut.RUnlock()
if rf, ok := m.folderFiles[folder]; ok {
rf.WithGlobalTruncated(func(f protocol.FileIntf) bool {
rf.WithGlobalTruncated(func(f files.FileIntf) bool {
fs, de, by := sizeOfFile(f)
files += fs
nfiles += fs
deleted += de
bytes += by
return true
@ -378,18 +378,18 @@ func (m *Model) GlobalSize(folder string) (files, deleted int, bytes int64) {
// LocalSize returns the number of files, deleted files and total bytes for all
// files in the local folder.
func (m *Model) LocalSize(folder string) (files, deleted int, bytes int64) {
func (m *Model) LocalSize(folder string) (nfiles, deleted int, bytes int64) {
defer m.leveldbPanicWorkaround()
m.fmut.RLock()
defer m.fmut.RUnlock()
if rf, ok := m.folderFiles[folder]; ok {
rf.WithHaveTruncated(protocol.LocalDeviceID, func(f protocol.FileIntf) bool {
rf.WithHaveTruncated(protocol.LocalDeviceID, func(f files.FileIntf) bool {
if f.IsInvalid() {
return true
}
fs, de, by := sizeOfFile(f)
files += fs
nfiles += fs
deleted += de
bytes += by
return true
@ -399,22 +399,22 @@ func (m *Model) LocalSize(folder string) (files, deleted int, bytes int64) {
}
// NeedSize returns the number and total size of currently needed files.
func (m *Model) NeedSize(folder string) (files int, bytes int64) {
func (m *Model) NeedSize(folder string) (nfiles int, bytes int64) {
defer m.leveldbPanicWorkaround()
m.fmut.RLock()
defer m.fmut.RUnlock()
if rf, ok := m.folderFiles[folder]; ok {
rf.WithNeedTruncated(protocol.LocalDeviceID, func(f protocol.FileIntf) bool {
rf.WithNeedTruncated(protocol.LocalDeviceID, func(f files.FileIntf) bool {
fs, de, by := sizeOfFile(f)
files += fs + de
nfiles += fs + de
bytes += by
return true
})
}
bytes -= m.progressEmitter.BytesCompleted(folder)
if debug {
l.Debugf("%v NeedSize(%q): %d %d", m, folder, files, bytes)
l.Debugf("%v NeedSize(%q): %d %d", m, folder, nfiles, bytes)
}
return
}
@ -441,21 +441,21 @@ func (m *Model) NeedFolderFiles(folder string, max int) ([]protocol.FileInfoTrun
for i, name := range progressNames {
if f, ok := rf.GetGlobal(name); ok {
progress[i] = f.ToTruncated() /// XXX: Should implement GetGlobalTruncated directly
progress[i] = protocol.Truncate(f) /// XXX: Should implement GetGlobalTruncated directly
seen[name] = true
}
}
for i, name := range queuedNames {
if f, ok := rf.GetGlobal(name); ok {
queued[i] = f.ToTruncated() /// XXX: Should implement GetGlobalTruncated directly
queued[i] = protocol.Truncate(f) /// XXX: Should implement GetGlobalTruncated directly
seen[name] = true
}
}
}
left := max - len(progress) - len(queued)
if max < 1 || left > 0 {
rf.WithNeedTruncated(protocol.LocalDeviceID, func(f protocol.FileIntf) bool {
rf.WithNeedTruncated(protocol.LocalDeviceID, func(f files.FileIntf) bool {
left--
ft := f.(protocol.FileInfoTruncated)
if !seen[ft.Name] {
@ -970,7 +970,7 @@ func sendIndexTo(initial bool, minLocalVer uint64, conn protocol.Connection, fol
maxLocalVer := uint64(0)
var err error
fs.WithHave(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool {
fs.WithHave(protocol.LocalDeviceID, func(fi files.FileIntf) bool {
f := fi.(protocol.FileInfo)
if f.LocalVersion <= minLocalVer {
return true
@ -1169,7 +1169,7 @@ func (m *Model) ScanFolderSub(folder, sub string) error {
batch = batch[:0]
// TODO: We should limit the Have scanning to start at sub
seenPrefix := false
fs.WithHaveTruncated(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool {
fs.WithHaveTruncated(protocol.LocalDeviceID, func(fi files.FileIntf) bool {
f := fi.(protocol.FileInfoTruncated)
if !strings.HasPrefix(f.Name, sub) {
// Return true so that we keep iterating, until we get to the part
@ -1310,7 +1310,7 @@ func (m *Model) Override(folder string) {
m.setState(folder, FolderScanning)
batch := make([]protocol.FileInfo, 0, indexBatchSize)
fs.WithNeed(protocol.LocalDeviceID, func(fi protocol.FileIntf) bool {
fs.WithNeed(protocol.LocalDeviceID, func(fi files.FileIntf) bool {
need := fi.(protocol.FileInfo)
if len(batch) == indexBatchSize {
fs.Update(protocol.LocalDeviceID, batch)

5
internal/model/puller.go
View File

@ -29,6 +29,7 @@ import (
"github.com/syncthing/syncthing/internal/config"
"github.com/syncthing/syncthing/internal/events"
"github.com/syncthing/syncthing/internal/files"
"github.com/syncthing/syncthing/internal/ignore"
"github.com/syncthing/syncthing/internal/osutil"
"github.com/syncthing/syncthing/internal/protocol"
@ -288,7 +289,7 @@ func (p *Puller) pullerIteration(ignores *ignore.Matcher) int {
}
p.model.fmut.RLock()
files := p.model.folderFiles[p.folder]
folderFiles := p.model.folderFiles[p.folder]
p.model.fmut.RUnlock()
// !!!
@ -301,7 +302,7 @@ func (p *Puller) pullerIteration(ignores *ignore.Matcher) int {
var deletions []protocol.FileInfo
files.WithNeed(protocol.LocalDeviceID, func(intf protocol.FileIntf) bool {
folderFiles.WithNeed(protocol.LocalDeviceID, func(intf files.FileIntf) bool {
// Needed items are delivered sorted lexicographically. This isn't
// really optimal from a performance point of view - it would be

9
internal/protocol/message.go
View File

@ -132,15 +132,6 @@ func (f FileInfoTruncated) HasPermissionBits() bool {
return f.Flags&FlagNoPermBits == 0
}
type FileIntf interface {
Size() int64
IsDeleted() bool
IsInvalid() bool
IsDirectory() bool
IsSymlink() bool
HasPermissionBits() bool
}
type BlockInfo struct {
Offset int64 // noencode (cache only)
Size uint32