octoleo/syncthing
1
0
Fork 0
mirror of https://github.com/octoleo/syncthing.git synced 2024-12-22 19:08:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

Deprecate scanner.Block & File

Browse Source
This commit is contained in:
Jakob Borg 2014-07-12 23:06:48 +02:00
parent 91b35118d9
commit 655acb4cb2
21 changed files with 269 additions and 480 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
build.sh
View File

@ -96,7 +96,7 @@ setup() {
} }
xdr() { xdr() {
for f in discover/packets files/leveldb protocol/message scanner/blocks scanner/file ; do for f in discover/packets files/leveldb protocol/message ; do
go run xdr/cmd/genxdr/main.go -- "${f}.go" > "${f}_xdr.go" go run xdr/cmd/genxdr/main.go -- "${f}.go" > "${f}_xdr.go"
done done
} }

5
config/config.go
View File

@ -18,7 +18,6 @@ import (
"code.google.com/p/go.crypto/bcrypt" "code.google.com/p/go.crypto/bcrypt"
"github.com/calmh/syncthing/logger" "github.com/calmh/syncthing/logger"
"github.com/calmh/syncthing/protocol" "github.com/calmh/syncthing/protocol"
"github.com/calmh/syncthing/scanner"
) )
var l = logger.DefaultLogger var l = logger.DefaultLogger
@ -122,11 +121,11 @@ func (r *RepositoryConfiguration) NodeIDs() []protocol.NodeID {
return r.nodeIDs return r.nodeIDs
} }
func (r RepositoryConfiguration) FileRanker() func(scanner.File) int { func (r RepositoryConfiguration) FileRanker() func(protocol.FileInfo) int {
if len(r.SyncOrderPatterns) <= 0 { if len(r.SyncOrderPatterns) <= 0 {
return nil return nil
} }
return func(f scanner.File) int { return func(f protocol.FileInfo) int {
ret := 0 ret := 0
for _, v := range r.SyncOrderPatterns { for _, v := range r.SyncOrderPatterns {
if v.CompiledPattern().MatchString(f.Name) { if v.CompiledPattern().MatchString(f.Name) {

7
config/config_test.go
View File

@ -13,7 +13,6 @@ import (
"github.com/calmh/syncthing/files" "github.com/calmh/syncthing/files"
"github.com/calmh/syncthing/protocol" "github.com/calmh/syncthing/protocol"
"github.com/calmh/syncthing/scanner"
) )
var node1, node2, node3, node4 protocol.NodeID var node1, node2, node3, node4 protocol.NodeID
@ -274,7 +273,7 @@ func TestFileSorter(t *testing.T) {
}, },
} }
f := []scanner.File{ f := []protocol.FileInfo{
{Name: "bar.mov"}, {Name: "bar.mov"},
{Name: "baz.txt"}, {Name: "baz.txt"},
{Name: "foo.jpg"}, {Name: "foo.jpg"},
@ -290,7 +289,7 @@ func TestFileSorter(t *testing.T) {
files.SortBy(rcfg.FileRanker()).Sort(f) files.SortBy(rcfg.FileRanker()).Sort(f)
expected := []scanner.File{ expected := []protocol.FileInfo{
{Name: "camera-uploads/foo.jpg"}, {Name: "camera-uploads/foo.jpg"},
{Name: "camera-uploads/herp.mov"}, {Name: "camera-uploads/herp.mov"},
{Name: "camera-uploads/hurr.pl"}, {Name: "camera-uploads/hurr.pl"},
@ -314,7 +313,7 @@ func TestFileSorter(t *testing.T) {
} }
} }
func formatFiles(f []scanner.File) string { func formatFiles(f []protocol.FileInfo) string {
ret := "" ret := ""
for _, v := range f { for _, v := range f {

5
files/cmd/pidx/main.go
View File

@ -8,7 +8,6 @@ import (
"github.com/calmh/syncthing/files" "github.com/calmh/syncthing/files"
"github.com/calmh/syncthing/protocol" "github.com/calmh/syncthing/protocol"
"github.com/calmh/syncthing/scanner"
"github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb"
) )
@ -29,7 +28,7 @@ func main() {
if *node == "" { if *node == "" {
log.Printf("*** Global index for repo %q", *repo) log.Printf("*** Global index for repo %q", *repo)
fs.WithGlobal(func(f scanner.File) bool { fs.WithGlobal(func(f protocol.FileInfo) bool {
fmt.Println(f) fmt.Println(f)
fmt.Println("\t", fs.Availability(f.Name)) fmt.Println("\t", fs.Availability(f.Name))
return true return true
@ -40,7 +39,7 @@ func main() {
log.Fatal(err) log.Fatal(err)
} }
log.Printf("*** Have index for repo %q node %q", *repo, n) log.Printf("*** Have index for repo %q node %q", *repo, n)
fs.WithHave(n, func(f scanner.File) bool { fs.WithHave(n, func(f protocol.FileInfo) bool {
fmt.Println(f) fmt.Println(f)
return true return true
}) })

39
files/leveldb.go
View File

@ -6,7 +6,6 @@ import (
"github.com/calmh/syncthing/lamport" "github.com/calmh/syncthing/lamport"
"github.com/calmh/syncthing/protocol" "github.com/calmh/syncthing/protocol"
"github.com/calmh/syncthing/scanner"
"github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/iterator" "github.com/syndtr/goleveldb/leveldb/iterator"
"github.com/syndtr/goleveldb/leveldb/opt" "github.com/syndtr/goleveldb/leveldb/opt"
@ -27,7 +26,7 @@ type versionList struct {
versions []fileVersion versions []fileVersion
} }
type fileList []scanner.File type fileList []protocol.FileInfo
func (l fileList) Len() int { func (l fileList) Len() int {
return len(l) return len(l)
@ -94,9 +93,9 @@ func globalKeyName(key []byte) []byte {
type deletionHandler func(db dbReader, batch dbWriter, repo, node, name []byte, dbi iterator.Iterator) bool type deletionHandler func(db dbReader, batch dbWriter, repo, node, name []byte, dbi iterator.Iterator) bool
type fileIterator func(f scanner.File) bool type fileIterator func(f protocol.FileInfo) bool
func ldbGenericReplace(db *leveldb.DB, repo, node []byte, fs []scanner.File, deleteFn deletionHandler) bool { func ldbGenericReplace(db *leveldb.DB, repo, node []byte, fs []protocol.FileInfo, deleteFn deletionHandler) bool {
sort.Sort(fileList(fs)) // sort list on name, same as on disk sort.Sort(fileList(fs)) // sort list on name, same as on disk
start := nodeKey(repo, node, nil) // before all repo/node files start := nodeKey(repo, node, nil) // before all repo/node files
@ -153,7 +152,7 @@ func ldbGenericReplace(db *leveldb.DB, repo, node []byte, fs []scanner.File, del
case cmp == 0: case cmp == 0:
// File exists on both sides - compare versions. // File exists on both sides - compare versions.
var ef scanner.File var ef protocol.FileInfo
ef.UnmarshalXDR(dbi.Value()) ef.UnmarshalXDR(dbi.Value())
if fs[fsi].Version > ef.Version { if fs[fsi].Version > ef.Version {
ldbInsert(batch, repo, node, newName, fs[fsi]) ldbInsert(batch, repo, node, newName, fs[fsi])
@ -182,7 +181,7 @@ func ldbGenericReplace(db *leveldb.DB, repo, node []byte, fs []scanner.File, del
return changed return changed
} }
func ldbReplace(db *leveldb.DB, repo, node []byte, fs []scanner.File) bool { func ldbReplace(db *leveldb.DB, repo, node []byte, fs []protocol.FileInfo) bool {
return ldbGenericReplace(db, repo, node, fs, func(db dbReader, batch dbWriter, repo, node, name []byte, dbi iterator.Iterator) bool { return ldbGenericReplace(db, repo, node, fs, func(db dbReader, batch dbWriter, repo, node, name []byte, dbi iterator.Iterator) bool {
// Disk has files that we are missing. Remove it. // Disk has files that we are missing. Remove it.
if debug { if debug {
@ -194,9 +193,9 @@ func ldbReplace(db *leveldb.DB, repo, node []byte, fs []scanner.File) bool {
}) })
} }
func ldbReplaceWithDelete(db *leveldb.DB, repo, node []byte, fs []scanner.File) bool { func ldbReplaceWithDelete(db *leveldb.DB, repo, node []byte, fs []protocol.FileInfo) bool {
return ldbGenericReplace(db, repo, node, fs, func(db dbReader, batch dbWriter, repo, node, name []byte, dbi iterator.Iterator) bool { return ldbGenericReplace(db, repo, node, fs, func(db dbReader, batch dbWriter, repo, node, name []byte, dbi iterator.Iterator) bool {
var f scanner.File var f protocol.FileInfo
err := f.UnmarshalXDR(dbi.Value()) err := f.UnmarshalXDR(dbi.Value())
if err != nil { if err != nil {
panic(err) panic(err)
@ -216,7 +215,7 @@ func ldbReplaceWithDelete(db *leveldb.DB, repo, node []byte, fs []scanner.File)
}) })
} }
func ldbUpdate(db *leveldb.DB, repo, node []byte, fs []scanner.File) bool { func ldbUpdate(db *leveldb.DB, repo, node []byte, fs []protocol.FileInfo) bool {
batch := new(leveldb.Batch) batch := new(leveldb.Batch)
snap, err := db.GetSnapshot() snap, err := db.GetSnapshot()
if err != nil { if err != nil {
@ -234,7 +233,7 @@ func ldbUpdate(db *leveldb.DB, repo, node []byte, fs []scanner.File) bool {
continue continue
} }
var ef scanner.File var ef protocol.FileInfo
err = ef.UnmarshalXDR(bs) err = ef.UnmarshalXDR(bs)
if err != nil { if err != nil {
panic(err) panic(err)
@ -253,7 +252,7 @@ func ldbUpdate(db *leveldb.DB, repo, node []byte, fs []scanner.File) bool {
return true return true
} }
func ldbInsert(batch dbWriter, repo, node, name []byte, file scanner.File) { func ldbInsert(batch dbWriter, repo, node, name []byte, file protocol.FileInfo) {
if debug { if debug {
l.Debugf("insert; repo=%q node=%x %v", repo, node, file) l.Debugf("insert; repo=%q node=%x %v", repo, node, file)
} }
@ -362,7 +361,7 @@ func ldbWithHave(db *leveldb.DB, repo, node []byte, fn fileIterator) {
defer dbi.Release() defer dbi.Release()
for dbi.Next() { for dbi.Next() {
var f scanner.File var f protocol.FileInfo
err := f.UnmarshalXDR(dbi.Value()) err := f.UnmarshalXDR(dbi.Value())
if err != nil { if err != nil {
panic(err) panic(err)
@ -373,17 +372,17 @@ func ldbWithHave(db *leveldb.DB, repo, node []byte, fn fileIterator) {
} }
} }
func ldbGet(db *leveldb.DB, repo, node, file []byte) scanner.File { func ldbGet(db *leveldb.DB, repo, node, file []byte) protocol.FileInfo {
nk := nodeKey(repo, node, file) nk := nodeKey(repo, node, file)
bs, err := db.Get(nk, nil) bs, err := db.Get(nk, nil)
if err == leveldb.ErrNotFound { if err == leveldb.ErrNotFound {
return scanner.File{} return protocol.FileInfo{}
} }
if err != nil { if err != nil {
panic(err) panic(err)
} }
var f scanner.File var f protocol.FileInfo
err = f.UnmarshalXDR(bs) err = f.UnmarshalXDR(bs)
if err != nil { if err != nil {
panic(err) panic(err)
@ -391,7 +390,7 @@ func ldbGet(db *leveldb.DB, repo, node, file []byte) scanner.File {
return f return f
} }
func ldbGetGlobal(db *leveldb.DB, repo, file []byte) scanner.File { func ldbGetGlobal(db *leveldb.DB, repo, file []byte) protocol.FileInfo {
k := globalKey(repo, file) k := globalKey(repo, file)
snap, err := db.GetSnapshot() snap, err := db.GetSnapshot()
if err != nil { if err != nil {
@ -401,7 +400,7 @@ func ldbGetGlobal(db *leveldb.DB, repo, file []byte) scanner.File {
bs, err := snap.Get(k, nil) bs, err := snap.Get(k, nil)
if err == leveldb.ErrNotFound { if err == leveldb.ErrNotFound {
return scanner.File{} return protocol.FileInfo{}
} }
if err != nil { if err != nil {
panic(err) panic(err)
@ -423,7 +422,7 @@ func ldbGetGlobal(db *leveldb.DB, repo, file []byte) scanner.File {
panic(err) panic(err)
} }
var f scanner.File var f protocol.FileInfo
err = f.UnmarshalXDR(bs) err = f.UnmarshalXDR(bs)
if err != nil { if err != nil {
panic(err) panic(err)
@ -458,7 +457,7 @@ func ldbWithGlobal(db *leveldb.DB, repo []byte, fn fileIterator) {
panic(err) panic(err)
} }
var f scanner.File var f protocol.FileInfo
err = f.UnmarshalXDR(bs) err = f.UnmarshalXDR(bs)
if err != nil { if err != nil {
panic(err) panic(err)
@ -544,7 +543,7 @@ func ldbWithNeed(db *leveldb.DB, repo, node []byte, fn fileIterator) {
panic(err) panic(err)
} }
var gf scanner.File var gf protocol.FileInfo
err = gf.UnmarshalXDR(bs) err = gf.UnmarshalXDR(bs)
if err != nil { if err != nil {
panic(err) panic(err)

13
files/set.go
View File

@ -9,12 +9,11 @@ import (
"sync" "sync"
"github.com/calmh/syncthing/protocol" "github.com/calmh/syncthing/protocol"
"github.com/calmh/syncthing/scanner"
"github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb"
) )
type fileRecord struct { type fileRecord struct {
File scanner.File File protocol.FileInfo
Usage int Usage int
Global bool Global bool
} }
@ -37,7 +36,7 @@ func NewSet(repo string, db *leveldb.DB) *Set {
return &s return &s
} }
func (s *Set) Replace(node protocol.NodeID, fs []scanner.File) { func (s *Set) Replace(node protocol.NodeID, fs []protocol.FileInfo) {
if debug { if debug {
l.Debugf("%s Replace(%v, [%d])", s.repo, node, len(fs)) l.Debugf("%s Replace(%v, [%d])", s.repo, node, len(fs))
} }
@ -48,7 +47,7 @@ func (s *Set) Replace(node protocol.NodeID, fs []scanner.File) {
} }
} }
func (s *Set) ReplaceWithDelete(node protocol.NodeID, fs []scanner.File) { func (s *Set) ReplaceWithDelete(node protocol.NodeID, fs []protocol.FileInfo) {
if debug { if debug {
l.Debugf("%s ReplaceWithDelete(%v, [%d])", s.repo, node, len(fs)) l.Debugf("%s ReplaceWithDelete(%v, [%d])", s.repo, node, len(fs))
} }
@ -59,7 +58,7 @@ func (s *Set) ReplaceWithDelete(node protocol.NodeID, fs []scanner.File) {
} }
} }
func (s *Set) Update(node protocol.NodeID, fs []scanner.File) { func (s *Set) Update(node protocol.NodeID, fs []protocol.FileInfo) {
if debug { if debug {
l.Debugf("%s Update(%v, [%d])", s.repo, node, len(fs)) l.Debugf("%s Update(%v, [%d])", s.repo, node, len(fs))
} }
@ -91,11 +90,11 @@ func (s *Set) WithGlobal(fn fileIterator) {
ldbWithGlobal(s.db, []byte(s.repo), fn) ldbWithGlobal(s.db, []byte(s.repo), fn)
} }
func (s *Set) Get(node protocol.NodeID, file string) scanner.File { func (s *Set) Get(node protocol.NodeID, file string) protocol.FileInfo {
return ldbGet(s.db, []byte(s.repo), node[:], []byte(file)) return ldbGet(s.db, []byte(s.repo), node[:], []byte(file))
} }
func (s *Set) GetGlobal(file string) scanner.File { func (s *Set) GetGlobal(file string) protocol.FileInfo {
return ldbGetGlobal(s.db, []byte(s.repo), []byte(file)) return ldbGetGlobal(s.db, []byte(s.repo), []byte(file))
} }

230
files/set_test.go
View File

@ -12,7 +12,6 @@ import (
"github.com/calmh/syncthing/files" "github.com/calmh/syncthing/files"
"github.com/calmh/syncthing/lamport" "github.com/calmh/syncthing/lamport"
"github.com/calmh/syncthing/protocol" "github.com/calmh/syncthing/protocol"
"github.com/calmh/syncthing/scanner"
"github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/storage" "github.com/syndtr/goleveldb/leveldb/storage"
) )
@ -23,48 +22,47 @@ func init() {
remoteNode, _ = protocol.NodeIDFromString("AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR") remoteNode, _ = protocol.NodeIDFromString("AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR")
} }
func genBlocks(n int) []scanner.Block { func genBlocks(n int) []protocol.BlockInfo {
b := make([]scanner.Block, n) b := make([]protocol.BlockInfo, n)
for i := range b { for i := range b {
h := make([]byte, 32) h := make([]byte, 32)
for j := range h { for j := range h {
h[j] = byte(i + j) h[j] = byte(i + j)
} }
b[i].Size = uint32(i) b[i].Size = uint32(i)
b[i].Offset = int64(i)
b[i].Hash = h b[i].Hash = h
} }
return b return b
} }
func globalList(s *files.Set) []scanner.File { func globalList(s *files.Set) []protocol.FileInfo {
var fs []scanner.File var fs []protocol.FileInfo
s.WithGlobal(func(f scanner.File) bool { s.WithGlobal(func(f protocol.FileInfo) bool {
fs = append(fs, f) fs = append(fs, f)
return true return true
}) })
return fs return fs
} }
func haveList(s *files.Set, n protocol.NodeID) []scanner.File { func haveList(s *files.Set, n protocol.NodeID) []protocol.FileInfo {
var fs []scanner.File var fs []protocol.FileInfo
s.WithHave(n, func(f scanner.File) bool { s.WithHave(n, func(f protocol.FileInfo) bool {
fs = append(fs, f) fs = append(fs, f)
return true return true
}) })
return fs return fs
} }
func needList(s *files.Set, n protocol.NodeID) []scanner.File { func needList(s *files.Set, n protocol.NodeID) []protocol.FileInfo {
var fs []scanner.File var fs []protocol.FileInfo
s.WithNeed(n, func(f scanner.File) bool { s.WithNeed(n, func(f protocol.FileInfo) bool {
fs = append(fs, f) fs = append(fs, f)
return true return true
}) })
return fs return fs
} }
type fileList []scanner.File type fileList []protocol.FileInfo
func (l fileList) Len() int { func (l fileList) Len() int {
return len(l) return len(l)
@ -88,44 +86,44 @@ func TestGlobalSet(t *testing.T) {
m := files.NewSet("test", db) m := files.NewSet("test", db)
local0 := []scanner.File{ local0 := []protocol.FileInfo{
scanner.File{Name: "a", Version: 1000, Blocks: genBlocks(1)}, protocol.FileInfo{Name: "a", Version: 1000, Blocks: genBlocks(1)},
scanner.File{Name: "b", Version: 1000, Blocks: genBlocks(2)}, protocol.FileInfo{Name: "b", Version: 1000, Blocks: genBlocks(2)},
scanner.File{Name: "c", Version: 1000, Blocks: genBlocks(3)}, protocol.FileInfo{Name: "c", Version: 1000, Blocks: genBlocks(3)},
scanner.File{Name: "d", Version: 1000, Blocks: genBlocks(4)}, protocol.FileInfo{Name: "d", Version: 1000, Blocks: genBlocks(4)},
scanner.File{Name: "z", Version: 1000, Blocks: genBlocks(8)}, protocol.FileInfo{Name: "z", Version: 1000, Blocks: genBlocks(8)},
} }
local1 := []scanner.File{ local1 := []protocol.FileInfo{
scanner.File{Name: "a", Version: 1000, Blocks: genBlocks(1)}, protocol.FileInfo{Name: "a", Version: 1000, Blocks: genBlocks(1)},
scanner.File{Name: "b", Version: 1000, Blocks: genBlocks(2)}, protocol.FileInfo{Name: "b", Version: 1000, Blocks: genBlocks(2)},
scanner.File{Name: "c", Version: 1000, Blocks: genBlocks(3)}, protocol.FileInfo{Name: "c", Version: 1000, Blocks: genBlocks(3)},
scanner.File{Name: "d", Version: 1000, Blocks: genBlocks(4)}, protocol.FileInfo{Name: "d", Version: 1000, Blocks: genBlocks(4)},
} }
localTot := []scanner.File{ localTot := []protocol.FileInfo{
local0[0], local0[0],
local0[1], local0[1],
local0[2], local0[2],
local0[3], local0[3],
scanner.File{Name: "z", Version: 1001, Flags: protocol.FlagDeleted}, protocol.FileInfo{Name: "z", Version: 1001, Flags: protocol.FlagDeleted},
} }
remote0 := []scanner.File{ remote0 := []protocol.FileInfo{
scanner.File{Name: "a", Version: 1000, Blocks: genBlocks(1)}, protocol.FileInfo{Name: "a", Version: 1000, Blocks: genBlocks(1)},
scanner.File{Name: "b", Version: 1000, Blocks: genBlocks(2)}, protocol.FileInfo{Name: "b", Version: 1000, Blocks: genBlocks(2)},
scanner.File{Name: "c", Version: 1002, Blocks: genBlocks(5)}, protocol.FileInfo{Name: "c", Version: 1002, Blocks: genBlocks(5)},
} }
remote1 := []scanner.File{ remote1 := []protocol.FileInfo{
scanner.File{Name: "b", Version: 1001, Blocks: genBlocks(6)}, protocol.FileInfo{Name: "b", Version: 1001, Blocks: genBlocks(6)},
scanner.File{Name: "e", Version: 1000, Blocks: genBlocks(7)}, protocol.FileInfo{Name: "e", Version: 1000, Blocks: genBlocks(7)},
} }
remoteTot := []scanner.File{ remoteTot := []protocol.FileInfo{
remote0[0], remote0[0],
remote1[0], remote1[0],
remote0[2], remote0[2],
remote1[1], remote1[1],
} }
expectedGlobal := []scanner.File{ expectedGlobal := []protocol.FileInfo{
remote0[0], remote0[0],
remote1[0], remote1[0],
remote0[2], remote0[2],
@ -134,13 +132,13 @@ func TestGlobalSet(t *testing.T) {
localTot[4], localTot[4],
} }
expectedLocalNeed := []scanner.File{ expectedLocalNeed := []protocol.FileInfo{
remote1[0], remote1[0],
remote0[2], remote0[2],
remote1[1], remote1[1],
} }
expectedRemoteNeed := []scanner.File{ expectedRemoteNeed := []protocol.FileInfo{
local0[3], local0[3],
} }
@ -201,12 +199,12 @@ func TestGlobalSet(t *testing.T) {
f = m.Get(protocol.LocalNodeID, "zz") f = m.Get(protocol.LocalNodeID, "zz")
if f.Name != "" { if f.Name != "" {
t.Errorf("Get incorrect;\n A: %v !=\n E: %v", f, scanner.File{}) t.Errorf("Get incorrect;\n A: %v !=\n E: %v", f, protocol.FileInfo{})
} }
f = m.GetGlobal("zz") f = m.GetGlobal("zz")
if f.Name != "" { if f.Name != "" {
t.Errorf("GetGlobal incorrect;\n A: %v !=\n E: %v", f, scanner.File{}) t.Errorf("GetGlobal incorrect;\n A: %v !=\n E: %v", f, protocol.FileInfo{})
} }
av := []protocol.NodeID{protocol.LocalNodeID, remoteNode} av := []protocol.NodeID{protocol.LocalNodeID, remoteNode}
@ -232,41 +230,41 @@ func TestLocalDeleted(t *testing.T) {
m := files.NewSet("test", db) m := files.NewSet("test", db)
lamport.Default = lamport.Clock{} lamport.Default = lamport.Clock{}
local1 := []scanner.File{ local1 := []protocol.FileInfo{
scanner.File{Name: "a", Version: 1000}, protocol.FileInfo{Name: "a", Version: 1000},
scanner.File{Name: "b", Version: 1000}, protocol.FileInfo{Name: "b", Version: 1000},
scanner.File{Name: "c", Version: 1000}, protocol.FileInfo{Name: "c", Version: 1000},
scanner.File{Name: "d", Version: 1000}, protocol.FileInfo{Name: "d", Version: 1000},
scanner.File{Name: "z", Version: 1000, Flags: protocol.FlagDirectory}, protocol.FileInfo{Name: "z", Version: 1000, Flags: protocol.FlagDirectory},
} }
m.ReplaceWithDelete(protocol.LocalNodeID, local1) m.ReplaceWithDelete(protocol.LocalNodeID, local1)
m.ReplaceWithDelete(protocol.LocalNodeID, []scanner.File{ m.ReplaceWithDelete(protocol.LocalNodeID, []protocol.FileInfo{
local1[0], local1[0],
// [1] removed // [1] removed
local1[2], local1[2],
local1[3], local1[3],
local1[4], local1[4],
}) })
m.ReplaceWithDelete(protocol.LocalNodeID, []scanner.File{ m.ReplaceWithDelete(protocol.LocalNodeID, []protocol.FileInfo{
local1[0], local1[0],
local1[2], local1[2],
// [3] removed // [3] removed
local1[4], local1[4],
}) })
m.ReplaceWithDelete(protocol.LocalNodeID, []scanner.File{ m.ReplaceWithDelete(protocol.LocalNodeID, []protocol.FileInfo{
local1[0], local1[0],
local1[2], local1[2],
// [4] removed // [4] removed
}) })
expectedGlobal1 := []scanner.File{ expectedGlobal1 := []protocol.FileInfo{
local1[0], local1[0],
scanner.File{Name: "b", Version: 1001, Flags: protocol.FlagDeleted}, protocol.FileInfo{Name: "b", Version: 1001, Flags: protocol.FlagDeleted},
local1[2], local1[2],
scanner.File{Name: "d", Version: 1002, Flags: protocol.FlagDeleted}, protocol.FileInfo{Name: "d", Version: 1002, Flags: protocol.FlagDeleted},
scanner.File{Name: "z", Version: 1003, Flags: protocol.FlagDeleted | protocol.FlagDirectory}, protocol.FileInfo{Name: "z", Version: 1003, Flags: protocol.FlagDeleted | protocol.FlagDirectory},
} }
g := globalList(m) g := globalList(m)
@ -277,17 +275,17 @@ func TestLocalDeleted(t *testing.T) {
t.Errorf("Global incorrect;\n A: %v !=\n E: %v", g, expectedGlobal1) t.Errorf("Global incorrect;\n A: %v !=\n E: %v", g, expectedGlobal1)
} }
m.ReplaceWithDelete(protocol.LocalNodeID, []scanner.File{ m.ReplaceWithDelete(protocol.LocalNodeID, []protocol.FileInfo{
local1[0], local1[0],
// [2] removed // [2] removed
}) })
expectedGlobal2 := []scanner.File{ expectedGlobal2 := []protocol.FileInfo{
local1[0], local1[0],
scanner.File{Name: "b", Version: 1001, Flags: protocol.FlagDeleted}, protocol.FileInfo{Name: "b", Version: 1001, Flags: protocol.FlagDeleted},
scanner.File{Name: "c", Version: 1004, Flags: protocol.FlagDeleted}, protocol.FileInfo{Name: "c", Version: 1004, Flags: protocol.FlagDeleted},
scanner.File{Name: "d", Version: 1002, Flags: protocol.FlagDeleted}, protocol.FileInfo{Name: "d", Version: 1002, Flags: protocol.FlagDeleted},
scanner.File{Name: "z", Version: 1003, Flags: protocol.FlagDeleted | protocol.FlagDirectory}, protocol.FileInfo{Name: "z", Version: 1003, Flags: protocol.FlagDeleted | protocol.FlagDirectory},
} }
g = globalList(m) g = globalList(m)
@ -305,9 +303,9 @@ func Benchmark10kReplace(b *testing.B) {
b.Fatal(err) b.Fatal(err)
} }
var local []scanner.File var local []protocol.FileInfo
for i := 0; i < 10000; i++ { for i := 0; i < 10000; i++ {
local = append(local, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000}) local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
} }
b.ResetTimer() b.ResetTimer()
@ -318,9 +316,9 @@ func Benchmark10kReplace(b *testing.B) {
} }
func Benchmark10kUpdateChg(b *testing.B) { func Benchmark10kUpdateChg(b *testing.B) {
var remote []scanner.File var remote []protocol.FileInfo
for i := 0; i < 10000; i++ { for i := 0; i < 10000; i++ {
remote = append(remote, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000}) remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
} }
db, err := leveldb.Open(storage.NewMemStorage(), nil) db, err := leveldb.Open(storage.NewMemStorage(), nil)
@ -331,9 +329,9 @@ func Benchmark10kUpdateChg(b *testing.B) {
m := files.NewSet("test", db) m := files.NewSet("test", db)
m.Replace(remoteNode, remote) m.Replace(remoteNode, remote)
var local []scanner.File var local []protocol.FileInfo
for i := 0; i < 10000; i++ { for i := 0; i < 10000; i++ {
local = append(local, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000}) local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
} }
m.ReplaceWithDelete(protocol.LocalNodeID, local) m.ReplaceWithDelete(protocol.LocalNodeID, local)
@ -350,9 +348,9 @@ func Benchmark10kUpdateChg(b *testing.B) {
} }
func Benchmark10kUpdateSme(b *testing.B) { func Benchmark10kUpdateSme(b *testing.B) {
var remote []scanner.File var remote []protocol.FileInfo
for i := 0; i < 10000; i++ { for i := 0; i < 10000; i++ {
remote = append(remote, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000}) remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
} }
db, err := leveldb.Open(storage.NewMemStorage(), nil) db, err := leveldb.Open(storage.NewMemStorage(), nil)
@ -362,9 +360,9 @@ func Benchmark10kUpdateSme(b *testing.B) {
m := files.NewSet("test", db) m := files.NewSet("test", db)
m.Replace(remoteNode, remote) m.Replace(remoteNode, remote)
var local []scanner.File var local []protocol.FileInfo
for i := 0; i < 10000; i++ { for i := 0; i < 10000; i++ {
local = append(local, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000}) local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
} }
m.ReplaceWithDelete(protocol.LocalNodeID, local) m.ReplaceWithDelete(protocol.LocalNodeID, local)
@ -376,9 +374,9 @@ func Benchmark10kUpdateSme(b *testing.B) {
} }
func Benchmark10kNeed2k(b *testing.B) { func Benchmark10kNeed2k(b *testing.B) {
var remote []scanner.File var remote []protocol.FileInfo
for i := 0; i < 10000; i++ { for i := 0; i < 10000; i++ {
remote = append(remote, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000}) remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
} }
db, err := leveldb.Open(storage.NewMemStorage(), nil) db, err := leveldb.Open(storage.NewMemStorage(), nil)
@ -389,12 +387,12 @@ func Benchmark10kNeed2k(b *testing.B) {
m := files.NewSet("test", db) m := files.NewSet("test", db)
m.Replace(remoteNode, remote) m.Replace(remoteNode, remote)
var local []scanner.File var local []protocol.FileInfo
for i := 0; i < 8000; i++ { for i := 0; i < 8000; i++ {
local = append(local, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000}) local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
} }
for i := 8000; i < 10000; i++ { for i := 8000; i < 10000; i++ {
local = append(local, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 980}) local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 980})
} }
m.ReplaceWithDelete(protocol.LocalNodeID, local) m.ReplaceWithDelete(protocol.LocalNodeID, local)
@ -409,9 +407,9 @@ func Benchmark10kNeed2k(b *testing.B) {
} }
func Benchmark10kHaveFullList(b *testing.B) { func Benchmark10kHaveFullList(b *testing.B) {
var remote []scanner.File var remote []protocol.FileInfo
for i := 0; i < 10000; i++ { for i := 0; i < 10000; i++ {
remote = append(remote, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000}) remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
} }
db, err := leveldb.Open(storage.NewMemStorage(), nil) db, err := leveldb.Open(storage.NewMemStorage(), nil)
@ -422,12 +420,12 @@ func Benchmark10kHaveFullList(b *testing.B) {
m := files.NewSet("test", db) m := files.NewSet("test", db)
m.Replace(remoteNode, remote) m.Replace(remoteNode, remote)
var local []scanner.File var local []protocol.FileInfo
for i := 0; i < 2000; i++ { for i := 0; i < 2000; i++ {
local = append(local, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000}) local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
} }
for i := 2000; i < 10000; i++ { for i := 2000; i < 10000; i++ {
local = append(local, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 980}) local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 980})
} }
m.ReplaceWithDelete(protocol.LocalNodeID, local) m.ReplaceWithDelete(protocol.LocalNodeID, local)
@ -442,9 +440,9 @@ func Benchmark10kHaveFullList(b *testing.B) {
} }
func Benchmark10kGlobal(b *testing.B) { func Benchmark10kGlobal(b *testing.B) {
var remote []scanner.File var remote []protocol.FileInfo
for i := 0; i < 10000; i++ { for i := 0; i < 10000; i++ {
remote = append(remote, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000}) remote = append(remote, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
} }
db, err := leveldb.Open(storage.NewMemStorage(), nil) db, err := leveldb.Open(storage.NewMemStorage(), nil)
@ -455,12 +453,12 @@ func Benchmark10kGlobal(b *testing.B) {
m := files.NewSet("test", db) m := files.NewSet("test", db)
m.Replace(remoteNode, remote) m.Replace(remoteNode, remote)
var local []scanner.File var local []protocol.FileInfo
for i := 0; i < 2000; i++ { for i := 0; i < 2000; i++ {
local = append(local, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000}) local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 1000})
} }
for i := 2000; i < 10000; i++ { for i := 2000; i < 10000; i++ {
local = append(local, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 980}) local = append(local, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Version: 980})
} }
m.ReplaceWithDelete(protocol.LocalNodeID, local) m.ReplaceWithDelete(protocol.LocalNodeID, local)
@ -482,18 +480,18 @@ func TestGlobalReset(t *testing.T) {
m := files.NewSet("test", db) m := files.NewSet("test", db)
local := []scanner.File{ local := []protocol.FileInfo{
scanner.File{Name: "a", Version: 1000}, protocol.FileInfo{Name: "a", Version: 1000},
scanner.File{Name: "b", Version: 1000}, protocol.FileInfo{Name: "b", Version: 1000},
scanner.File{Name: "c", Version: 1000}, protocol.FileInfo{Name: "c", Version: 1000},
scanner.File{Name: "d", Version: 1000}, protocol.FileInfo{Name: "d", Version: 1000},
} }
remote := []scanner.File{ remote := []protocol.FileInfo{
scanner.File{Name: "a", Version: 1000}, protocol.FileInfo{Name: "a", Version: 1000},
scanner.File{Name: "b", Version: 1001}, protocol.FileInfo{Name: "b", Version: 1001},
scanner.File{Name: "c", Version: 1002}, protocol.FileInfo{Name: "c", Version: 1002},
scanner.File{Name: "e", Version: 1000}, protocol.FileInfo{Name: "e", Version: 1000},
} }
m.ReplaceWithDelete(protocol.LocalNodeID, local) m.ReplaceWithDelete(protocol.LocalNodeID, local)
@ -523,24 +521,24 @@ func TestNeed(t *testing.T) {
m := files.NewSet("test", db) m := files.NewSet("test", db)
local := []scanner.File{ local := []protocol.FileInfo{
scanner.File{Name: "a", Version: 1000}, protocol.FileInfo{Name: "a", Version: 1000},
scanner.File{Name: "b", Version: 1000}, protocol.FileInfo{Name: "b", Version: 1000},
scanner.File{Name: "c", Version: 1000}, protocol.FileInfo{Name: "c", Version: 1000},
scanner.File{Name: "d", Version: 1000}, protocol.FileInfo{Name: "d", Version: 1000},
} }
remote := []scanner.File{ remote := []protocol.FileInfo{
scanner.File{Name: "a", Version: 1000}, protocol.FileInfo{Name: "a", Version: 1000},
scanner.File{Name: "b", Version: 1001}, protocol.FileInfo{Name: "b", Version: 1001},
scanner.File{Name: "c", Version: 1002}, protocol.FileInfo{Name: "c", Version: 1002},
scanner.File{Name: "e", Version: 1000}, protocol.FileInfo{Name: "e", Version: 1000},
} }
shouldNeed := []scanner.File{ shouldNeed := []protocol.FileInfo{
scanner.File{Name: "b", Version: 1001}, protocol.FileInfo{Name: "b", Version: 1001},
scanner.File{Name: "c", Version: 1002}, protocol.FileInfo{Name: "c", Version: 1002},
scanner.File{Name: "e", Version: 1000}, protocol.FileInfo{Name: "e", Version: 1000},
} }
m.ReplaceWithDelete(protocol.LocalNodeID, local) m.ReplaceWithDelete(protocol.LocalNodeID, local)
@ -564,19 +562,19 @@ func TestChanges(t *testing.T) {
m := files.NewSet("test", db) m := files.NewSet("test", db)
local1 := []scanner.File{ local1 := []protocol.FileInfo{
scanner.File{Name: "a", Version: 1000}, protocol.FileInfo{Name: "a", Version: 1000},
scanner.File{Name: "b", Version: 1000}, protocol.FileInfo{Name: "b", Version: 1000},
scanner.File{Name: "c", Version: 1000}, protocol.FileInfo{Name: "c", Version: 1000},
scanner.File{Name: "d", Version: 1000}, protocol.FileInfo{Name: "d", Version: 1000},
} }
local2 := []scanner.File{ local2 := []protocol.FileInfo{
local1[0], local1[0],
// [1] deleted // [1] deleted
local1[2], local1[2],
scanner.File{Name: "d", Version: 1002}, protocol.FileInfo{Name: "d", Version: 1002},
scanner.File{Name: "e", Version: 1000}, protocol.FileInfo{Name: "e", Version: 1000},
} }
m.ReplaceWithDelete(protocol.LocalNodeID, local1) m.ReplaceWithDelete(protocol.LocalNodeID, local1)

11
files/sort.go
View File

@ -2,13 +2,12 @@ package files
import ( import (
"sort" "sort"
"github.com/calmh/syncthing/protocol"
"github.com/calmh/syncthing/scanner"
) )
type SortBy func(p scanner.File) int type SortBy func(p protocol.FileInfo) int
func (by SortBy) Sort(files []scanner.File) { func (by SortBy) Sort(files []protocol.FileInfo) {
ps := &fileSorter{ ps := &fileSorter{
files: files, files: files,
by: by, by: by,
@ -17,8 +16,8 @@ func (by SortBy) Sort(files []scanner.File) {
} }
type fileSorter struct { type fileSorter struct {
files []scanner.File files []protocol.FileInfo
by func(p1 scanner.File) int by func(p1 protocol.FileInfo) int
} }
func (s *fileSorter) Len() int { func (s *fileSorter) Len() int {

14
model/blockqueue.go
View File

@ -7,19 +7,19 @@ package model
import ( import (
"sync" "sync"
"github.com/calmh/syncthing/scanner" "github.com/calmh/syncthing/protocol"
) )
type bqAdd struct { type bqAdd struct {
file scanner.File file protocol.FileInfo
have []scanner.Block have []protocol.BlockInfo
need []scanner.Block need []protocol.BlockInfo
} }
type bqBlock struct { type bqBlock struct {
file scanner.File file protocol.FileInfo
block scanner.Block // get this block from the network block protocol.BlockInfo // get this block from the network
copy []scanner.Block // copy these blocks from the old version of the file copy []protocol.BlockInfo // copy these blocks from the old version of the file
first bool first bool
last bool last bool
} }

101
model/model.go
View File

@ -162,11 +162,13 @@ func (m *Model) ConnectionStats() map[string]ConnectionInfo {
var have int64 var have int64
for _, repo := range m.nodeRepos[node] { for _, repo := range m.nodeRepos[node] {
m.repoFiles[repo].WithGlobal(func(f scanner.File) bool { m.repoFiles[repo].WithGlobal(func(f protocol.FileInfo) bool {
if !protocol.IsDeleted(f.Flags) { if !protocol.IsDeleted(f.Flags) {
size := f.Size var size int64
if protocol.IsDirectory(f.Flags) { if protocol.IsDirectory(f.Flags) {
size = zeroEntrySize size = zeroEntrySize
} else {
size = f.Size()
} }
tot += size tot += size
have += size have += size
@ -174,11 +176,13 @@ func (m *Model) ConnectionStats() map[string]ConnectionInfo {
return true return true
}) })
m.repoFiles[repo].WithNeed(node, func(f scanner.File) bool { m.repoFiles[repo].WithNeed(node, func(f protocol.FileInfo) bool {
if !protocol.IsDeleted(f.Flags) { if !protocol.IsDeleted(f.Flags) {
size := f.Size var size int64
if protocol.IsDirectory(f.Flags) { if protocol.IsDirectory(f.Flags) {
size = zeroEntrySize size = zeroEntrySize
} else {
size = f.Size()
} }
have -= size have -= size
} }
@ -209,7 +213,7 @@ func (m *Model) ConnectionStats() map[string]ConnectionInfo {
return res return res
} }
func sizeOf(fs []scanner.File) (files, deleted int, bytes int64) { func sizeOf(fs []protocol.FileInfo) (files, deleted int, bytes int64) {
for _, f := range fs { for _, f := range fs {
fs, de, by := sizeOfFile(f) fs, de, by := sizeOfFile(f)
files += fs files += fs
@ -219,11 +223,11 @@ func sizeOf(fs []scanner.File) (files, deleted int, bytes int64) {
return return
} }
func sizeOfFile(f scanner.File) (files, deleted int, bytes int64) { func sizeOfFile(f protocol.FileInfo) (files, deleted int, bytes int64) {
if !protocol.IsDeleted(f.Flags) { if !protocol.IsDeleted(f.Flags) {
files++ files++
if !protocol.IsDirectory(f.Flags) { if !protocol.IsDirectory(f.Flags) {
bytes += f.Size bytes += f.Size()
} else { } else {
bytes += zeroEntrySize bytes += zeroEntrySize
} }
@ -240,7 +244,7 @@ func (m *Model) GlobalSize(repo string) (files, deleted int, bytes int64) {
m.rmut.RLock() m.rmut.RLock()
defer m.rmut.RUnlock() defer m.rmut.RUnlock()
if rf, ok := m.repoFiles[repo]; ok { if rf, ok := m.repoFiles[repo]; ok {
rf.WithGlobal(func(f scanner.File) bool { rf.WithGlobal(func(f protocol.FileInfo) bool {
fs, de, by := sizeOfFile(f) fs, de, by := sizeOfFile(f)
files += fs files += fs
deleted += de deleted += de
@ -257,7 +261,7 @@ func (m *Model) LocalSize(repo string) (files, deleted int, bytes int64) {
m.rmut.RLock() m.rmut.RLock()
defer m.rmut.RUnlock() defer m.rmut.RUnlock()
if rf, ok := m.repoFiles[repo]; ok { if rf, ok := m.repoFiles[repo]; ok {
rf.WithHave(protocol.LocalNodeID, func(f scanner.File) bool { rf.WithHave(protocol.LocalNodeID, func(f protocol.FileInfo) bool {
fs, de, by := sizeOfFile(f) fs, de, by := sizeOfFile(f)
files += fs files += fs
deleted += de deleted += de
@ -275,12 +279,12 @@ func (m *Model) NeedSize(repo string) (files int, bytes int64) {
} }
// NeedFiles returns the list of currently needed files // NeedFiles returns the list of currently needed files
func (m *Model) NeedFilesRepo(repo string) []scanner.File { func (m *Model) NeedFilesRepo(repo string) []protocol.FileInfo {
m.rmut.RLock() m.rmut.RLock()
defer m.rmut.RUnlock() defer m.rmut.RUnlock()
if rf, ok := m.repoFiles[repo]; ok { if rf, ok := m.repoFiles[repo]; ok {
var fs []scanner.File var fs []protocol.FileInfo
rf.WithNeed(protocol.LocalNodeID, func(f scanner.File) bool { rf.WithNeed(protocol.LocalNodeID, func(f protocol.FileInfo) bool {
fs = append(fs, f) fs = append(fs, f)
return true return true
}) })
@ -304,23 +308,9 @@ func (m *Model) Index(nodeID protocol.NodeID, repo string, fs []protocol.FileInf
return return
} }
var files = make([]scanner.File, len(fs))
for i := range fs {
f := fs[i]
lamport.Default.Tick(f.Version)
if debug {
var flagComment string
if protocol.IsDeleted(f.Flags) {
flagComment = " (deleted)"
}
l.Debugf("IDX(in): %s %q/%q m=%d f=%o%s v=%d (%d blocks)", nodeID, repo, f.Name, f.Modified, f.Flags, flagComment, f.Version, len(f.Blocks))
}
files[i] = fileFromFileInfo(f)
}
m.rmut.RLock() m.rmut.RLock()
if r, ok := m.repoFiles[repo]; ok { if r, ok := m.repoFiles[repo]; ok {
r.Replace(nodeID, files) r.Replace(nodeID, fs)
} else { } else {
l.Fatalf("Index for nonexistant repo %q", repo) l.Fatalf("Index for nonexistant repo %q", repo)
} }
@ -339,23 +329,9 @@ func (m *Model) IndexUpdate(nodeID protocol.NodeID, repo string, fs []protocol.F
return return
} }
var files = make([]scanner.File, len(fs))
for i := range fs {
f := fs[i]
lamport.Default.Tick(f.Version)
if debug {
var flagComment string
if protocol.IsDeleted(f.Flags) {
flagComment = " (deleted)"
}
l.Debugf("IDXUP(in): %s %q/%q m=%d f=%o%s v=%d (%d blocks)", nodeID, repo, f.Name, f.Modified, f.Flags, flagComment, f.Version, len(f.Blocks))
}
files[i] = fileFromFileInfo(f)
}
m.rmut.RLock() m.rmut.RLock()
if r, ok := m.repoFiles[repo]; ok { if r, ok := m.repoFiles[repo]; ok {
r.Update(nodeID, files) r.Update(nodeID, fs)
} else { } else {
l.Fatalf("IndexUpdate for nonexistant repo %q", repo) l.Fatalf("IndexUpdate for nonexistant repo %q", repo)
} }
@ -432,14 +408,14 @@ func (m *Model) Request(nodeID protocol.NodeID, repo, name string, offset int64,
} }
lf := r.Get(protocol.LocalNodeID, name) lf := r.Get(protocol.LocalNodeID, name)
if lf.Suppressed || protocol.IsDeleted(lf.Flags) { if protocol.IsInvalid(lf.Flags) || protocol.IsDeleted(lf.Flags) {
if debug { if debug {
l.Debugf("REQ(in): %s: %q / %q o=%d s=%d; invalid: %v", nodeID, repo, name, offset, size, lf) l.Debugf("REQ(in): %s: %q / %q o=%d s=%d; invalid: %v", nodeID, repo, name, offset, size, lf)
} }
return nil, ErrInvalid return nil, ErrInvalid
} }
if offset > lf.Size { if offset > lf.Size() {
if debug { if debug {
l.Debugf("REQ(in; nonexistent): %s: %q o=%d s=%d", nodeID, name, offset, size) l.Debugf("REQ(in; nonexistent): %s: %q o=%d s=%d", nodeID, name, offset, size)
} }
@ -468,20 +444,20 @@ func (m *Model) Request(nodeID protocol.NodeID, repo, name string, offset int64,
} }
// ReplaceLocal replaces the local repository index with the given list of files. // ReplaceLocal replaces the local repository index with the given list of files.
func (m *Model) ReplaceLocal(repo string, fs []scanner.File) { func (m *Model) ReplaceLocal(repo string, fs []protocol.FileInfo) {
m.rmut.RLock() m.rmut.RLock()
m.repoFiles[repo].ReplaceWithDelete(protocol.LocalNodeID, fs) m.repoFiles[repo].ReplaceWithDelete(protocol.LocalNodeID, fs)
m.rmut.RUnlock() m.rmut.RUnlock()
} }
func (m *Model) CurrentRepoFile(repo string, file string) scanner.File { func (m *Model) CurrentRepoFile(repo string, file string) protocol.FileInfo {
m.rmut.RLock() m.rmut.RLock()
f := m.repoFiles[repo].Get(protocol.LocalNodeID, file) f := m.repoFiles[repo].Get(protocol.LocalNodeID, file)
m.rmut.RUnlock() m.rmut.RUnlock()
return f return f
} }
func (m *Model) CurrentGlobalFile(repo string, file string) scanner.File { func (m *Model) CurrentGlobalFile(repo string, file string) protocol.FileInfo {
m.rmut.RLock() m.rmut.RLock()
f := m.repoFiles[repo].GetGlobal(file) f := m.repoFiles[repo].GetGlobal(file)
m.rmut.RUnlock() m.rmut.RUnlock()
@ -494,7 +470,7 @@ type cFiler struct {
} }
// Implements scanner.CurrentFiler // Implements scanner.CurrentFiler
func (cf cFiler) CurrentFile(file string) scanner.File { func (cf cFiler) CurrentFile(file string) protocol.FileInfo {
return cf.m.CurrentRepoFile(cf.r, file) return cf.m.CurrentRepoFile(cf.r, file)
} }
@ -552,32 +528,18 @@ func (m *Model) AddConnection(rawConn io.Closer, protoConn protocol.Connection)
// protocolIndex returns the current local index in protocol data types. // protocolIndex returns the current local index in protocol data types.
func (m *Model) protocolIndex(repo string) []protocol.FileInfo { func (m *Model) protocolIndex(repo string) []protocol.FileInfo {
var index []protocol.FileInfo var fs []protocol.FileInfo
m.repoFiles[repo].WithHave(protocol.LocalNodeID, func(f protocol.FileInfo) bool {
var fs []scanner.File
m.repoFiles[repo].WithHave(protocol.LocalNodeID, func(f scanner.File) bool {
fs = append(fs, f) fs = append(fs, f)
return true return true
}) })
for _, f := range fs { return fs
mf := fileInfoFromFile(f)
if debug {
var flagComment string
if protocol.IsDeleted(mf.Flags) {
flagComment = " (deleted)"
}
l.Debugf("IDX(out): %q/%q m=%d f=%o%s v=%d (%d blocks)", repo, mf.Name, mf.Modified, mf.Flags, flagComment, mf.Version, len(mf.Blocks))
}
index = append(index, mf)
}
return index
} }
func (m *Model) updateLocal(repo string, f scanner.File) { func (m *Model) updateLocal(repo string, f protocol.FileInfo) {
m.rmut.RLock() m.rmut.RLock()
m.repoFiles[repo].Update(protocol.LocalNodeID, []scanner.File{f}) m.repoFiles[repo].Update(protocol.LocalNodeID, []protocol.FileInfo{f})
m.rmut.RUnlock() m.rmut.RUnlock()
} }
@ -736,11 +698,10 @@ func (m *Model) LoadIndexes(dir string) {
for repo := range m.repoCfgs { for repo := range m.repoCfgs {
fs := m.loadIndex(repo, dir) fs := m.loadIndex(repo, dir)
var sfs = make([]scanner.File, len(fs)) var sfs = make([]protocol.FileInfo, len(fs))
for i := 0; i < len(fs); i++ { for i := 0; i < len(fs); i++ {
lamport.Default.Tick(fs[i].Version) lamport.Default.Tick(fs[i].Version)
sfs[i] = fileFromFileInfo(fs[i]) fs[i].Flags &= ^uint32(protocol.FlagInvalid) // we might have saved an index with files that were suppressed; the should not be on startup
sfs[i].Suppressed = false // we might have saved an index with files that were suppressed; the should not be on startup
} }
m.repoFiles[repo].Replace(protocol.LocalNodeID, sfs) m.repoFiles[repo].Replace(protocol.LocalNodeID, sfs)

22
model/model_test.go
View File

@ -13,7 +13,6 @@ import (
"github.com/calmh/syncthing/config" "github.com/calmh/syncthing/config"
"github.com/calmh/syncthing/protocol" "github.com/calmh/syncthing/protocol"
"github.com/calmh/syncthing/scanner"
"github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/storage" "github.com/syndtr/goleveldb/leveldb/storage"
) )
@ -25,27 +24,24 @@ func init() {
node2, _ = protocol.NodeIDFromString("GYRZZQB-IRNPV4Z-T7TC52W-EQYJ3TT-FDQW6MW-DFLMU42-SSSU6EM-FBK2VAY") node2, _ = protocol.NodeIDFromString("GYRZZQB-IRNPV4Z-T7TC52W-EQYJ3TT-FDQW6MW-DFLMU42-SSSU6EM-FBK2VAY")
} }
var testDataExpected = map[string]scanner.File{ var testDataExpected = map[string]protocol.FileInfo{
"foo": scanner.File{ "foo": protocol.FileInfo{
Name: "foo", Name: "foo",
Flags: 0, Flags: 0,
Modified: 0, Modified: 0,
Size: 7, Blocks: []protocol.BlockInfo{{Offset: 0x0, Size: 0x7, Hash: []uint8{0xae, 0xc0, 0x70, 0x64, 0x5f, 0xe5, 0x3e, 0xe3, 0xb3, 0x76, 0x30, 0x59, 0x37, 0x61, 0x34, 0xf0, 0x58, 0xcc, 0x33, 0x72, 0x47, 0xc9, 0x78, 0xad, 0xd1, 0x78, 0xb6, 0xcc, 0xdf, 0xb0, 0x1, 0x9f}}},
Blocks: []scanner.Block{{Offset: 0x0, Size: 0x7, Hash: []uint8{0xae, 0xc0, 0x70, 0x64, 0x5f, 0xe5, 0x3e, 0xe3, 0xb3, 0x76, 0x30, 0x59, 0x37, 0x61, 0x34, 0xf0, 0x58, 0xcc, 0x33, 0x72, 0x47, 0xc9, 0x78, 0xad, 0xd1, 0x78, 0xb6, 0xcc, 0xdf, 0xb0, 0x1, 0x9f}}},
}, },
"empty": scanner.File{ "empty": protocol.FileInfo{
Name: "empty", Name: "empty",
Flags: 0, Flags: 0,
Modified: 0, Modified: 0,
Size: 0, Blocks: []protocol.BlockInfo{{Offset: 0x0, Size: 0x0, Hash: []uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55}}},
Blocks: []scanner.Block{{Offset: 0x0, Size: 0x0, Hash: []uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55}}},
}, },
"bar": scanner.File{ "bar": protocol.FileInfo{
Name: "bar", Name: "bar",
Flags: 0, Flags: 0,
Modified: 0, Modified: 0,
Size: 10, Blocks: []protocol.BlockInfo{{Offset: 0x0, Size: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}},
Blocks: []scanner.Block{{Offset: 0x0, Size: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}},
}, },
} }
@ -89,7 +85,7 @@ func genFiles(n int) []protocol.FileInfo {
files[i] = protocol.FileInfo{ files[i] = protocol.FileInfo{
Name: fmt.Sprintf("file%d", i), Name: fmt.Sprintf("file%d", i),
Modified: t, Modified: t,
Blocks: []protocol.BlockInfo{{100, []byte("some hash bytes")}}, Blocks: []protocol.BlockInfo{{0, 100, []byte("some hash bytes")}},
} }
} }
@ -212,7 +208,7 @@ func BenchmarkRequest(b *testing.B) {
files[i] = protocol.FileInfo{ files[i] = protocol.FileInfo{
Name: fmt.Sprintf("file%d", i), Name: fmt.Sprintf("file%d", i),
Modified: t, Modified: t,
Blocks: []protocol.BlockInfo{{100, []byte("some hash bytes")}}, Blocks: []protocol.BlockInfo{{0, 100, []byte("some hash bytes")}},
} }
} }

4
model/puller.go
View File

@ -21,7 +21,7 @@ import (
type requestResult struct { type requestResult struct {
node protocol.NodeID node protocol.NodeID
file scanner.File file protocol.FileInfo
filepath string // full filepath name filepath string // full filepath name
offset int64 offset int64
data []byte data []byte
@ -621,7 +621,7 @@ func (p *puller) queueNeededBlocks() {
} }
} }
func (p *puller) closeFile(f scanner.File) { func (p *puller) closeFile(f protocol.FileInfo) {
if debug { if debug {
l.Debugf("pull: closing %q / %q", p.repoCfg.ID, f.Name) l.Debugf("pull: closing %q / %q", p.repoCfg.ID, f.Name)
} }

46
model/util.go
View File

@ -6,58 +6,12 @@ package model
import ( import (
"fmt" "fmt"
"path/filepath"
"sync" "sync"
"time" "time"
"github.com/calmh/syncthing/protocol" "github.com/calmh/syncthing/protocol"
"github.com/calmh/syncthing/scanner"
) )
func fileFromFileInfo(f protocol.FileInfo) scanner.File {
var blocks = make([]scanner.Block, len(f.Blocks))
var offset int64
for i, b := range f.Blocks {
blocks[i] = scanner.Block{
Offset: offset,
Size: b.Size,
Hash: b.Hash,
}
offset += int64(b.Size)
}
return scanner.File{
// Name is with native separator and normalization
Name: filepath.FromSlash(f.Name),
Size: offset,
Flags: f.Flags &^ protocol.FlagInvalid,
Modified: f.Modified,
Version: f.Version,
Blocks: blocks,
Suppressed: f.Flags&protocol.FlagInvalid != 0,
}
}
func fileInfoFromFile(f scanner.File) protocol.FileInfo {
var blocks = make([]protocol.BlockInfo, len(f.Blocks))
for i, b := range f.Blocks {
blocks[i] = protocol.BlockInfo{
Size: b.Size,
Hash: b.Hash,
}
}
pf := protocol.FileInfo{
Name: filepath.ToSlash(f.Name),
Flags: f.Flags,
Modified: f.Modified,
Version: f.Version,
Blocks: blocks,
}
if f.Suppressed {
pf.Flags |= protocol.FlagInvalid
}
return pf
}
func cmMap(cm protocol.ClusterConfigMessage) map[string]map[protocol.NodeID]uint32 { func cmMap(cm protocol.ClusterConfigMessage) map[string]map[protocol.NodeID]uint32 {
m := make(map[string]map[protocol.NodeID]uint32) m := make(map[string]map[protocol.NodeID]uint32)
for _, repo := range cm.Repositories { for _, repo := range cm.Repositories {

27
protocol/message.go
View File

@ -4,6 +4,8 @@
package protocol package protocol
import "fmt"
type IndexMessage struct { type IndexMessage struct {
Repository string // max:64 Repository string // max:64
Files []FileInfo // max:10000000 Files []FileInfo // max:10000000
@ -17,11 +19,36 @@ type FileInfo struct {
Blocks []BlockInfo // max:1000000 Blocks []BlockInfo // max:1000000
} }
func (f FileInfo) String() string {
return fmt.Sprintf("File{Name:%q, Flags:0%o, Modified:%d, Version:%d, Size:%d, Blocks:%v}",
f.Name, f.Flags, f.Modified, f.Version, f.Size(), f.Blocks)
}
func (f FileInfo) Size() (bytes int64) {
for _, b := range f.Blocks {
bytes += int64(b.Size)
}
return
}
func (a FileInfo) Equals(b FileInfo) bool {
return a.Version == b.Version
}
func (a FileInfo) NewerThan(b FileInfo) bool {
return a.Version > b.Version
}
type BlockInfo struct { type BlockInfo struct {
Offset int64 // noencode (cache only)
Size uint32 Size uint32
Hash []byte // max:64 Hash []byte // max:64
} }
func (b BlockInfo) String() string {
return fmt.Sprintf("Block{%d/%d/%x}", b.Offset, b.Size, b.Hash)
}
type RequestMessage struct { type RequestMessage struct {
Repository string // max:64 Repository string // max:64
Name string // max:1024 Name string // max:1024

32
scanner/blocks.go
View File

@ -7,25 +7,16 @@ package scanner
import ( import (
"bytes" "bytes"
"crypto/sha256" "crypto/sha256"
"fmt"
"io" "io"
"github.com/calmh/syncthing/protocol"
) )
const StandardBlockSize = 128 * 1024 const StandardBlockSize = 128 * 1024
type Block struct {
Offset int64
Size uint32
Hash []byte
}
func (b Block) String() string {
return fmt.Sprintf("%d/%d/%x", b.Offset, b.Size, b.Hash)
}
// Blocks returns the blockwise hash of the reader. // Blocks returns the blockwise hash of the reader.
func Blocks(r io.Reader, blocksize int) ([]Block, error) { func Blocks(r io.Reader, blocksize int) ([]protocol.BlockInfo, error) {
var blocks []Block var blocks []protocol.BlockInfo
var offset int64 var offset int64
for { for {
lr := &io.LimitedReader{R: r, N: int64(blocksize)} lr := &io.LimitedReader{R: r, N: int64(blocksize)}
@ -39,9 +30,9 @@ func Blocks(r io.Reader, blocksize int) ([]Block, error) {
break break
} }
b := Block{ b := protocol.BlockInfo{
Offset: offset,
Size: uint32(n), Size: uint32(n),
Offset: offset,
Hash: hf.Sum(nil), Hash: hf.Sum(nil),
} }
blocks = append(blocks, b) blocks = append(blocks, b)
@ -50,7 +41,7 @@ func Blocks(r io.Reader, blocksize int) ([]Block, error) {
if len(blocks) == 0 { if len(blocks) == 0 {
// Empty file // Empty file
blocks = append(blocks, Block{ blocks = append(blocks, protocol.BlockInfo{
Offset: 0, Offset: 0,
Size: 0, Size: 0,
Hash: []uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55}, Hash: []uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55},
@ -62,11 +53,18 @@ func Blocks(r io.Reader, blocksize int) ([]Block, error) {
// BlockDiff returns lists of common and missing (to transform src into tgt) // BlockDiff returns lists of common and missing (to transform src into tgt)
// blocks. Both block lists must have been created with the same block size. // blocks. Both block lists must have been created with the same block size.
func BlockDiff(src, tgt []Block) (have, need []Block) { func BlockDiff(src, tgt []protocol.BlockInfo) (have, need []protocol.BlockInfo) {
if len(tgt) == 0 && len(src) != 0 { if len(tgt) == 0 && len(src) != 0 {
return nil, nil return nil, nil
} }
// Set the Offset field on each target block
var offset int64
for i := range tgt {
tgt[i].Offset = offset
offset += int64(tgt[i].Size)
}
if len(tgt) != 0 && len(src) == 0 { if len(tgt) != 0 && len(src) == 0 {
// Copy the entire file // Copy the entire file
return nil, tgt return nil, tgt

26
scanner/blocks_test.go
View File

@ -8,6 +8,8 @@ import (
"bytes" "bytes"
"fmt" "fmt"
"testing" "testing"
"github.com/calmh/syncthing/protocol"
) )
var blocksTestData = []struct { var blocksTestData = []struct {
@ -83,20 +85,20 @@ var diffTestData = []struct {
a string a string
b string b string
s int s int
d []Block d []protocol.BlockInfo
}{ }{
{"contents", "contents", 1024, []Block{}}, {"contents", "contents", 1024, []protocol.BlockInfo{}},
{"", "", 1024, []Block{}}, {"", "", 1024, []protocol.BlockInfo{}},
{"contents", "contents", 3, []Block{}}, {"contents", "contents", 3, []protocol.BlockInfo{}},
{"contents", "cantents", 3, []Block{{0, 3, nil}}}, {"contents", "cantents", 3, []protocol.BlockInfo{{0, 3, nil}}},
{"contents", "contants", 3, []Block{{3, 3, nil}}}, {"contents", "contants", 3, []protocol.BlockInfo{{3, 3, nil}}},
{"contents", "cantants", 3, []Block{{0, 3, nil}, {3, 3, nil}}}, {"contents", "cantants", 3, []protocol.BlockInfo{{0, 3, nil}, {3, 3, nil}}},
{"contents", "", 3, []Block{{0, 0, nil}}}, {"contents", "", 3, []protocol.BlockInfo{{0, 0, nil}}},
{"", "contents", 3, []Block{{0, 3, nil}, {3, 3, nil}, {6, 2, nil}}}, {"", "contents", 3, []protocol.BlockInfo{{0, 3, nil}, {3, 3, nil}, {6, 2, nil}}},
{"con", "contents", 3, []Block{{3, 3, nil}, {6, 2, nil}}}, {"con", "contents", 3, []protocol.BlockInfo{{3, 3, nil}, {6, 2, nil}}},
{"contents", "con", 3, nil}, {"contents", "con", 3, nil},
{"contents", "cont", 3, []Block{{3, 1, nil}}}, {"contents", "cont", 3, []protocol.BlockInfo{{3, 1, nil}}},
{"cont", "contents", 3, []Block{{3, 3, nil}, {6, 2, nil}}}, {"cont", "contents", 3, []protocol.BlockInfo{{3, 3, nil}, {6, 2, nil}}},
} }
func TestDiff(t *testing.T) { func TestDiff(t *testing.T) {

49
scanner/blocks_xdr.go
View File

@ -1,49 +0,0 @@
package scanner
import (
"bytes"
"io"
"github.com/calmh/syncthing/xdr"
)
func (o Block) EncodeXDR(w io.Writer) (int, error) {
var xw = xdr.NewWriter(w)
return o.encodeXDR(xw)
}
func (o Block) MarshalXDR() []byte {
return o.AppendXDR(make([]byte, 0, 128))
}
func (o Block) AppendXDR(bs []byte) []byte {
var aw = xdr.AppendWriter(bs)
var xw = xdr.NewWriter(&aw)
o.encodeXDR(xw)
return []byte(aw)
}
func (o Block) encodeXDR(xw *xdr.Writer) (int, error) {
xw.WriteUint64(uint64(o.Offset))
xw.WriteUint32(o.Size)
xw.WriteBytes(o.Hash)
return xw.Tot(), xw.Error()
}
func (o *Block) DecodeXDR(r io.Reader) error {
xr := xdr.NewReader(r)
return o.decodeXDR(xr)
}
func (o *Block) UnmarshalXDR(bs []byte) error {
var br = bytes.NewReader(bs)
var xr = xdr.NewReader(br)
return o.decodeXDR(xr)
}
func (o *Block) decodeXDR(xr *xdr.Reader) error {
o.Offset = int64(xr.ReadUint64())
o.Size = xr.ReadUint32()
o.Hash = xr.ReadBytes()
return xr.Error()
}

30
scanner/file.go
View File

@ -1,30 +0,0 @@
// Copyright (C) 2014 Jakob Borg and other contributors. All rights reserved.
// Use of this source code is governed by an MIT-style license that can be
// found in the LICENSE file.
package scanner
import "fmt"
type File struct {
Name string
Flags uint32
Modified int64
Version uint64
Size int64
Blocks []Block
Suppressed bool
}
func (f File) String() string {
return fmt.Sprintf("File{Name:%q, Flags:0%o, Modified:%d, Version:%d, Size:%d, Blocks:%v, Sup:%v}",
f.Name, f.Flags, f.Modified, f.Version, f.Size, f.Blocks, f.Suppressed)
}
func (f File) Equals(o File) bool {
return f.Modified == o.Modified && f.Version == o.Version
}
func (f File) NewerThan(o File) bool {
return f.Modified > o.Modified || (f.Modified == o.Modified && f.Version > o.Version)
}

64
scanner/file_xdr.go
View File

@ -1,64 +0,0 @@
package scanner
import (
"bytes"
"io"
"github.com/calmh/syncthing/xdr"
)
func (o File) EncodeXDR(w io.Writer) (int, error) {
var xw = xdr.NewWriter(w)
return o.encodeXDR(xw)
}
func (o File) MarshalXDR() []byte {
return o.AppendXDR(make([]byte, 0, 128))
}
func (o File) AppendXDR(bs []byte) []byte {
var aw = xdr.AppendWriter(bs)
var xw = xdr.NewWriter(&aw)
o.encodeXDR(xw)
return []byte(aw)
}
func (o File) encodeXDR(xw *xdr.Writer) (int, error) {
xw.WriteString(o.Name)
xw.WriteUint32(o.Flags)
xw.WriteUint64(uint64(o.Modified))
xw.WriteUint64(o.Version)
xw.WriteUint64(uint64(o.Size))
xw.WriteUint32(uint32(len(o.Blocks)))
for i := range o.Blocks {
o.Blocks[i].encodeXDR(xw)
}
xw.WriteBool(o.Suppressed)
return xw.Tot(), xw.Error()
}
func (o *File) DecodeXDR(r io.Reader) error {
xr := xdr.NewReader(r)
return o.decodeXDR(xr)
}
func (o *File) UnmarshalXDR(bs []byte) error {
var br = bytes.NewReader(bs)
var xr = xdr.NewReader(br)
return o.decodeXDR(xr)
}
func (o *File) decodeXDR(xr *xdr.Reader) error {
o.Name = xr.ReadString()
o.Flags = xr.ReadUint32()
o.Modified = int64(xr.ReadUint64())
o.Version = xr.ReadUint64()
o.Size = int64(xr.ReadUint64())
_BlocksSize := int(xr.ReadUint32())
o.Blocks = make([]Block, _BlocksSize)
for i := range o.Blocks {
(&o.Blocks[i]).decodeXDR(xr)
}
o.Suppressed = xr.ReadBool()
return xr.Error()
}

15
scanner/walk.go
View File

@ -55,12 +55,12 @@ type Suppressor interface {
type CurrentFiler interface { type CurrentFiler interface {
// CurrentFile returns the file as seen at last scan. // CurrentFile returns the file as seen at last scan.
CurrentFile(name string) File CurrentFile(name string) protocol.FileInfo
} }
// Walk returns the list of files found in the local repository by scanning the // Walk returns the list of files found in the local repository by scanning the
// file system. Files are blockwise hashed. // file system. Files are blockwise hashed.
func (w *Walker) Walk() (files []File, ignore map[string][]string, err error) { func (w *Walker) Walk() (files []protocol.FileInfo, ignore map[string][]string, err error) {
if debug { if debug {
l.Debugln("Walk", w.Dir, w.BlockSize, w.IgnoreFile) l.Debugln("Walk", w.Dir, w.BlockSize, w.IgnoreFile)
} }
@ -122,7 +122,7 @@ func (w *Walker) loadIgnoreFiles(dir string, ign map[string][]string) filepath.W
} }
} }
func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath.WalkFunc { func (w *Walker) walkAndHashFiles(res *[]protocol.FileInfo, ign map[string][]string) filepath.WalkFunc {
return func(p string, info os.FileInfo, err error) error { return func(p string, info os.FileInfo, err error) error {
if err != nil { if err != nil {
if debug { if debug {
@ -183,7 +183,7 @@ func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath
} else { } else {
flags |= uint32(info.Mode() & os.ModePerm) flags |= uint32(info.Mode() & os.ModePerm)
} }
f := File{ f := protocol.FileInfo{
Name: rn, Name: rn,
Version: lamport.Default.Tick(0), Version: lamport.Default.Tick(0),
Flags: flags, Flags: flags,
@ -213,8 +213,8 @@ func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath
if w.Suppressor != nil { if w.Suppressor != nil {
if cur, prev := w.Suppressor.Suppress(rn, info); cur && !prev { if cur, prev := w.Suppressor.Suppress(rn, info); cur && !prev {
l.Infof("Changes to %q are being temporarily suppressed because it changes too frequently.", p) l.Infof("Changes to %q are being temporarily suppressed because it changes too frequently.", p)
cf.Suppressed = true cf.Flags |= protocol.FlagInvalid
cf.Version++ cf.Version = lamport.Default.Tick(cf.Version)
if debug { if debug {
l.Debugln("suppressed:", cf) l.Debugln("suppressed:", cf)
} }
@ -256,10 +256,9 @@ func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath
if w.IgnorePerms { if w.IgnorePerms {
flags = protocol.FlagNoPermBits | 0666 flags = protocol.FlagNoPermBits | 0666
} }
f := File{ f := protocol.FileInfo{
Name: rn, Name: rn,
Version: lamport.Default.Tick(0), Version: lamport.Default.Tick(0),
Size: info.Size(),
Flags: flags, Flags: flags,
Modified: info.ModTime().Unix(), Modified: info.ModTime().Unix(),
Blocks: blocks, Blocks: blocks,

3
xdr/cmd/genxdr/main.go
View File

@ -178,6 +178,9 @@ func handleStruct(name string, t *ast.StructType) {
if m := maxRe.FindStringSubmatch(c); m != nil { if m := maxRe.FindStringSubmatch(c); m != nil {
max, _ = strconv.Atoi(m[1]) max, _ = strconv.Atoi(m[1])
} }
if strings.Contains(c, "noencode") {
continue
}
} }
var f field var f field