syncthing/lib/db/set_test.go

1902 lines
59 KiB
Go
Raw Permalink Normal View History

2014-11-16 20:13:20 +00:00
// Copyright (C) 2014 The Syncthing Authors.
2014-09-29 19:43:32 +00:00
//
2015-03-07 20:36:35 +00:00
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
2014-06-01 20:50:14 +00:00
package db_test
import (
"bytes"
"fmt"
2016-01-25 09:04:53 +00:00
"os"
"path/filepath"
"sort"
"testing"
"time"
2016-03-06 20:32:10 +00:00
"github.com/d4l3k/messagediff"
refactor: use modern Protobuf encoder (#9817) At a high level, this is what I've done and why: - I'm moving the protobuf generation for the `protocol`, `discovery` and `db` packages to the modern alternatives, and using `buf` to generate because it's nice and simple. - After trying various approaches on how to integrate the new types with the existing code, I opted for splitting off our own data model types from the on-the-wire generated types. This means we can have a `FileInfo` type with nicer ergonomics and lots of methods, while the protobuf generated type stays clean and close to the wire protocol. It does mean copying between the two when required, which certainly adds a small amount of inefficiency. If we want to walk this back in the future and use the raw generated type throughout, that's possible, this however makes the refactor smaller (!) as it doesn't change everything about the type for everyone at the same time. - I have simply removed in cold blood a significant number of old database migrations. These depended on previous generations of generated messages of various kinds and were annoying to support in the new fashion. The oldest supported database version now is the one from Syncthing 1.9.0 from Sep 7, 2020. - I changed config structs to be regular manually defined structs. For the sake of discussion, some things I tried that turned out not to work... ### Embedding / wrapping Embedding the protobuf generated structs in our existing types as a data container and keeping our methods and stuff: ``` package protocol type FileInfo struct { *generated.FileInfo } ``` This generates a lot of problems because the internal shape of the generated struct is quite different (different names, different types, more pointers), because initializing it doesn't work like you'd expect (i.e., you end up with an embedded nil pointer and a panic), and because the types of child types don't get wrapped. That is, even if we also have a similar wrapper around a `Vector`, that's not the type you get when accessing `someFileInfo.Version`, you get the `*generated.Vector` that doesn't have methods, etc. ### Aliasing ``` package protocol type FileInfo = generated.FileInfo ``` Doesn't help because you can't attach methods to it, plus all the above. ### Generating the types into the target package like we do now and attaching methods This fails because of the different shape of the generated type (as in the embedding case above) plus the generated struct already has a bunch of methods that we can't necessarily override properly (like `String()` and a bunch of getters). ### Methods to functions I considered just moving all the methods we attach to functions in a specific package, so that for example ``` package protocol func (f FileInfo) Equal(other FileInfo) bool ``` would become ``` package fileinfos func Equal(a, b *generated.FileInfo) bool ``` and this would mostly work, but becomes quite verbose and cumbersome, and somewhat limits discoverability (you can't see what methods are available on the type in auto completions, etc). In the end I did this in some cases, like in the database layer where a lot of things like `func (fv *FileVersion) IsEmpty() bool` becomes `func fvIsEmpty(fv *generated.FileVersion)` because they were anyway just internal methods. Fixes #8247
2024-12-01 15:50:17 +00:00
2015-08-06 09:29:25 +00:00
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/db/backend"
"github.com/syncthing/syncthing/lib/events"
2015-09-22 17:38:46 +00:00
"github.com/syncthing/syncthing/lib/protocol"
)
var remoteDevice0, remoteDevice1 protocol.DeviceID
2014-07-06 12:46:48 +00:00
func init() {
remoteDevice0, _ = protocol.DeviceIDFromString("AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR")
remoteDevice1, _ = protocol.DeviceIDFromString("I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU")
2014-07-06 12:46:48 +00:00
}
2015-03-25 21:37:23 +00:00
const myID = 1
func genBlocks(n int) []protocol.BlockInfo {
b := make([]protocol.BlockInfo, n)
for i := range b {
h := make([]byte, 32)
for j := range h {
h[j] = byte(i + j)
}
b[i].Size = i
b[i].Hash = h
}
return b
}
func globalList(t testing.TB, s *db.FileSet) []protocol.FileInfo {
2014-07-12 21:06:48 +00:00
var fs []protocol.FileInfo
snap := snapshot(t, s)
defer snap.Release()
refactor: use modern Protobuf encoder (#9817) At a high level, this is what I've done and why: - I'm moving the protobuf generation for the `protocol`, `discovery` and `db` packages to the modern alternatives, and using `buf` to generate because it's nice and simple. - After trying various approaches on how to integrate the new types with the existing code, I opted for splitting off our own data model types from the on-the-wire generated types. This means we can have a `FileInfo` type with nicer ergonomics and lots of methods, while the protobuf generated type stays clean and close to the wire protocol. It does mean copying between the two when required, which certainly adds a small amount of inefficiency. If we want to walk this back in the future and use the raw generated type throughout, that's possible, this however makes the refactor smaller (!) as it doesn't change everything about the type for everyone at the same time. - I have simply removed in cold blood a significant number of old database migrations. These depended on previous generations of generated messages of various kinds and were annoying to support in the new fashion. The oldest supported database version now is the one from Syncthing 1.9.0 from Sep 7, 2020. - I changed config structs to be regular manually defined structs. For the sake of discussion, some things I tried that turned out not to work... ### Embedding / wrapping Embedding the protobuf generated structs in our existing types as a data container and keeping our methods and stuff: ``` package protocol type FileInfo struct { *generated.FileInfo } ``` This generates a lot of problems because the internal shape of the generated struct is quite different (different names, different types, more pointers), because initializing it doesn't work like you'd expect (i.e., you end up with an embedded nil pointer and a panic), and because the types of child types don't get wrapped. That is, even if we also have a similar wrapper around a `Vector`, that's not the type you get when accessing `someFileInfo.Version`, you get the `*generated.Vector` that doesn't have methods, etc. ### Aliasing ``` package protocol type FileInfo = generated.FileInfo ``` Doesn't help because you can't attach methods to it, plus all the above. ### Generating the types into the target package like we do now and attaching methods This fails because of the different shape of the generated type (as in the embedding case above) plus the generated struct already has a bunch of methods that we can't necessarily override properly (like `String()` and a bunch of getters). ### Methods to functions I considered just moving all the methods we attach to functions in a specific package, so that for example ``` package protocol func (f FileInfo) Equal(other FileInfo) bool ``` would become ``` package fileinfos func Equal(a, b *generated.FileInfo) bool ``` and this would mostly work, but becomes quite verbose and cumbersome, and somewhat limits discoverability (you can't see what methods are available on the type in auto completions, etc). In the end I did this in some cases, like in the database layer where a lot of things like `func (fv *FileVersion) IsEmpty() bool` becomes `func fvIsEmpty(fv *generated.FileVersion)` because they were anyway just internal methods. Fixes #8247
2024-12-01 15:50:17 +00:00
snap.WithGlobal(func(fi protocol.FileInfo) bool {
fs = append(fs, fi)
2014-07-06 12:46:48 +00:00
return true
})
return fs
}
refactor: use modern Protobuf encoder (#9817) At a high level, this is what I've done and why: - I'm moving the protobuf generation for the `protocol`, `discovery` and `db` packages to the modern alternatives, and using `buf` to generate because it's nice and simple. - After trying various approaches on how to integrate the new types with the existing code, I opted for splitting off our own data model types from the on-the-wire generated types. This means we can have a `FileInfo` type with nicer ergonomics and lots of methods, while the protobuf generated type stays clean and close to the wire protocol. It does mean copying between the two when required, which certainly adds a small amount of inefficiency. If we want to walk this back in the future and use the raw generated type throughout, that's possible, this however makes the refactor smaller (!) as it doesn't change everything about the type for everyone at the same time. - I have simply removed in cold blood a significant number of old database migrations. These depended on previous generations of generated messages of various kinds and were annoying to support in the new fashion. The oldest supported database version now is the one from Syncthing 1.9.0 from Sep 7, 2020. - I changed config structs to be regular manually defined structs. For the sake of discussion, some things I tried that turned out not to work... ### Embedding / wrapping Embedding the protobuf generated structs in our existing types as a data container and keeping our methods and stuff: ``` package protocol type FileInfo struct { *generated.FileInfo } ``` This generates a lot of problems because the internal shape of the generated struct is quite different (different names, different types, more pointers), because initializing it doesn't work like you'd expect (i.e., you end up with an embedded nil pointer and a panic), and because the types of child types don't get wrapped. That is, even if we also have a similar wrapper around a `Vector`, that's not the type you get when accessing `someFileInfo.Version`, you get the `*generated.Vector` that doesn't have methods, etc. ### Aliasing ``` package protocol type FileInfo = generated.FileInfo ``` Doesn't help because you can't attach methods to it, plus all the above. ### Generating the types into the target package like we do now and attaching methods This fails because of the different shape of the generated type (as in the embedding case above) plus the generated struct already has a bunch of methods that we can't necessarily override properly (like `String()` and a bunch of getters). ### Methods to functions I considered just moving all the methods we attach to functions in a specific package, so that for example ``` package protocol func (f FileInfo) Equal(other FileInfo) bool ``` would become ``` package fileinfos func Equal(a, b *generated.FileInfo) bool ``` and this would mostly work, but becomes quite verbose and cumbersome, and somewhat limits discoverability (you can't see what methods are available on the type in auto completions, etc). In the end I did this in some cases, like in the database layer where a lot of things like `func (fv *FileVersion) IsEmpty() bool` becomes `func fvIsEmpty(fv *generated.FileVersion)` because they were anyway just internal methods. Fixes #8247
2024-12-01 15:50:17 +00:00
func globalListPrefixed(t testing.TB, s *db.FileSet, prefix string) []protocol.FileInfo {
var fs []protocol.FileInfo
snap := snapshot(t, s)
defer snap.Release()
refactor: use modern Protobuf encoder (#9817) At a high level, this is what I've done and why: - I'm moving the protobuf generation for the `protocol`, `discovery` and `db` packages to the modern alternatives, and using `buf` to generate because it's nice and simple. - After trying various approaches on how to integrate the new types with the existing code, I opted for splitting off our own data model types from the on-the-wire generated types. This means we can have a `FileInfo` type with nicer ergonomics and lots of methods, while the protobuf generated type stays clean and close to the wire protocol. It does mean copying between the two when required, which certainly adds a small amount of inefficiency. If we want to walk this back in the future and use the raw generated type throughout, that's possible, this however makes the refactor smaller (!) as it doesn't change everything about the type for everyone at the same time. - I have simply removed in cold blood a significant number of old database migrations. These depended on previous generations of generated messages of various kinds and were annoying to support in the new fashion. The oldest supported database version now is the one from Syncthing 1.9.0 from Sep 7, 2020. - I changed config structs to be regular manually defined structs. For the sake of discussion, some things I tried that turned out not to work... ### Embedding / wrapping Embedding the protobuf generated structs in our existing types as a data container and keeping our methods and stuff: ``` package protocol type FileInfo struct { *generated.FileInfo } ``` This generates a lot of problems because the internal shape of the generated struct is quite different (different names, different types, more pointers), because initializing it doesn't work like you'd expect (i.e., you end up with an embedded nil pointer and a panic), and because the types of child types don't get wrapped. That is, even if we also have a similar wrapper around a `Vector`, that's not the type you get when accessing `someFileInfo.Version`, you get the `*generated.Vector` that doesn't have methods, etc. ### Aliasing ``` package protocol type FileInfo = generated.FileInfo ``` Doesn't help because you can't attach methods to it, plus all the above. ### Generating the types into the target package like we do now and attaching methods This fails because of the different shape of the generated type (as in the embedding case above) plus the generated struct already has a bunch of methods that we can't necessarily override properly (like `String()` and a bunch of getters). ### Methods to functions I considered just moving all the methods we attach to functions in a specific package, so that for example ``` package protocol func (f FileInfo) Equal(other FileInfo) bool ``` would become ``` package fileinfos func Equal(a, b *generated.FileInfo) bool ``` and this would mostly work, but becomes quite verbose and cumbersome, and somewhat limits discoverability (you can't see what methods are available on the type in auto completions, etc). In the end I did this in some cases, like in the database layer where a lot of things like `func (fv *FileVersion) IsEmpty() bool` becomes `func fvIsEmpty(fv *generated.FileVersion)` because they were anyway just internal methods. Fixes #8247
2024-12-01 15:50:17 +00:00
snap.WithPrefixedGlobalTruncated(prefix, func(fi protocol.FileInfo) bool {
fs = append(fs, fi)
return true
})
return fs
}
2014-07-06 12:46:48 +00:00
func haveList(t testing.TB, s *db.FileSet, n protocol.DeviceID) []protocol.FileInfo {
2014-07-12 21:06:48 +00:00
var fs []protocol.FileInfo
snap := snapshot(t, s)
defer snap.Release()
refactor: use modern Protobuf encoder (#9817) At a high level, this is what I've done and why: - I'm moving the protobuf generation for the `protocol`, `discovery` and `db` packages to the modern alternatives, and using `buf` to generate because it's nice and simple. - After trying various approaches on how to integrate the new types with the existing code, I opted for splitting off our own data model types from the on-the-wire generated types. This means we can have a `FileInfo` type with nicer ergonomics and lots of methods, while the protobuf generated type stays clean and close to the wire protocol. It does mean copying between the two when required, which certainly adds a small amount of inefficiency. If we want to walk this back in the future and use the raw generated type throughout, that's possible, this however makes the refactor smaller (!) as it doesn't change everything about the type for everyone at the same time. - I have simply removed in cold blood a significant number of old database migrations. These depended on previous generations of generated messages of various kinds and were annoying to support in the new fashion. The oldest supported database version now is the one from Syncthing 1.9.0 from Sep 7, 2020. - I changed config structs to be regular manually defined structs. For the sake of discussion, some things I tried that turned out not to work... ### Embedding / wrapping Embedding the protobuf generated structs in our existing types as a data container and keeping our methods and stuff: ``` package protocol type FileInfo struct { *generated.FileInfo } ``` This generates a lot of problems because the internal shape of the generated struct is quite different (different names, different types, more pointers), because initializing it doesn't work like you'd expect (i.e., you end up with an embedded nil pointer and a panic), and because the types of child types don't get wrapped. That is, even if we also have a similar wrapper around a `Vector`, that's not the type you get when accessing `someFileInfo.Version`, you get the `*generated.Vector` that doesn't have methods, etc. ### Aliasing ``` package protocol type FileInfo = generated.FileInfo ``` Doesn't help because you can't attach methods to it, plus all the above. ### Generating the types into the target package like we do now and attaching methods This fails because of the different shape of the generated type (as in the embedding case above) plus the generated struct already has a bunch of methods that we can't necessarily override properly (like `String()` and a bunch of getters). ### Methods to functions I considered just moving all the methods we attach to functions in a specific package, so that for example ``` package protocol func (f FileInfo) Equal(other FileInfo) bool ``` would become ``` package fileinfos func Equal(a, b *generated.FileInfo) bool ``` and this would mostly work, but becomes quite verbose and cumbersome, and somewhat limits discoverability (you can't see what methods are available on the type in auto completions, etc). In the end I did this in some cases, like in the database layer where a lot of things like `func (fv *FileVersion) IsEmpty() bool` becomes `func fvIsEmpty(fv *generated.FileVersion)` because they were anyway just internal methods. Fixes #8247
2024-12-01 15:50:17 +00:00
snap.WithHave(n, func(fi protocol.FileInfo) bool {
fs = append(fs, fi)
2014-07-06 12:46:48 +00:00
return true
})
return fs
}
refactor: use modern Protobuf encoder (#9817) At a high level, this is what I've done and why: - I'm moving the protobuf generation for the `protocol`, `discovery` and `db` packages to the modern alternatives, and using `buf` to generate because it's nice and simple. - After trying various approaches on how to integrate the new types with the existing code, I opted for splitting off our own data model types from the on-the-wire generated types. This means we can have a `FileInfo` type with nicer ergonomics and lots of methods, while the protobuf generated type stays clean and close to the wire protocol. It does mean copying between the two when required, which certainly adds a small amount of inefficiency. If we want to walk this back in the future and use the raw generated type throughout, that's possible, this however makes the refactor smaller (!) as it doesn't change everything about the type for everyone at the same time. - I have simply removed in cold blood a significant number of old database migrations. These depended on previous generations of generated messages of various kinds and were annoying to support in the new fashion. The oldest supported database version now is the one from Syncthing 1.9.0 from Sep 7, 2020. - I changed config structs to be regular manually defined structs. For the sake of discussion, some things I tried that turned out not to work... ### Embedding / wrapping Embedding the protobuf generated structs in our existing types as a data container and keeping our methods and stuff: ``` package protocol type FileInfo struct { *generated.FileInfo } ``` This generates a lot of problems because the internal shape of the generated struct is quite different (different names, different types, more pointers), because initializing it doesn't work like you'd expect (i.e., you end up with an embedded nil pointer and a panic), and because the types of child types don't get wrapped. That is, even if we also have a similar wrapper around a `Vector`, that's not the type you get when accessing `someFileInfo.Version`, you get the `*generated.Vector` that doesn't have methods, etc. ### Aliasing ``` package protocol type FileInfo = generated.FileInfo ``` Doesn't help because you can't attach methods to it, plus all the above. ### Generating the types into the target package like we do now and attaching methods This fails because of the different shape of the generated type (as in the embedding case above) plus the generated struct already has a bunch of methods that we can't necessarily override properly (like `String()` and a bunch of getters). ### Methods to functions I considered just moving all the methods we attach to functions in a specific package, so that for example ``` package protocol func (f FileInfo) Equal(other FileInfo) bool ``` would become ``` package fileinfos func Equal(a, b *generated.FileInfo) bool ``` and this would mostly work, but becomes quite verbose and cumbersome, and somewhat limits discoverability (you can't see what methods are available on the type in auto completions, etc). In the end I did this in some cases, like in the database layer where a lot of things like `func (fv *FileVersion) IsEmpty() bool` becomes `func fvIsEmpty(fv *generated.FileVersion)` because they were anyway just internal methods. Fixes #8247
2024-12-01 15:50:17 +00:00
func haveListPrefixed(t testing.TB, s *db.FileSet, n protocol.DeviceID, prefix string) []protocol.FileInfo {
var fs []protocol.FileInfo
snap := snapshot(t, s)
defer snap.Release()
refactor: use modern Protobuf encoder (#9817) At a high level, this is what I've done and why: - I'm moving the protobuf generation for the `protocol`, `discovery` and `db` packages to the modern alternatives, and using `buf` to generate because it's nice and simple. - After trying various approaches on how to integrate the new types with the existing code, I opted for splitting off our own data model types from the on-the-wire generated types. This means we can have a `FileInfo` type with nicer ergonomics and lots of methods, while the protobuf generated type stays clean and close to the wire protocol. It does mean copying between the two when required, which certainly adds a small amount of inefficiency. If we want to walk this back in the future and use the raw generated type throughout, that's possible, this however makes the refactor smaller (!) as it doesn't change everything about the type for everyone at the same time. - I have simply removed in cold blood a significant number of old database migrations. These depended on previous generations of generated messages of various kinds and were annoying to support in the new fashion. The oldest supported database version now is the one from Syncthing 1.9.0 from Sep 7, 2020. - I changed config structs to be regular manually defined structs. For the sake of discussion, some things I tried that turned out not to work... ### Embedding / wrapping Embedding the protobuf generated structs in our existing types as a data container and keeping our methods and stuff: ``` package protocol type FileInfo struct { *generated.FileInfo } ``` This generates a lot of problems because the internal shape of the generated struct is quite different (different names, different types, more pointers), because initializing it doesn't work like you'd expect (i.e., you end up with an embedded nil pointer and a panic), and because the types of child types don't get wrapped. That is, even if we also have a similar wrapper around a `Vector`, that's not the type you get when accessing `someFileInfo.Version`, you get the `*generated.Vector` that doesn't have methods, etc. ### Aliasing ``` package protocol type FileInfo = generated.FileInfo ``` Doesn't help because you can't attach methods to it, plus all the above. ### Generating the types into the target package like we do now and attaching methods This fails because of the different shape of the generated type (as in the embedding case above) plus the generated struct already has a bunch of methods that we can't necessarily override properly (like `String()` and a bunch of getters). ### Methods to functions I considered just moving all the methods we attach to functions in a specific package, so that for example ``` package protocol func (f FileInfo) Equal(other FileInfo) bool ``` would become ``` package fileinfos func Equal(a, b *generated.FileInfo) bool ``` and this would mostly work, but becomes quite verbose and cumbersome, and somewhat limits discoverability (you can't see what methods are available on the type in auto completions, etc). In the end I did this in some cases, like in the database layer where a lot of things like `func (fv *FileVersion) IsEmpty() bool` becomes `func fvIsEmpty(fv *generated.FileVersion)` because they were anyway just internal methods. Fixes #8247
2024-12-01 15:50:17 +00:00
snap.WithPrefixedHaveTruncated(n, prefix, func(fi protocol.FileInfo) bool {
fs = append(fs, fi)
return true
})
return fs
}
func needList(t testing.TB, s *db.FileSet, n protocol.DeviceID) []protocol.FileInfo {
2014-07-12 21:06:48 +00:00
var fs []protocol.FileInfo
snap := snapshot(t, s)
defer snap.Release()
refactor: use modern Protobuf encoder (#9817) At a high level, this is what I've done and why: - I'm moving the protobuf generation for the `protocol`, `discovery` and `db` packages to the modern alternatives, and using `buf` to generate because it's nice and simple. - After trying various approaches on how to integrate the new types with the existing code, I opted for splitting off our own data model types from the on-the-wire generated types. This means we can have a `FileInfo` type with nicer ergonomics and lots of methods, while the protobuf generated type stays clean and close to the wire protocol. It does mean copying between the two when required, which certainly adds a small amount of inefficiency. If we want to walk this back in the future and use the raw generated type throughout, that's possible, this however makes the refactor smaller (!) as it doesn't change everything about the type for everyone at the same time. - I have simply removed in cold blood a significant number of old database migrations. These depended on previous generations of generated messages of various kinds and were annoying to support in the new fashion. The oldest supported database version now is the one from Syncthing 1.9.0 from Sep 7, 2020. - I changed config structs to be regular manually defined structs. For the sake of discussion, some things I tried that turned out not to work... ### Embedding / wrapping Embedding the protobuf generated structs in our existing types as a data container and keeping our methods and stuff: ``` package protocol type FileInfo struct { *generated.FileInfo } ``` This generates a lot of problems because the internal shape of the generated struct is quite different (different names, different types, more pointers), because initializing it doesn't work like you'd expect (i.e., you end up with an embedded nil pointer and a panic), and because the types of child types don't get wrapped. That is, even if we also have a similar wrapper around a `Vector`, that's not the type you get when accessing `someFileInfo.Version`, you get the `*generated.Vector` that doesn't have methods, etc. ### Aliasing ``` package protocol type FileInfo = generated.FileInfo ``` Doesn't help because you can't attach methods to it, plus all the above. ### Generating the types into the target package like we do now and attaching methods This fails because of the different shape of the generated type (as in the embedding case above) plus the generated struct already has a bunch of methods that we can't necessarily override properly (like `String()` and a bunch of getters). ### Methods to functions I considered just moving all the methods we attach to functions in a specific package, so that for example ``` package protocol func (f FileInfo) Equal(other FileInfo) bool ``` would become ``` package fileinfos func Equal(a, b *generated.FileInfo) bool ``` and this would mostly work, but becomes quite verbose and cumbersome, and somewhat limits discoverability (you can't see what methods are available on the type in auto completions, etc). In the end I did this in some cases, like in the database layer where a lot of things like `func (fv *FileVersion) IsEmpty() bool` becomes `func fvIsEmpty(fv *generated.FileVersion)` because they were anyway just internal methods. Fixes #8247
2024-12-01 15:50:17 +00:00
snap.WithNeed(n, func(fi protocol.FileInfo) bool {
fs = append(fs, fi)
2014-07-06 12:46:48 +00:00
return true
})
return fs
}
2014-07-12 21:06:48 +00:00
type fileList []protocol.FileInfo
func (l fileList) Len() int {
return len(l)
}
func (l fileList) Less(a, b int) bool {
return l[a].Name < l[b].Name
}
func (l fileList) Swap(a, b int) {
l[a], l[b] = l[b], l[a]
}
func (l fileList) String() string {
var b bytes.Buffer
b.WriteString("[]protocol.FileList{\n")
for _, f := range l {
fmt.Fprintf(&b, " %q: #%v, %d bytes, %d blocks, perms=%o\n", f.Name, f.Version, f.Size, len(f.Blocks), f.Permissions)
}
b.WriteString("}")
return b.String()
}
func setSequence(seq int64, files fileList) int64 {
for i := range files {
seq++
files[i].Sequence = seq
}
return seq
}
func setBlocksHash(files fileList) {
for i, f := range files {
files[i].BlocksHash = protocol.BlocksHash(f.Blocks)
}
}
func TestGlobalSet(t *testing.T) {
ldb := newLowlevelMemory(t)
defer ldb.Close()
2014-07-06 12:46:48 +00:00
m := newFileSet(t, "test", ldb)
2014-07-06 12:46:48 +00:00
local0 := fileList{
protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(3)},
protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "z", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(8)},
2014-07-06 12:46:48 +00:00
}
localSeq := setSequence(0, local0)
setBlocksHash(local0)
local1 := fileList{
protocol.FileInfo{Name: "a", Sequence: 6, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
protocol.FileInfo{Name: "b", Sequence: 7, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "c", Sequence: 8, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(3)},
protocol.FileInfo{Name: "d", Sequence: 9, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "z", Sequence: 10, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Deleted: true},
2014-07-06 12:46:48 +00:00
}
setSequence(localSeq, local1)
setBlocksHash(local1)
localTot := fileList{
local1[0],
local1[1],
local1[2],
local1[3],
protocol.FileInfo{Name: "z", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Deleted: true},
}
remote0 := fileList{
protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(5)},
}
remoteSeq := setSequence(0, remote0)
setBlocksHash(remote0)
remote1 := fileList{
protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(6)},
protocol.FileInfo{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(7)},
}
setSequence(remoteSeq, remote1)
setBlocksHash(remote1)
remoteTot := fileList{
2014-07-06 12:46:48 +00:00
remote0[0],
remote1[0],
remote0[2],
remote1[1],
}
expectedGlobal := fileList{
remote0[0], // a
remote1[0], // b
remote0[2], // c
localTot[3], // d
remote1[1], // e
localTot[4], // z
}
expectedLocalNeed := fileList{
2014-07-06 12:46:48 +00:00
remote1[0],
remote0[2],
remote1[1],
}
expectedRemoteNeed := fileList{
2014-07-06 12:46:48 +00:00
local0[3],
}
replace(m, protocol.LocalDeviceID, local0)
replace(m, protocol.LocalDeviceID, local1)
replace(m, remoteDevice0, remote0)
m.Update(remoteDevice0, remote1)
check := func() {
t.Helper()
g := fileList(globalList(t, m))
sort.Sort(g)
if fmt.Sprint(g) != fmt.Sprint(expectedGlobal) {
t.Errorf("Global incorrect;\n A: %v !=\n E: %v", g, expectedGlobal)
}
var globalFiles, globalDirectories, globalDeleted int
var globalBytes int64
for _, f := range g {
if f.IsInvalid() {
continue
}
switch {
case f.IsDeleted():
globalDeleted++
case f.IsDirectory():
globalDirectories++
default:
globalFiles++
}
globalBytes += f.FileSize()
}
gs := globalSize(t, m)
if gs.Files != globalFiles {
t.Errorf("Incorrect GlobalSize files; %d != %d", gs.Files, globalFiles)
}
if gs.Directories != globalDirectories {
t.Errorf("Incorrect GlobalSize directories; %d != %d", gs.Directories, globalDirectories)
}
if gs.Deleted != globalDeleted {
t.Errorf("Incorrect GlobalSize deleted; %d != %d", gs.Deleted, globalDeleted)
}
if gs.Bytes != globalBytes {
t.Errorf("Incorrect GlobalSize bytes; %d != %d", gs.Bytes, globalBytes)
}
h := fileList(haveList(t, m, protocol.LocalDeviceID))
sort.Sort(h)
if fmt.Sprint(h) != fmt.Sprint(localTot) {
t.Errorf("Have incorrect (local);\n A: %v !=\n E: %v", h, localTot)
}
var haveFiles, haveDirectories, haveDeleted int
var haveBytes int64
for _, f := range h {
if f.IsInvalid() {
continue
}
switch {
case f.IsDeleted():
haveDeleted++
case f.IsDirectory():
haveDirectories++
default:
haveFiles++
}
haveBytes += f.FileSize()
}
ls := localSize(t, m)
if ls.Files != haveFiles {
t.Errorf("Incorrect LocalSize files; %d != %d", ls.Files, haveFiles)
}
if ls.Directories != haveDirectories {
t.Errorf("Incorrect LocalSize directories; %d != %d", ls.Directories, haveDirectories)
}
if ls.Deleted != haveDeleted {
t.Errorf("Incorrect LocalSize deleted; %d != %d", ls.Deleted, haveDeleted)
}
if ls.Bytes != haveBytes {
t.Errorf("Incorrect LocalSize bytes; %d != %d", ls.Bytes, haveBytes)
}
h = fileList(haveList(t, m, remoteDevice0))
sort.Sort(h)
if fmt.Sprint(h) != fmt.Sprint(remoteTot) {
t.Errorf("Have incorrect (remote);\n A: %v !=\n E: %v", h, remoteTot)
}
n := fileList(needList(t, m, protocol.LocalDeviceID))
sort.Sort(n)
if fmt.Sprint(n) != fmt.Sprint(expectedLocalNeed) {
t.Errorf("Need incorrect (local);\n A: %v !=\n E: %v", n, expectedLocalNeed)
}
checkNeed(t, m, protocol.LocalDeviceID, expectedLocalNeed)
n = fileList(needList(t, m, remoteDevice0))
sort.Sort(n)
if fmt.Sprint(n) != fmt.Sprint(expectedRemoteNeed) {
t.Errorf("Need incorrect (remote);\n A: %v !=\n E: %v", n, expectedRemoteNeed)
}
checkNeed(t, m, remoteDevice0, expectedRemoteNeed)
snap := snapshot(t, m)
defer snap.Release()
f, ok := snap.Get(protocol.LocalDeviceID, "b")
if !ok {
t.Error("Unexpectedly not OK")
}
if fmt.Sprint(f) != fmt.Sprint(localTot[1]) {
t.Errorf("Get incorrect;\n A: %v !=\n E: %v", f, localTot[1])
}
f, ok = snap.Get(remoteDevice0, "b")
if !ok {
t.Error("Unexpectedly not OK")
}
if fmt.Sprint(f) != fmt.Sprint(remote1[0]) {
t.Errorf("Get incorrect (remote);\n A: %v !=\n E: %v", f, remote1[0])
}
f, ok = snap.GetGlobal("b")
if !ok {
t.Error("Unexpectedly not OK")
}
if fmt.Sprint(f) != fmt.Sprint(expectedGlobal[1]) {
t.Errorf("GetGlobal incorrect;\n A: %v !=\n E: %v", f, remote1[0])
}
f, ok = snap.Get(protocol.LocalDeviceID, "zz")
if ok {
t.Error("Unexpectedly OK")
}
if f.Name != "" {
t.Errorf("Get incorrect (local);\n A: %v !=\n E: %v", f, protocol.FileInfo{})
}
2014-07-06 12:46:48 +00:00
f, ok = snap.GetGlobal("zz")
if ok {
t.Error("Unexpectedly OK")
}
if f.Name != "" {
t.Errorf("GetGlobal incorrect;\n A: %v !=\n E: %v", f, protocol.FileInfo{})
}
}
2014-07-06 12:46:48 +00:00
check()
snap := snapshot(t, m)
av := []protocol.DeviceID{protocol.LocalDeviceID, remoteDevice0}
a := snap.Availability("a")
2014-07-06 12:46:48 +00:00
if !(len(a) == 2 && (a[0] == av[0] && a[1] == av[1] || a[0] == av[1] && a[1] == av[0])) {
t.Errorf("Availability incorrect;\n A: %v !=\n E: %v", a, av)
}
a = snap.Availability("b")
if len(a) != 1 || a[0] != remoteDevice0 {
t.Errorf("Availability incorrect;\n A: %v !=\n E: %v", a, remoteDevice0)
2014-07-06 12:46:48 +00:00
}
a = snap.Availability("d")
if len(a) != 1 || a[0] != protocol.LocalDeviceID {
t.Errorf("Availability incorrect;\n A: %v !=\n E: %v", a, protocol.LocalDeviceID)
2014-07-06 12:46:48 +00:00
}
snap.Release()
// Now bring another remote into play
secRemote := fileList{
local1[0], // a
remote1[0], // b
local1[3], // d
remote1[1], // e
local1[4], // z
}
secRemote[0].Version = secRemote[0].Version.Update(remoteDevice1.Short())
secRemote[1].Version = secRemote[1].Version.Update(remoteDevice1.Short())
secRemote[4].Version = secRemote[4].Version.Update(remoteDevice1.Short())
secRemote[4].Deleted = false
secRemote[4].Blocks = genBlocks(1)
setSequence(0, secRemote)
expectedGlobal = fileList{
secRemote[0], // a
secRemote[1], // b
remote0[2], // c
localTot[3], // d
secRemote[3], // e
secRemote[4], // z
}
expectedLocalNeed = fileList{
secRemote[0], // a
secRemote[1], // b
remote0[2], // c
secRemote[3], // e
secRemote[4], // z
}
expectedRemoteNeed = fileList{
secRemote[0], // a
secRemote[1], // b
local0[3], // d
secRemote[4], // z
}
expectedSecRemoteNeed := fileList{
remote0[2], // c
}
m.Update(remoteDevice1, secRemote)
check()
h := fileList(haveList(t, m, remoteDevice1))
sort.Sort(h)
if fmt.Sprint(h) != fmt.Sprint(secRemote) {
t.Errorf("Have incorrect (secRemote);\n A: %v !=\n E: %v", h, secRemote)
}
n := fileList(needList(t, m, remoteDevice1))
sort.Sort(n)
if fmt.Sprint(n) != fmt.Sprint(expectedSecRemoteNeed) {
t.Errorf("Need incorrect (secRemote);\n A: %v !=\n E: %v", n, expectedSecRemoteNeed)
}
checkNeed(t, m, remoteDevice1, expectedSecRemoteNeed)
}
func TestNeedWithInvalid(t *testing.T) {
ldb := newLowlevelMemory(t)
defer ldb.Close()
s := newFileSet(t, "test", ldb)
localHave := fileList{
protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
}
remote0Have := fileList{
protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(5), RawInvalid: true},
protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(7)},
}
remote1Have := fileList{
protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(7)},
protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(5), RawInvalid: true},
protocol.FileInfo{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1004}}}, Blocks: genBlocks(5), RawInvalid: true},
}
expectedNeed := fileList{
remote0Have[0],
remote1Have[0],
remote0Have[2],
}
replace(s, protocol.LocalDeviceID, localHave)
replace(s, remoteDevice0, remote0Have)
replace(s, remoteDevice1, remote1Have)
need := fileList(needList(t, s, protocol.LocalDeviceID))
sort.Sort(need)
if fmt.Sprint(need) != fmt.Sprint(expectedNeed) {
t.Errorf("Need incorrect;\n A: %v !=\n E: %v", need, expectedNeed)
}
checkNeed(t, s, protocol.LocalDeviceID, expectedNeed)
}
func TestUpdateToInvalid(t *testing.T) {
ldb := newLowlevelMemory(t)
defer ldb.Close()
folder := "test"
s := newFileSet(t, folder, ldb)
f := db.NewBlockFinder(ldb)
localHave := fileList{
protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1), Size: 1},
protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2), Size: 1},
protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(5), LocalFlags: protocol.FlagLocalIgnored, Size: 1},
protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(7), Size: 1},
protocol.FileInfo{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, LocalFlags: protocol.FlagLocalIgnored, Size: 1},
}
replace(s, protocol.LocalDeviceID, localHave)
have := fileList(haveList(t, s, protocol.LocalDeviceID))
sort.Sort(have)
if fmt.Sprint(have) != fmt.Sprint(localHave) {
t.Errorf("Have incorrect before invalidation;\n A: %v !=\n E: %v", have, localHave)
}
oldBlockHash := localHave[1].Blocks[0].Hash
localHave[1].LocalFlags = protocol.FlagLocalIgnored
localHave[1].Blocks = nil
localHave[4].LocalFlags = 0
localHave[4].Blocks = genBlocks(3)
s.Update(protocol.LocalDeviceID, append(fileList{}, localHave[1], localHave[4]))
have = fileList(haveList(t, s, protocol.LocalDeviceID))
sort.Sort(have)
if fmt.Sprint(have) != fmt.Sprint(localHave) {
t.Errorf("Have incorrect after invalidation;\n A: %v !=\n E: %v", have, localHave)
}
f.Iterate([]string{folder}, oldBlockHash, func(folder, file string, index int32) bool {
if file == localHave[1].Name {
t.Errorf("Found unexpected block in blockmap for invalidated file")
return true
}
return false
})
if !f.Iterate([]string{folder}, localHave[4].Blocks[0].Hash, func(folder, file string, index int32) bool {
return file == localHave[4].Name
}) {
t.Errorf("First block of un-invalidated file is missing from blockmap")
}
}
func TestInvalidAvailability(t *testing.T) {
ldb := newLowlevelMemory(t)
defer ldb.Close()
s := newFileSet(t, "test", ldb)
remote0Have := fileList{
protocol.FileInfo{Name: "both", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "r1only", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(5), RawInvalid: true},
protocol.FileInfo{Name: "r0only", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(7)},
protocol.FileInfo{Name: "none", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1004}}}, Blocks: genBlocks(5), RawInvalid: true},
}
remote1Have := fileList{
protocol.FileInfo{Name: "both", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "r1only", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(7)},
protocol.FileInfo{Name: "r0only", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(5), RawInvalid: true},
protocol.FileInfo{Name: "none", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1004}}}, Blocks: genBlocks(5), RawInvalid: true},
}
replace(s, remoteDevice0, remote0Have)
replace(s, remoteDevice1, remote1Have)
snap := snapshot(t, s)
defer snap.Release()
if av := snap.Availability("both"); len(av) != 2 {
t.Error("Incorrect availability for 'both':", av)
}
if av := snap.Availability("r0only"); len(av) != 1 || av[0] != remoteDevice0 {
t.Error("Incorrect availability for 'r0only':", av)
}
if av := snap.Availability("r1only"); len(av) != 1 || av[0] != remoteDevice1 {
t.Error("Incorrect availability for 'r1only':", av)
}
if av := snap.Availability("none"); len(av) != 0 {
t.Error("Incorrect availability for 'none':", av)
}
}
func TestGlobalReset(t *testing.T) {
ldb := newLowlevelMemory(t)
defer ldb.Close()
2014-07-06 12:46:48 +00:00
m := newFileSet(t, "test", ldb)
2014-07-12 21:06:48 +00:00
local := []protocol.FileInfo{
{Name: "a", Sequence: 1, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: "b", Sequence: 2, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: "c", Sequence: 3, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: "d", Sequence: 4, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
}
2014-07-12 21:06:48 +00:00
remote := []protocol.FileInfo{
{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}},
{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
}
replace(m, protocol.LocalDeviceID, local)
g := globalList(t, m)
sort.Sort(fileList(g))
if diff, equal := messagediff.PrettyDiff(local, g); !equal {
t.Errorf("Global incorrect;\nglobal: %v\n!=\nlocal: %v\ndiff:\n%s", g, local, diff)
}
replace(m, remoteDevice0, remote)
replace(m, remoteDevice0, nil)
g = globalList(t, m)
sort.Sort(fileList(g))
if diff, equal := messagediff.PrettyDiff(local, g); !equal {
t.Errorf("Global incorrect;\nglobal: %v\n!=\nlocal: %v\ndiff:\n%s", g, local, diff)
}
}
func TestNeed(t *testing.T) {
ldb := newLowlevelMemory(t)
defer ldb.Close()
2014-07-06 12:46:48 +00:00
m := newFileSet(t, "test", ldb)
2014-07-12 21:06:48 +00:00
local := []protocol.FileInfo{
{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
}
2014-07-12 21:06:48 +00:00
remote := []protocol.FileInfo{
{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}},
{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
}
2014-07-12 21:06:48 +00:00
shouldNeed := []protocol.FileInfo{
{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}},
{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
}
replace(m, protocol.LocalDeviceID, local)
replace(m, remoteDevice0, remote)
need := needList(t, m, protocol.LocalDeviceID)
2014-04-09 08:24:43 +00:00
sort.Sort(fileList(need))
sort.Sort(fileList(shouldNeed))
2014-07-06 12:46:48 +00:00
if fmt.Sprint(need) != fmt.Sprint(shouldNeed) {
t.Errorf("Need incorrect;\n%v !=\n%v", need, shouldNeed)
}
checkNeed(t, m, protocol.LocalDeviceID, shouldNeed)
}
func TestSequence(t *testing.T) {
ldb := newLowlevelMemory(t)
defer ldb.Close()
2014-07-06 12:46:48 +00:00
m := newFileSet(t, "test", ldb)
2014-07-12 21:06:48 +00:00
local1 := []protocol.FileInfo{
{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
}
2014-07-12 21:06:48 +00:00
local2 := []protocol.FileInfo{
local1[0],
// [1] deleted
local1[2],
{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
}
replace(m, protocol.LocalDeviceID, local1)
c0 := m.Sequence(protocol.LocalDeviceID)
replace(m, protocol.LocalDeviceID, local2)
c1 := m.Sequence(protocol.LocalDeviceID)
if !(c1 > c0) {
t.Fatal("Local version number should have incremented")
}
}
func TestListDropFolder(t *testing.T) {
ldb := newLowlevelMemory(t)
defer ldb.Close()
s0 := newFileSet(t, "test0", ldb)
local1 := []protocol.FileInfo{
{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
}
replace(s0, protocol.LocalDeviceID, local1)
s1 := newFileSet(t, "test1", ldb)
local2 := []protocol.FileInfo{
{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
{Name: "f", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
}
replace(s1, remoteDevice0, local2)
// Check that we have both folders and their data is in the global list
expectedFolderList := []string{"test0", "test1"}
2016-03-06 20:32:10 +00:00
actualFolderList := ldb.ListFolders()
if diff, equal := messagediff.PrettyDiff(expectedFolderList, actualFolderList); !equal {
2016-03-06 20:32:10 +00:00
t.Fatalf("FolderList mismatch. Diff:\n%s", diff)
}
if l := len(globalList(t, s0)); l != 3 {
t.Errorf("Incorrect global length %d != 3 for s0", l)
}
if l := len(globalList(t, s1)); l != 3 {
t.Errorf("Incorrect global length %d != 3 for s1", l)
}
// Drop one of them and check that it's gone.
db.DropFolder(ldb, "test1")
expectedFolderList = []string{"test0"}
2016-03-06 20:32:10 +00:00
actualFolderList = ldb.ListFolders()
if diff, equal := messagediff.PrettyDiff(expectedFolderList, actualFolderList); !equal {
2016-03-06 20:32:10 +00:00
t.Fatalf("FolderList mismatch. Diff:\n%s", diff)
}
if l := len(globalList(t, s0)); l != 3 {
t.Errorf("Incorrect global length %d != 3 for s0", l)
}
if l := len(globalList(t, s1)); l != 0 {
t.Errorf("Incorrect global length %d != 0 for s1", l)
}
}
func TestGlobalNeedWithInvalid(t *testing.T) {
ldb := newLowlevelMemory(t)
defer ldb.Close()
s := newFileSet(t, "test1", ldb)
rem0 := fileList{
protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, RawInvalid: true},
protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: remoteDevice0.Short(), Value: 1002}}}},
}
replace(s, remoteDevice0, rem0)
rem1 := fileList{
protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, RawInvalid: true},
protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, RawInvalid: true, ModifiedS: 10},
}
replace(s, remoteDevice1, rem1)
total := fileList{
// There's a valid copy of each file, so it should be merged
protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
// in conflict and older, but still wins as the other is invalid
protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: remoteDevice0.Short(), Value: 1002}}}},
}
need := fileList(needList(t, s, protocol.LocalDeviceID))
if fmt.Sprint(need) != fmt.Sprint(total) {
t.Errorf("Need incorrect;\n A: %v !=\n E: %v", need, total)
}
checkNeed(t, s, protocol.LocalDeviceID, total)
global := fileList(globalList(t, s))
if fmt.Sprint(global) != fmt.Sprint(total) {
t.Errorf("Global incorrect;\n A: %v !=\n E: %v", global, total)
}
}
func TestLongPath(t *testing.T) {
ldb := newLowlevelMemory(t)
defer ldb.Close()
s := newFileSet(t, "test", ldb)
var b bytes.Buffer
for i := 0; i < 100; i++ {
b.WriteString("012345678901234567890123456789012345678901234567890")
}
name := b.String() // 5000 characters
local := []protocol.FileInfo{
{Name: name, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
}
replace(s, protocol.LocalDeviceID, local)
gf := globalList(t, s)
if l := len(gf); l != 1 {
t.Fatalf("Incorrect len %d != 1 for global list", l)
}
if gf[0].Name != local[0].Name {
t.Errorf("Incorrect long filename;\n%q !=\n%q",
gf[0].Name, local[0].Name)
}
}
2016-01-25 09:04:53 +00:00
func BenchmarkUpdateOneFile(b *testing.B) {
local0 := fileList{
protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(3)},
protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(4)},
2016-01-25 09:04:53 +00:00
// A longer name is more realistic and causes more allocations
protocol.FileInfo{Name: "zajksdhaskjdh/askjdhaskjdashkajshd/kasjdhaskjdhaskdjhaskdjash/dkjashdaksjdhaskdjahskdjh", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(8)},
2016-01-25 09:04:53 +00:00
}
be, err := backend.Open("testdata/benchmarkupdate.db", backend.TuningAuto)
2016-01-25 09:04:53 +00:00
if err != nil {
b.Fatal(err)
}
ldb := newLowlevel(b, be)
2016-01-25 09:04:53 +00:00
defer func() {
ldb.Close()
os.RemoveAll("testdata/benchmarkupdate.db")
}()
m := newFileSet(b, "test", ldb)
replace(m, protocol.LocalDeviceID, local0)
2016-01-25 09:04:53 +00:00
l := local0[4:5]
for i := 0; i < b.N; i++ {
l[0].Version = l[0].Version.Update(myID)
m.Update(protocol.LocalDeviceID, local0)
}
b.ReportAllocs()
}
func TestIndexID(t *testing.T) {
ldb := newLowlevelMemory(t)
defer ldb.Close()
s := newFileSet(t, "test", ldb)
// The Index ID for some random device is zero by default.
id := s.IndexID(remoteDevice0)
if id != 0 {
t.Errorf("index ID for remote device should default to zero, not %d", id)
}
// The Index ID for someone else should be settable
s.SetIndexID(remoteDevice0, 42)
id = s.IndexID(remoteDevice0)
if id != 42 {
t.Errorf("index ID for remote device should be remembered; got %d, expected %d", id, 42)
}
// Our own index ID should be generated randomly.
id = s.IndexID(protocol.LocalDeviceID)
if id == 0 {
t.Errorf("index ID for local device should be random, not zero")
}
t.Logf("random index ID is 0x%016x", id)
// But of course always the same after that.
again := s.IndexID(protocol.LocalDeviceID)
if again != id {
t.Errorf("index ID changed; %d != %d", again, id)
}
}
func TestDropFiles(t *testing.T) {
ldb := newLowlevelMemory(t)
m := newFileSet(t, "test", ldb)
local0 := fileList{
protocol.FileInfo{Name: "a", Sequence: 1, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
protocol.FileInfo{Name: "b", Sequence: 2, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "c", Sequence: 3, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(3)},
protocol.FileInfo{Name: "d", Sequence: 4, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(4)},
protocol.FileInfo{Name: "z", Sequence: 5, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(8)},
}
remote0 := fileList{
protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(5)},
}
// Insert files
m.Update(protocol.LocalDeviceID, local0)
m.Update(remoteDevice0, remote0)
// Check that they're there
h := haveList(t, m, protocol.LocalDeviceID)
if len(h) != len(local0) {
t.Errorf("Incorrect number of files after update, %d != %d", len(h), len(local0))
}
h = haveList(t, m, remoteDevice0)
if len(h) != len(remote0) {
t.Errorf("Incorrect number of files after update, %d != %d", len(h), len(local0))
}
g := globalList(t, m)
if len(g) != len(local0) {
// local0 covers all files
t.Errorf("Incorrect global files after update, %d != %d", len(g), len(local0))
}
// Drop the local files and recheck
m.Drop(protocol.LocalDeviceID)
h = haveList(t, m, protocol.LocalDeviceID)
if len(h) != 0 {
t.Errorf("Incorrect number of files after drop, %d != %d", len(h), 0)
}
h = haveList(t, m, remoteDevice0)
if len(h) != len(remote0) {
t.Errorf("Incorrect number of files after update, %d != %d", len(h), len(local0))
}
g = globalList(t, m)
if len(g) != len(remote0) {
// the ones in remote0 remain
t.Errorf("Incorrect global files after update, %d != %d", len(g), len(remote0))
}
}
func TestIssue4701(t *testing.T) {
ldb := newLowlevelMemory(t)
defer ldb.Close()
s := newFileSet(t, "test", ldb)
localHave := fileList{
protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, LocalFlags: protocol.FlagLocalIgnored},
}
s.Update(protocol.LocalDeviceID, localHave)
if c := localSize(t, s); c.Files != 1 {
t.Errorf("Expected 1 local file, got %v", c.Files)
}
if c := globalSize(t, s); c.Files != 1 {
t.Errorf("Expected 1 global file, got %v", c.Files)
}
localHave[1].LocalFlags = 0
s.Update(protocol.LocalDeviceID, localHave)
if c := localSize(t, s); c.Files != 2 {
t.Errorf("Expected 2 local files, got %v", c.Files)
}
if c := globalSize(t, s); c.Files != 2 {
t.Errorf("Expected 2 global files, got %v", c.Files)
}
localHave[0].LocalFlags = protocol.FlagLocalIgnored
localHave[1].LocalFlags = protocol.FlagLocalIgnored
s.Update(protocol.LocalDeviceID, localHave)
if c := localSize(t, s); c.Files != 0 {
t.Errorf("Expected 0 local files, got %v", c.Files)
}
if c := globalSize(t, s); c.Files != 0 {
t.Errorf("Expected 0 global files, got %v", c.Files)
}
}
func TestWithHaveSequence(t *testing.T) {
ldb := newLowlevelMemory(t)
defer ldb.Close()
folder := "test"
s := newFileSet(t, folder, ldb)
// The files must not be in alphabetical order
localHave := fileList{
protocol.FileInfo{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, RawInvalid: true},
protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)},
protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(7)},
protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(5), RawInvalid: true},
}
replace(s, protocol.LocalDeviceID, localHave)
i := 2
snap := snapshot(t, s)
defer snap.Release()
refactor: use modern Protobuf encoder (#9817) At a high level, this is what I've done and why: - I'm moving the protobuf generation for the `protocol`, `discovery` and `db` packages to the modern alternatives, and using `buf` to generate because it's nice and simple. - After trying various approaches on how to integrate the new types with the existing code, I opted for splitting off our own data model types from the on-the-wire generated types. This means we can have a `FileInfo` type with nicer ergonomics and lots of methods, while the protobuf generated type stays clean and close to the wire protocol. It does mean copying between the two when required, which certainly adds a small amount of inefficiency. If we want to walk this back in the future and use the raw generated type throughout, that's possible, this however makes the refactor smaller (!) as it doesn't change everything about the type for everyone at the same time. - I have simply removed in cold blood a significant number of old database migrations. These depended on previous generations of generated messages of various kinds and were annoying to support in the new fashion. The oldest supported database version now is the one from Syncthing 1.9.0 from Sep 7, 2020. - I changed config structs to be regular manually defined structs. For the sake of discussion, some things I tried that turned out not to work... ### Embedding / wrapping Embedding the protobuf generated structs in our existing types as a data container and keeping our methods and stuff: ``` package protocol type FileInfo struct { *generated.FileInfo } ``` This generates a lot of problems because the internal shape of the generated struct is quite different (different names, different types, more pointers), because initializing it doesn't work like you'd expect (i.e., you end up with an embedded nil pointer and a panic), and because the types of child types don't get wrapped. That is, even if we also have a similar wrapper around a `Vector`, that's not the type you get when accessing `someFileInfo.Version`, you get the `*generated.Vector` that doesn't have methods, etc. ### Aliasing ``` package protocol type FileInfo = generated.FileInfo ``` Doesn't help because you can't attach methods to it, plus all the above. ### Generating the types into the target package like we do now and attaching methods This fails because of the different shape of the generated type (as in the embedding case above) plus the generated struct already has a bunch of methods that we can't necessarily override properly (like `String()` and a bunch of getters). ### Methods to functions I considered just moving all the methods we attach to functions in a specific package, so that for example ``` package protocol func (f FileInfo) Equal(other FileInfo) bool ``` would become ``` package fileinfos func Equal(a, b *generated.FileInfo) bool ``` and this would mostly work, but becomes quite verbose and cumbersome, and somewhat limits discoverability (you can't see what methods are available on the type in auto completions, etc). In the end I did this in some cases, like in the database layer where a lot of things like `func (fv *FileVersion) IsEmpty() bool` becomes `func fvIsEmpty(fv *generated.FileVersion)` because they were anyway just internal methods. Fixes #8247
2024-12-01 15:50:17 +00:00
snap.WithHaveSequence(int64(i), func(fi protocol.FileInfo) bool {
if !fi.IsEquivalent(localHave[i-1], 0) {
t.Fatalf("Got %v\nExpected %v", fi, localHave[i-1])
}
i++
return true
})
}
func TestStressWithHaveSequence(t *testing.T) {
// This races two loops against each other: one that continuously does
// updates, and one that continuously does sequence walks. The test fails
// if the sequence walker sees a discontinuity.
if testing.Short() {
t.Skip("Takes a long time")
}
ldb := newLowlevelMemory(t)
defer ldb.Close()
folder := "test"
s := newFileSet(t, folder, ldb)
var localHave []protocol.FileInfo
for i := 0; i < 100; i++ {
localHave = append(localHave, protocol.FileInfo{Name: fmt.Sprintf("file%d", i), Blocks: genBlocks(i * 10)})
}
done := make(chan struct{})
t0 := time.Now()
go func() {
for time.Since(t0) < 10*time.Second {
for j, f := range localHave {
localHave[j].Version = f.Version.Update(42)
}
s.Update(protocol.LocalDeviceID, localHave)
}
close(done)
}()
var prevSeq int64
loop:
for {
select {
case <-done:
break loop
default:
}
snap := snapshot(t, s)
refactor: use modern Protobuf encoder (#9817) At a high level, this is what I've done and why: - I'm moving the protobuf generation for the `protocol`, `discovery` and `db` packages to the modern alternatives, and using `buf` to generate because it's nice and simple. - After trying various approaches on how to integrate the new types with the existing code, I opted for splitting off our own data model types from the on-the-wire generated types. This means we can have a `FileInfo` type with nicer ergonomics and lots of methods, while the protobuf generated type stays clean and close to the wire protocol. It does mean copying between the two when required, which certainly adds a small amount of inefficiency. If we want to walk this back in the future and use the raw generated type throughout, that's possible, this however makes the refactor smaller (!) as it doesn't change everything about the type for everyone at the same time. - I have simply removed in cold blood a significant number of old database migrations. These depended on previous generations of generated messages of various kinds and were annoying to support in the new fashion. The oldest supported database version now is the one from Syncthing 1.9.0 from Sep 7, 2020. - I changed config structs to be regular manually defined structs. For the sake of discussion, some things I tried that turned out not to work... ### Embedding / wrapping Embedding the protobuf generated structs in our existing types as a data container and keeping our methods and stuff: ``` package protocol type FileInfo struct { *generated.FileInfo } ``` This generates a lot of problems because the internal shape of the generated struct is quite different (different names, different types, more pointers), because initializing it doesn't work like you'd expect (i.e., you end up with an embedded nil pointer and a panic), and because the types of child types don't get wrapped. That is, even if we also have a similar wrapper around a `Vector`, that's not the type you get when accessing `someFileInfo.Version`, you get the `*generated.Vector` that doesn't have methods, etc. ### Aliasing ``` package protocol type FileInfo = generated.FileInfo ``` Doesn't help because you can't attach methods to it, plus all the above. ### Generating the types into the target package like we do now and attaching methods This fails because of the different shape of the generated type (as in the embedding case above) plus the generated struct already has a bunch of methods that we can't necessarily override properly (like `String()` and a bunch of getters). ### Methods to functions I considered just moving all the methods we attach to functions in a specific package, so that for example ``` package protocol func (f FileInfo) Equal(other FileInfo) bool ``` would become ``` package fileinfos func Equal(a, b *generated.FileInfo) bool ``` and this would mostly work, but becomes quite verbose and cumbersome, and somewhat limits discoverability (you can't see what methods are available on the type in auto completions, etc). In the end I did this in some cases, like in the database layer where a lot of things like `func (fv *FileVersion) IsEmpty() bool` becomes `func fvIsEmpty(fv *generated.FileVersion)` because they were anyway just internal methods. Fixes #8247
2024-12-01 15:50:17 +00:00
snap.WithHaveSequence(prevSeq+1, func(fi protocol.FileInfo) bool {
if fi.SequenceNo() < prevSeq+1 {
t.Fatal("Skipped ", prevSeq+1, fi.SequenceNo())
}
prevSeq = fi.SequenceNo()
return true
})
snap.Release()
}
}
func TestIssue4925(t *testing.T) {
ldb := newLowlevelMemory(t)
defer ldb.Close()
folder := "test"
s := newFileSet(t, folder, ldb)
localHave := fileList{
protocol.FileInfo{Name: "dir"},
protocol.FileInfo{Name: "dir.file"},
protocol.FileInfo{Name: "dir/file"},
}
replace(s, protocol.LocalDeviceID, localHave)
for _, prefix := range []string{"dir", "dir/"} {
pl := haveListPrefixed(t, s, protocol.LocalDeviceID, prefix)
if l := len(pl); l != 2 {
t.Errorf("Expected 2, got %v local items below %v", l, prefix)
}
pl = globalListPrefixed(t, s, prefix)
if l := len(pl); l != 2 {
t.Errorf("Expected 2, got %v global items below %v", l, prefix)
}
}
}
func TestMoveGlobalBack(t *testing.T) {
ldb := newLowlevelMemory(t)
defer ldb.Close()
folder := "test"
file := "foo"
s := newFileSet(t, folder, ldb)
localHave := fileList{{Name: file, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}}}, Blocks: genBlocks(1), ModifiedS: 10, Size: 1}}
remote0Have := fileList{{Name: file, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}, {ID: remoteDevice0.Short(), Value: 1}}}, Blocks: genBlocks(2), ModifiedS: 0, Size: 2}}
s.Update(protocol.LocalDeviceID, localHave)
s.Update(remoteDevice0, remote0Have)
if need := needList(t, s, protocol.LocalDeviceID); len(need) != 1 {
t.Error("Expected 1 local need, got", need)
} else if !need[0].IsEquivalent(remote0Have[0], 0) {
t.Errorf("Local need incorrect;\n A: %v !=\n E: %v", need[0], remote0Have[0])
}
checkNeed(t, s, protocol.LocalDeviceID, remote0Have[:1])
if need := needList(t, s, remoteDevice0); len(need) != 0 {
t.Error("Expected no need for remote 0, got", need)
}
checkNeed(t, s, remoteDevice0, nil)
ls := localSize(t, s)
if haveBytes := localHave[0].Size; ls.Bytes != haveBytes {
t.Errorf("Incorrect LocalSize bytes; %d != %d", ls.Bytes, haveBytes)
}
gs := globalSize(t, s)
if globalBytes := remote0Have[0].Size; gs.Bytes != globalBytes {
t.Errorf("Incorrect GlobalSize bytes; %d != %d", gs.Bytes, globalBytes)
}
// That's what happens when something becomes unignored or something.
// In any case it will be moved back from first spot in the global list
// which is the scenario to be tested here.
remote0Have[0].Version = remote0Have[0].Version.Update(remoteDevice0.Short()).DropOthers(remoteDevice0.Short())
s.Update(remoteDevice0, remote0Have)
if need := needList(t, s, remoteDevice0); len(need) != 1 {
t.Error("Expected 1 need for remote 0, got", need)
} else if !need[0].IsEquivalent(localHave[0], 0) {
t.Errorf("Need for remote 0 incorrect;\n A: %v !=\n E: %v", need[0], localHave[0])
}
checkNeed(t, s, remoteDevice0, localHave[:1])
if need := needList(t, s, protocol.LocalDeviceID); len(need) != 0 {
t.Error("Expected no local need, got", need)
}
checkNeed(t, s, protocol.LocalDeviceID, nil)
ls = localSize(t, s)
if haveBytes := localHave[0].Size; ls.Bytes != haveBytes {
t.Errorf("Incorrect LocalSize bytes; %d != %d", ls.Bytes, haveBytes)
}
gs = globalSize(t, s)
if globalBytes := localHave[0].Size; gs.Bytes != globalBytes {
t.Errorf("Incorrect GlobalSize bytes; %d != %d", gs.Bytes, globalBytes)
}
}
// TestIssue5007 checks, that updating the local device with an invalid file
// info with the newest version does indeed remove that file from the list of
// needed files.
// https://github.com/syncthing/syncthing/issues/5007
func TestIssue5007(t *testing.T) {
ldb := newLowlevelMemory(t)
defer ldb.Close()
folder := "test"
file := "foo"
s := newFileSet(t, folder, ldb)
fs := fileList{{Name: file, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}}}}}
s.Update(remoteDevice0, fs)
if need := needList(t, s, protocol.LocalDeviceID); len(need) != 1 {
t.Fatal("Expected 1 local need, got", need)
} else if !need[0].IsEquivalent(fs[0], 0) {
t.Fatalf("Local need incorrect;\n A: %v !=\n E: %v", need[0], fs[0])
}
checkNeed(t, s, protocol.LocalDeviceID, fs[:1])
fs[0].LocalFlags = protocol.FlagLocalIgnored
s.Update(protocol.LocalDeviceID, fs)
if need := needList(t, s, protocol.LocalDeviceID); len(need) != 0 {
t.Fatal("Expected no local need, got", need)
}
checkNeed(t, s, protocol.LocalDeviceID, nil)
}
// TestNeedDeleted checks that a file that doesn't exist locally isn't needed
// when the global file is deleted.
func TestNeedDeleted(t *testing.T) {
ldb := newLowlevelMemory(t)
defer ldb.Close()
folder := "test"
file := "foo"
s := newFileSet(t, folder, ldb)
fs := fileList{{Name: file, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}}}, Deleted: true}}
s.Update(remoteDevice0, fs)
if need := needList(t, s, protocol.LocalDeviceID); len(need) != 0 {
t.Fatal("Expected no local need, got", need)
}
checkNeed(t, s, protocol.LocalDeviceID, nil)
fs[0].Deleted = false
fs[0].Version = fs[0].Version.Update(remoteDevice0.Short())
s.Update(remoteDevice0, fs)
if need := needList(t, s, protocol.LocalDeviceID); len(need) != 1 {
t.Fatal("Expected 1 local need, got", need)
} else if !need[0].IsEquivalent(fs[0], 0) {
t.Fatalf("Local need incorrect;\n A: %v !=\n E: %v", need[0], fs[0])
}
checkNeed(t, s, protocol.LocalDeviceID, fs[:1])
fs[0].Deleted = true
fs[0].Version = fs[0].Version.Update(remoteDevice0.Short())
s.Update(remoteDevice0, fs)
if need := needList(t, s, protocol.LocalDeviceID); len(need) != 0 {
t.Fatal("Expected no local need, got", need)
}
checkNeed(t, s, protocol.LocalDeviceID, nil)
}
func TestReceiveOnlyAccounting(t *testing.T) {
ldb := newLowlevelMemory(t)
defer ldb.Close()
folder := "test"
s := newFileSet(t, folder, ldb)
local := protocol.DeviceID{1}
remote := protocol.DeviceID{2}
// Three files that have been created by the remote device
version := protocol.Vector{Counters: []protocol.Counter{{ID: remote.Short(), Value: 1}}}
files := fileList{
protocol.FileInfo{Name: "f1", Size: 10, Sequence: 1, Version: version},
protocol.FileInfo{Name: "f2", Size: 10, Sequence: 1, Version: version},
protocol.FileInfo{Name: "f3", Size: 10, Sequence: 1, Version: version},
}
// We have synced them locally
replace(s, protocol.LocalDeviceID, files)
replace(s, remote, files)
if n := localSize(t, s).Files; n != 3 {
t.Fatal("expected 3 local files initially, not", n)
}
if n := localSize(t, s).Bytes; n != 30 {
t.Fatal("expected 30 local bytes initially, not", n)
}
if n := globalSize(t, s).Files; n != 3 {
t.Fatal("expected 3 global files initially, not", n)
}
if n := globalSize(t, s).Bytes; n != 30 {
t.Fatal("expected 30 global bytes initially, not", n)
}
if n := receiveOnlyChangedSize(t, s).Files; n != 0 {
t.Fatal("expected 0 receive only changed files initially, not", n)
}
if n := receiveOnlyChangedSize(t, s).Bytes; n != 0 {
t.Fatal("expected 0 receive only changed bytes initially, not", n)
}
// Detected a local change in a receive only folder
changed := files[0]
changed.Version = changed.Version.Update(local.Short())
changed.Size = 100
changed.ModifiedBy = local.Short()
changed.LocalFlags = protocol.FlagLocalReceiveOnly
s.Update(protocol.LocalDeviceID, []protocol.FileInfo{changed})
// Check that we see the files
if n := localSize(t, s).Files; n != 3 {
t.Fatal("expected 3 local files after local change, not", n)
}
if n := localSize(t, s).Bytes; n != 120 {
t.Fatal("expected 120 local bytes after local change, not", n)
}
if n := globalSize(t, s).Files; n != 3 {
t.Fatal("expected 3 global files after local change, not", n)
}
if n := globalSize(t, s).Bytes; n != 30 {
t.Fatal("expected 30 global files after local change, not", n)
}
if n := receiveOnlyChangedSize(t, s).Files; n != 1 {
t.Fatal("expected 1 receive only changed file after local change, not", n)
}
if n := receiveOnlyChangedSize(t, s).Bytes; n != 100 {
t.Fatal("expected 100 receive only changed bytes after local change, not", n)
}
// Fake a revert. That's a two step process, first converting our
// changed file into a less preferred variant, then pulling down the old
// version.
changed.Version = protocol.Vector{}
changed.LocalFlags &^= protocol.FlagLocalReceiveOnly
s.Update(protocol.LocalDeviceID, []protocol.FileInfo{changed})
s.Update(protocol.LocalDeviceID, []protocol.FileInfo{files[0]})
// Check that we see the files, same data as initially
if n := localSize(t, s).Files; n != 3 {
t.Fatal("expected 3 local files after revert, not", n)
}
if n := localSize(t, s).Bytes; n != 30 {
t.Fatal("expected 30 local bytes after revert, not", n)
}
if n := globalSize(t, s).Files; n != 3 {
t.Fatal("expected 3 global files after revert, not", n)
}
if n := globalSize(t, s).Bytes; n != 30 {
t.Fatal("expected 30 global bytes after revert, not", n)
}
if n := receiveOnlyChangedSize(t, s).Files; n != 0 {
t.Fatal("expected 0 receive only changed files after revert, not", n)
}
if n := receiveOnlyChangedSize(t, s).Bytes; n != 0 {
t.Fatal("expected 0 receive only changed bytes after revert, not", n)
}
}
func TestNeedAfterUnignore(t *testing.T) {
ldb := newLowlevelMemory(t)
defer ldb.Close()
folder := "test"
file := "foo"
s := newFileSet(t, folder, ldb)
remID := remoteDevice0.Short()
// Initial state: Devices in sync, locally ignored
local := protocol.FileInfo{Name: file, Version: protocol.Vector{Counters: []protocol.Counter{{ID: remID, Value: 1}, {ID: myID, Value: 1}}}, ModifiedS: 10}
local.SetIgnored()
remote := protocol.FileInfo{Name: file, Version: protocol.Vector{Counters: []protocol.Counter{{ID: remID, Value: 1}, {ID: myID, Value: 1}}}, ModifiedS: 10}
s.Update(protocol.LocalDeviceID, fileList{local})
s.Update(remoteDevice0, fileList{remote})
// Unignore locally -> conflicting changes. Remote is newer, thus winning.
local.Version = local.Version.Update(myID)
local.Version = local.Version.DropOthers(myID)
local.LocalFlags = 0
local.ModifiedS = 0
s.Update(protocol.LocalDeviceID, fileList{local})
if need := needList(t, s, protocol.LocalDeviceID); len(need) != 1 {
t.Fatal("Expected one local need, got", need)
} else if !need[0].IsEquivalent(remote, 0) {
t.Fatalf("Got %v, expected %v", need[0], remote)
}
checkNeed(t, s, protocol.LocalDeviceID, []protocol.FileInfo{remote})
}
func TestRemoteInvalidNotAccounted(t *testing.T) {
// Remote files with the invalid bit should not count.
ldb := newLowlevelMemory(t)
defer ldb.Close()
s := newFileSet(t, "test", ldb)
files := []protocol.FileInfo{
{Name: "a", Size: 1234, Sequence: 42, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}}, // valid, should count
{Name: "b", Size: 1234, Sequence: 43, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, RawInvalid: true}, // invalid, doesn't count
}
s.Update(remoteDevice0, files)
global := globalSize(t, s)
if global.Files != 1 {
t.Error("Expected one file in global size, not", global.Files)
}
if global.Bytes != 1234 {
t.Error("Expected 1234 bytes in global size, not", global.Bytes)
}
}
func TestNeedWithNewerInvalid(t *testing.T) {
ldb := newLowlevelMemory(t)
defer ldb.Close()
s := newFileSet(t, "default", ldb)
rem0ID := remoteDevice0.Short()
rem1ID := remoteDevice1.Short()
// Initial state: file present on rem0 and rem1, but not locally.
file := protocol.FileInfo{Name: "foo"}
file.Version = file.Version.Update(rem0ID)
s.Update(remoteDevice0, fileList{file})
s.Update(remoteDevice1, fileList{file})
need := needList(t, s, protocol.LocalDeviceID)
if len(need) != 1 {
t.Fatal("Locally missing file should be needed")
}
if !need[0].IsEquivalent(file, 0) {
t.Fatalf("Got needed file %v, expected %v", need[0], file)
}
checkNeed(t, s, protocol.LocalDeviceID, []protocol.FileInfo{file})
// rem1 sends an invalid file with increased version
inv := file
inv.Version = inv.Version.Update(rem1ID)
inv.RawInvalid = true
s.Update(remoteDevice1, fileList{inv})
// We still have an old file, we need the newest valid file
need = needList(t, s, protocol.LocalDeviceID)
if len(need) != 1 {
t.Fatal("Locally missing file should be needed regardless of invalid files")
}
if !need[0].IsEquivalent(file, 0) {
t.Fatalf("Got needed file %v, expected %v", need[0], file)
}
checkNeed(t, s, protocol.LocalDeviceID, []protocol.FileInfo{file})
}
func TestNeedAfterDeviceRemove(t *testing.T) {
ldb := newLowlevelMemory(t)
defer ldb.Close()
file := "foo"
s := newFileSet(t, "test", ldb)
fs := fileList{{Name: file, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}}}}}
s.Update(protocol.LocalDeviceID, fs)
fs[0].Version = fs[0].Version.Update(myID)
s.Update(remoteDevice0, fs)
if need := needList(t, s, protocol.LocalDeviceID); len(need) != 1 {
t.Fatal("Expected one local need, got", need)
}
s.Drop(remoteDevice0)
if need := needList(t, s, protocol.LocalDeviceID); len(need) != 0 {
t.Fatal("Expected no local need, got", need)
}
checkNeed(t, s, protocol.LocalDeviceID, nil)
}
2018-11-26 16:58:00 +00:00
func TestCaseSensitive(t *testing.T) {
// Normal case sensitive lookup should work
ldb := newLowlevelMemory(t)
defer ldb.Close()
s := newFileSet(t, "test", ldb)
2018-11-26 16:58:00 +00:00
local := []protocol.FileInfo{
{Name: filepath.FromSlash("D1/f1"), Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: filepath.FromSlash("F1"), Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: filepath.FromSlash("d1/F1"), Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: filepath.FromSlash("d1/f1"), Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: filepath.FromSlash("f1"), Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
2018-11-26 16:58:00 +00:00
}
replace(s, protocol.LocalDeviceID, local)
gf := globalList(t, s)
2018-11-26 16:58:00 +00:00
if l := len(gf); l != len(local) {
t.Fatalf("Incorrect len %d != %d for global list", l, len(local))
}
for i := range local {
if gf[i].Name != local[i].Name {
t.Errorf("Incorrect filename;\n%q !=\n%q",
gf[i].Name, local[i].Name)
}
}
}
func TestSequenceIndex(t *testing.T) {
// This test attempts to verify correct operation of the sequence index.
// It's a stress test and needs to run for a long time, but we don't
// really have time for that in normal builds.
runtime := time.Minute
if testing.Short() {
runtime = time.Second
}
// Set up a db and a few files that we will manipulate.
ldb := newLowlevelMemory(t)
defer ldb.Close()
s := newFileSet(t, "test", ldb)
local := []protocol.FileInfo{
{Name: filepath.FromSlash("banana"), Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: filepath.FromSlash("pineapple"), Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: filepath.FromSlash("orange"), Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: filepath.FromSlash("apple"), Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
{Name: filepath.FromSlash("jackfruit"), Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
}
// Start a background routine that makes updates to these files as fast
// as it can. We always update the same files in the same order.
done := make(chan struct{})
defer close(done)
go func() {
for {
select {
case <-done:
return
default:
}
for i := range local {
local[i].Version = local[i].Version.Update(42)
}
s.Update(protocol.LocalDeviceID, local)
}
}()
// Start a routine to walk the sequence index and inspect the result.
refactor: use modern Protobuf encoder (#9817) At a high level, this is what I've done and why: - I'm moving the protobuf generation for the `protocol`, `discovery` and `db` packages to the modern alternatives, and using `buf` to generate because it's nice and simple. - After trying various approaches on how to integrate the new types with the existing code, I opted for splitting off our own data model types from the on-the-wire generated types. This means we can have a `FileInfo` type with nicer ergonomics and lots of methods, while the protobuf generated type stays clean and close to the wire protocol. It does mean copying between the two when required, which certainly adds a small amount of inefficiency. If we want to walk this back in the future and use the raw generated type throughout, that's possible, this however makes the refactor smaller (!) as it doesn't change everything about the type for everyone at the same time. - I have simply removed in cold blood a significant number of old database migrations. These depended on previous generations of generated messages of various kinds and were annoying to support in the new fashion. The oldest supported database version now is the one from Syncthing 1.9.0 from Sep 7, 2020. - I changed config structs to be regular manually defined structs. For the sake of discussion, some things I tried that turned out not to work... ### Embedding / wrapping Embedding the protobuf generated structs in our existing types as a data container and keeping our methods and stuff: ``` package protocol type FileInfo struct { *generated.FileInfo } ``` This generates a lot of problems because the internal shape of the generated struct is quite different (different names, different types, more pointers), because initializing it doesn't work like you'd expect (i.e., you end up with an embedded nil pointer and a panic), and because the types of child types don't get wrapped. That is, even if we also have a similar wrapper around a `Vector`, that's not the type you get when accessing `someFileInfo.Version`, you get the `*generated.Vector` that doesn't have methods, etc. ### Aliasing ``` package protocol type FileInfo = generated.FileInfo ``` Doesn't help because you can't attach methods to it, plus all the above. ### Generating the types into the target package like we do now and attaching methods This fails because of the different shape of the generated type (as in the embedding case above) plus the generated struct already has a bunch of methods that we can't necessarily override properly (like `String()` and a bunch of getters). ### Methods to functions I considered just moving all the methods we attach to functions in a specific package, so that for example ``` package protocol func (f FileInfo) Equal(other FileInfo) bool ``` would become ``` package fileinfos func Equal(a, b *generated.FileInfo) bool ``` and this would mostly work, but becomes quite verbose and cumbersome, and somewhat limits discoverability (you can't see what methods are available on the type in auto completions, etc). In the end I did this in some cases, like in the database layer where a lot of things like `func (fv *FileVersion) IsEmpty() bool` becomes `func fvIsEmpty(fv *generated.FileVersion)` because they were anyway just internal methods. Fixes #8247
2024-12-01 15:50:17 +00:00
seen := make(map[string]protocol.FileInfo)
latest := make([]protocol.FileInfo, 0, len(local))
var seq int64
t0 := time.Now()
for time.Since(t0) < runtime {
// Walk the changes since our last iteration. This should give is
// one instance each of the files that are changed all the time, or
// a subset of those files if we manage to run before a complete
// update has happened since our last iteration.
latest = latest[:0]
snap := snapshot(t, s)
refactor: use modern Protobuf encoder (#9817) At a high level, this is what I've done and why: - I'm moving the protobuf generation for the `protocol`, `discovery` and `db` packages to the modern alternatives, and using `buf` to generate because it's nice and simple. - After trying various approaches on how to integrate the new types with the existing code, I opted for splitting off our own data model types from the on-the-wire generated types. This means we can have a `FileInfo` type with nicer ergonomics and lots of methods, while the protobuf generated type stays clean and close to the wire protocol. It does mean copying between the two when required, which certainly adds a small amount of inefficiency. If we want to walk this back in the future and use the raw generated type throughout, that's possible, this however makes the refactor smaller (!) as it doesn't change everything about the type for everyone at the same time. - I have simply removed in cold blood a significant number of old database migrations. These depended on previous generations of generated messages of various kinds and were annoying to support in the new fashion. The oldest supported database version now is the one from Syncthing 1.9.0 from Sep 7, 2020. - I changed config structs to be regular manually defined structs. For the sake of discussion, some things I tried that turned out not to work... ### Embedding / wrapping Embedding the protobuf generated structs in our existing types as a data container and keeping our methods and stuff: ``` package protocol type FileInfo struct { *generated.FileInfo } ``` This generates a lot of problems because the internal shape of the generated struct is quite different (different names, different types, more pointers), because initializing it doesn't work like you'd expect (i.e., you end up with an embedded nil pointer and a panic), and because the types of child types don't get wrapped. That is, even if we also have a similar wrapper around a `Vector`, that's not the type you get when accessing `someFileInfo.Version`, you get the `*generated.Vector` that doesn't have methods, etc. ### Aliasing ``` package protocol type FileInfo = generated.FileInfo ``` Doesn't help because you can't attach methods to it, plus all the above. ### Generating the types into the target package like we do now and attaching methods This fails because of the different shape of the generated type (as in the embedding case above) plus the generated struct already has a bunch of methods that we can't necessarily override properly (like `String()` and a bunch of getters). ### Methods to functions I considered just moving all the methods we attach to functions in a specific package, so that for example ``` package protocol func (f FileInfo) Equal(other FileInfo) bool ``` would become ``` package fileinfos func Equal(a, b *generated.FileInfo) bool ``` and this would mostly work, but becomes quite verbose and cumbersome, and somewhat limits discoverability (you can't see what methods are available on the type in auto completions, etc). In the end I did this in some cases, like in the database layer where a lot of things like `func (fv *FileVersion) IsEmpty() bool` becomes `func fvIsEmpty(fv *generated.FileVersion)` because they were anyway just internal methods. Fixes #8247
2024-12-01 15:50:17 +00:00
snap.WithHaveSequence(seq+1, func(f protocol.FileInfo) bool {
seen[f.FileName()] = f
latest = append(latest, f)
seq = f.SequenceNo()
return true
})
snap.Release()
// Calculate the spread in sequence number.
var max, min int64
for _, v := range seen {
s := v.SequenceNo()
if max == 0 || max < s {
max = s
}
if min == 0 || min > s {
min = s
}
}
// We shouldn't see a spread larger than the number of files, as
// that would mean we have missed updates. For example, if we were
// to see the following:
//
// banana N
// pineapple N+1
// orange N+2
// apple N+10
// jackfruit N+11
//
// that would mean that there have been updates to banana, pineapple
// and orange that we didn't see in this pass. If those files aren't
// updated again, those updates are permanently lost.
if max-min > int64(len(local)) {
for _, v := range seen {
t.Log("seen", v.FileName(), v.SequenceNo())
}
for _, v := range latest {
t.Log("latest", v.FileName(), v.SequenceNo())
}
t.Fatal("large spread")
}
time.Sleep(time.Millisecond)
}
}
func TestIgnoreAfterReceiveOnly(t *testing.T) {
ldb := newLowlevelMemory(t)
defer ldb.Close()
file := "foo"
s := newFileSet(t, "test", ldb)
fs := fileList{{
Name: file,
Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}}},
LocalFlags: protocol.FlagLocalReceiveOnly,
}}
s.Update(protocol.LocalDeviceID, fs)
fs[0].LocalFlags = protocol.FlagLocalIgnored
s.Update(protocol.LocalDeviceID, fs)
snap := snapshot(t, s)
defer snap.Release()
if f, ok := snap.Get(protocol.LocalDeviceID, file); !ok {
t.Error("File missing in db")
} else if f.IsReceiveOnlyChanged() {
t.Error("File is still receive-only changed")
} else if !f.IsIgnored() {
t.Error("File is not ignored")
}
}
// https://github.com/syncthing/syncthing/issues/6650
func TestUpdateWithOneFileTwice(t *testing.T) {
ldb := newLowlevelMemory(t)
defer ldb.Close()
file := "foo"
s := newFileSet(t, "test", ldb)
fs := fileList{{
Name: file,
Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}}},
Sequence: 1,
}}
s.Update(protocol.LocalDeviceID, fs)
fs = append(fs, fs[0])
for i := range fs {
fs[i].Sequence++
fs[i].Version = fs[i].Version.Update(myID)
}
fs[1].Sequence++
fs[1].Version = fs[1].Version.Update(myID)
s.Update(protocol.LocalDeviceID, fs)
snap := snapshot(t, s)
defer snap.Release()
count := 0
refactor: use modern Protobuf encoder (#9817) At a high level, this is what I've done and why: - I'm moving the protobuf generation for the `protocol`, `discovery` and `db` packages to the modern alternatives, and using `buf` to generate because it's nice and simple. - After trying various approaches on how to integrate the new types with the existing code, I opted for splitting off our own data model types from the on-the-wire generated types. This means we can have a `FileInfo` type with nicer ergonomics and lots of methods, while the protobuf generated type stays clean and close to the wire protocol. It does mean copying between the two when required, which certainly adds a small amount of inefficiency. If we want to walk this back in the future and use the raw generated type throughout, that's possible, this however makes the refactor smaller (!) as it doesn't change everything about the type for everyone at the same time. - I have simply removed in cold blood a significant number of old database migrations. These depended on previous generations of generated messages of various kinds and were annoying to support in the new fashion. The oldest supported database version now is the one from Syncthing 1.9.0 from Sep 7, 2020. - I changed config structs to be regular manually defined structs. For the sake of discussion, some things I tried that turned out not to work... ### Embedding / wrapping Embedding the protobuf generated structs in our existing types as a data container and keeping our methods and stuff: ``` package protocol type FileInfo struct { *generated.FileInfo } ``` This generates a lot of problems because the internal shape of the generated struct is quite different (different names, different types, more pointers), because initializing it doesn't work like you'd expect (i.e., you end up with an embedded nil pointer and a panic), and because the types of child types don't get wrapped. That is, even if we also have a similar wrapper around a `Vector`, that's not the type you get when accessing `someFileInfo.Version`, you get the `*generated.Vector` that doesn't have methods, etc. ### Aliasing ``` package protocol type FileInfo = generated.FileInfo ``` Doesn't help because you can't attach methods to it, plus all the above. ### Generating the types into the target package like we do now and attaching methods This fails because of the different shape of the generated type (as in the embedding case above) plus the generated struct already has a bunch of methods that we can't necessarily override properly (like `String()` and a bunch of getters). ### Methods to functions I considered just moving all the methods we attach to functions in a specific package, so that for example ``` package protocol func (f FileInfo) Equal(other FileInfo) bool ``` would become ``` package fileinfos func Equal(a, b *generated.FileInfo) bool ``` and this would mostly work, but becomes quite verbose and cumbersome, and somewhat limits discoverability (you can't see what methods are available on the type in auto completions, etc). In the end I did this in some cases, like in the database layer where a lot of things like `func (fv *FileVersion) IsEmpty() bool` becomes `func fvIsEmpty(fv *generated.FileVersion)` because they were anyway just internal methods. Fixes #8247
2024-12-01 15:50:17 +00:00
snap.WithHaveSequence(0, func(_ protocol.FileInfo) bool {
count++
return true
})
if count != 1 {
t.Error("Expected to have one file, got", count)
}
}
// https://github.com/syncthing/syncthing/issues/6668
func TestNeedRemoteOnly(t *testing.T) {
ldb := newLowlevelMemory(t)
defer ldb.Close()
s := newFileSet(t, "test", ldb)
remote0Have := fileList{
protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)},
}
s.Update(remoteDevice0, remote0Have)
need := needSize(t, s, remoteDevice0)
if !need.Equal(db.Counts{}) {
t.Error("Expected nothing needed, got", need)
}
}
// https://github.com/syncthing/syncthing/issues/6784
func TestNeedRemoteAfterReset(t *testing.T) {
ldb := newLowlevelMemory(t)
defer ldb.Close()
s := newFileSet(t, "test", ldb)
files := fileList{
protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)},
}
s.Update(protocol.LocalDeviceID, files)
s.Update(remoteDevice0, files)
need := needSize(t, s, remoteDevice0)
if !need.Equal(db.Counts{}) {
t.Error("Expected nothing needed, got", need)
}
s.Drop(remoteDevice0)
need = needSize(t, s, remoteDevice0)
if exp := (db.Counts{Files: 1}); !need.Equal(exp) {
t.Errorf("Expected %v, got %v", exp, need)
}
}
// https://github.com/syncthing/syncthing/issues/6850
func TestIgnoreLocalChanged(t *testing.T) {
ldb := newLowlevelMemory(t)
defer ldb.Close()
s := newFileSet(t, "test", ldb)
// Add locally changed file
files := fileList{
protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2), LocalFlags: protocol.FlagLocalReceiveOnly},
}
s.Update(protocol.LocalDeviceID, files)
if c := globalSize(t, s).Files; c != 0 {
t.Error("Expected no global file, got", c)
}
if c := localSize(t, s).Files; c != 1 {
t.Error("Expected one local file, got", c)
}
// Change file to ignored
files[0].LocalFlags = protocol.FlagLocalIgnored
s.Update(protocol.LocalDeviceID, files)
if c := globalSize(t, s).Files; c != 0 {
t.Error("Expected no global file, got", c)
}
if c := localSize(t, s).Files; c != 0 {
t.Error("Expected no local file, got", c)
}
}
// Dropping the index ID on Drop is bad, because Drop gets called when receiving
// an Index (as opposed to an IndexUpdate), and we don't want to loose the index
// ID when that happens.
func TestNoIndexIDResetOnDrop(t *testing.T) {
ldb := newLowlevelMemory(t)
defer ldb.Close()
s := newFileSet(t, "test", ldb)
s.SetIndexID(remoteDevice0, 1)
s.Drop(remoteDevice0)
if got := s.IndexID(remoteDevice0); got != 1 {
t.Errorf("Expected unchanged (%v), got %v", 1, got)
}
}
func TestConcurrentIndexID(t *testing.T) {
done := make(chan struct{})
var ids [2]protocol.IndexID
setID := func(s *db.FileSet, i int) {
ids[i] = s.IndexID(protocol.LocalDeviceID)
done <- struct{}{}
}
max := 100
if testing.Short() {
max = 10
}
for i := 0; i < max; i++ {
ldb := newLowlevelMemory(t)
s := newFileSet(t, "test", ldb)
go setID(s, 0)
go setID(s, 1)
<-done
<-done
ldb.Close()
if ids[0] != ids[1] {
t.Fatalf("IDs differ after %v rounds", i)
}
}
}
func TestNeedRemoveLastValid(t *testing.T) {
db := newLowlevelMemory(t)
defer db.Close()
folder := "test"
fs := newFileSet(t, folder, db)
files := []protocol.FileInfo{
{Name: "foo", Version: protocol.Vector{}.Update(myID), Sequence: 1},
}
fs.Update(remoteDevice0, files)
files[0].Version = files[0].Version.Update(myID)
fs.Update(remoteDevice1, files)
files[0].LocalFlags = protocol.FlagLocalIgnored
fs.Update(protocol.LocalDeviceID, files)
snap := snapshot(t, fs)
c := snap.NeedSize(remoteDevice0)
if c.Files != 1 {
t.Errorf("Expected 1 needed files initially, got %v", c.Files)
}
snap.Release()
fs.Drop(remoteDevice1)
snap = snapshot(t, fs)
c = snap.NeedSize(remoteDevice0)
if c.Files != 0 {
t.Errorf("Expected no needed files, got %v", c.Files)
}
snap.Release()
}
func replace(fs *db.FileSet, device protocol.DeviceID, files []protocol.FileInfo) {
fs.Drop(device)
fs.Update(device, files)
}
func localSize(t testing.TB, fs *db.FileSet) db.Counts {
snap := snapshot(t, fs)
defer snap.Release()
return snap.LocalSize()
}
func globalSize(t testing.TB, fs *db.FileSet) db.Counts {
snap := snapshot(t, fs)
defer snap.Release()
return snap.GlobalSize()
}
func needSize(t testing.TB, fs *db.FileSet, id protocol.DeviceID) db.Counts {
snap := snapshot(t, fs)
defer snap.Release()
return snap.NeedSize(id)
}
func receiveOnlyChangedSize(t testing.TB, fs *db.FileSet) db.Counts {
snap := snapshot(t, fs)
defer snap.Release()
return snap.ReceiveOnlyChangedSize()
}
func filesToCounts(files []protocol.FileInfo) db.Counts {
cp := db.Counts{}
for _, f := range files {
switch {
case f.IsDeleted():
cp.Deleted++
case f.IsDirectory() && !f.IsSymlink():
cp.Directories++
case f.IsSymlink():
cp.Symlinks++
default:
cp.Files++
}
cp.Bytes += f.FileSize()
}
return cp
}
func checkNeed(t testing.TB, s *db.FileSet, dev protocol.DeviceID, expected []protocol.FileInfo) {
t.Helper()
counts := needSize(t, s, dev)
if exp := filesToCounts(expected); !exp.Equal(counts) {
t.Errorf("Count incorrect (%v): expected %v, got %v", dev, exp, counts)
}
}
func newLowlevel(t testing.TB, backend backend.Backend) *db.Lowlevel {
t.Helper()
ll, err := db.NewLowlevel(backend, events.NoopLogger)
if err != nil {
t.Fatal(err)
}
return ll
}
func newLowlevelMemory(t testing.TB) *db.Lowlevel {
return newLowlevel(t, backend.OpenMemory())
}
func newFileSet(t testing.TB, folder string, ll *db.Lowlevel) *db.FileSet {
t.Helper()
fset, err := db.NewFileSet(folder, ll)
if err != nil {
t.Fatal(err)
}
return fset
}
func snapshot(t testing.TB, fset *db.FileSet) *db.Snapshot {
t.Helper()
snap, err := fset.Snapshot()
if err != nil {
t.Fatal(err)
}
return snap
}