lib/model: Don't set ignore bit when it's already set

This adds a metric for "committed items" to the database instance that I
use in the test code, and a couple of tests that ensure that scans that
don't change anything also don't commit anything.

There was a case in the scanner where we set the invalid bit on files
that are ignored, even though they were already ignored and had the
invalid bit set. I had assumed this would result in an extra database
commit, but it was in fact filtered out by the Set... Anyway, I think we
can save some work on not pushing that change to the Set at all.

GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/3298
This commit is contained in:
Jakob Borg 2016-06-13 17:44:03 +00:00 committed by Audrius Butkevicius
parent bb5b1f8f01
commit b779e22205
5 changed files with 103 additions and 7 deletions

View File

@ -13,6 +13,7 @@ import (
"path/filepath"
"sort"
"strings"
"sync/atomic"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/protocol"
@ -31,6 +32,7 @@ type Instance struct {
*leveldb.DB
folderIdx *smallIndex
deviceIdx *smallIndex
committed int64
}
const (
@ -91,6 +93,11 @@ func newDBInstance(db *leveldb.DB) *Instance {
return i
}
// Committed returns the number of items committed to the database since startup
func (db *Instance) Committed() int64 {
return atomic.LoadInt64(&db.committed)
}
func (db *Instance) genericReplace(folder, device []byte, fs []protocol.FileInfo, localSize, globalSize *sizeTracker, deleteFn deletionHandler) int64 {
sort.Sort(fileList(fs)) // sort list on name, same as in the database

View File

@ -8,6 +8,7 @@ package db
import (
"bytes"
"sync/atomic"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syndtr/goleveldb/leveldb"
@ -55,19 +56,22 @@ func (db *Instance) newReadWriteTransaction() readWriteTransaction {
}
func (t readWriteTransaction) close() {
if err := t.db.Write(t.Batch, nil); err != nil {
panic(err)
}
t.flush()
t.readOnlyTransaction.close()
}
func (t readWriteTransaction) checkFlush() {
if t.Batch.Len() > batchFlushSize {
t.flush()
t.Batch.Reset()
}
}
func (t readWriteTransaction) flush() {
if err := t.db.Write(t.Batch, nil); err != nil {
panic(err)
}
t.Batch.Reset()
}
atomic.AddInt64(&t.db.committed, int64(t.Batch.Len()))
}
func (t readWriteTransaction) insertFile(folder, device []byte, file protocol.FileInfo) int64 {

View File

@ -624,6 +624,39 @@ func TestLongPath(t *testing.T) {
}
}
func TestCommitted(t *testing.T) {
// Verify that the Committed counter increases when we change things and
// doesn't increase when we don't.
ldb := db.OpenMemory()
s := db.NewFileSet("test", ldb)
local := []protocol.FileInfo{
{Name: string("file"), Version: protocol.Vector{{ID: myID, Value: 1000}}},
}
// Adding a file should increase the counter
c0 := ldb.Committed()
s.Replace(protocol.LocalDeviceID, local)
c1 := ldb.Committed()
if c1 <= c0 {
t.Errorf("committed data didn't increase; %d <= %d", c1, c0)
}
// Updating with something identical should not do anything
s.Update(protocol.LocalDeviceID, local)
c2 := ldb.Committed()
if c2 > c1 {
t.Errorf("replace with same contents should do nothing but %d > %d", c2, c1)
}
}
func BenchmarkUpdateOneFile(b *testing.B) {
local0 := fileList{
protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(1)},

View File

@ -1551,7 +1551,7 @@ func (m *Model) internalScanFolderSubdirs(folder string, subs []string) error {
batch = batch[:0]
}
if ignores.Match(f.Name).IsIgnored() || symlinkInvalid(folder, f) {
if !f.IsInvalid() && (ignores.Match(f.Name).IsIgnored() || symlinkInvalid(folder, f)) {
// File has been ignored or an unsupported symlink. Set invalid bit.
l.Debugln("setting invalid bit on ignored", f)
nf := protocol.FileInfo{

View File

@ -1455,6 +1455,58 @@ func TestIssue3164(t *testing.T) {
}
}
func TestScanNoDatabaseWrite(t *testing.T) {
// When scanning, nothing should be committed to database unless
// something actually changed.
db := db.OpenMemory()
m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
m.AddFolder(defaultFolderConfig)
m.StartFolder("default")
m.ServeBackground()
// Start with no ignores, and restore the previous state when the test completes
curIgn, _, err := m.GetIgnores("default")
if err != nil {
t.Fatal(err)
}
defer m.SetIgnores("default", curIgn)
m.SetIgnores("default", nil)
// Scan the folder twice. The second scan should be a no-op database wise
m.ScanFolder("default")
c0 := db.Committed()
m.ScanFolder("default")
c1 := db.Committed()
if c1 != c0 {
t.Errorf("scan should not commit data when nothing changed but %d != %d", c1, c0)
}
// Ignore a file we know exists. It'll be updated in the database.
m.SetIgnores("default", []string{"foo"})
m.ScanFolder("default")
c2 := db.Committed()
if c2 <= c1 {
t.Errorf("scan should commit data when something got ignored but %d <= %d", c2, c1)
}
// Scan again. Nothing should happen.
m.ScanFolder("default")
c3 := db.Committed()
if c3 != c2 {
t.Errorf("scan should not commit data when nothing changed (with ignores) but %d != %d", c3, c2)
}
}
type fakeAddr struct{}
func (fakeAddr) Network() string {