lib/db: Add db and test with invalid files (#4954)

This adds a couple of utilities for transporting databases in JSON and a
test to load a database and verify a couple of invalid bits. The test
itself is quite pointless at the moment, but it lays the groundwork for
testing the migration of this data in the next step (after the invalid
bit should be changed to local flags for local files).

When that happens we need to have a database in the old format already
there in order to be able to test the migration.

lib/db/leveldb_test.go

@@ -10,6 +10,7 @@ import (
 	"bytes"
 	"testing"
 
+	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
@@ -152,3 +153,74 @@ func TestDropIndexIDs(t *testing.T) {
 		t.Fatal("fail remote 4")
 	}
 }
+
+func TestInvalidFiles(t *testing.T) {
+	ldb, err := openJSONS("testdata/v0.14.48-ignoredfiles.db.jsons")
+	if err != nil {
+		t.Fatal(err)
+	}
+	db := newDBInstance(ldb, "<memory>")
+	fs := NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), db)
+
+	// The contents of the database are like this:
+	//
+	// 	fs := NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), db)
+	// 	fs.Update(protocol.LocalDeviceID, []protocol.FileInfo{
+	// 		{ // invalid (ignored) file
+	// 			Name:    "foo",
+	// 			Type:    protocol.FileInfoTypeFile,
+	// 			Invalid: true,
+	// 			Version: protocol.Vector{Counters: []protocol.Counter{{ID: 1, Value: 1000}}},
+	// 		},
+	// 		{ // regular file
+	// 			Name:    "bar",
+	// 			Type:    protocol.FileInfoTypeFile,
+	// 			Version: protocol.Vector{Counters: []protocol.Counter{{ID: 1, Value: 1001}}},
+	// 		},
+	// 	})
+	// 	fs.Update(protocol.DeviceID{42}, []protocol.FileInfo{
+	// 		{ // invalid file
+	// 			Name:    "baz",
+	// 			Type:    protocol.FileInfoTypeFile,
+	// 			Invalid: true,
+	// 			Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 1000}}},
+	// 		},
+	// 		{ // regular file
+	// 			Name:    "quux",
+	// 			Type:    protocol.FileInfoTypeFile,
+	// 			Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 1002}}},
+	// 		},
+	// 	})
+
+	fi, ok := fs.Get(protocol.LocalDeviceID, "foo")
+	if !ok {
+		t.Fatal("foo should exist")
+	}
+	if !fi.Invalid {
+		t.Error("foo should be invalid")
+	}
+
+	fi, ok = fs.Get(protocol.LocalDeviceID, "bar")
+	if !ok {
+		t.Fatal("bar should exist")
+	}
+	if fi.Invalid {
+		t.Error("bar should not be invalid")
+	}
+
+	fi, ok = fs.Get(protocol.DeviceID{42}, "baz")
+	if !ok {
+		t.Fatal("baz should exist")
+	}
+	if !fi.Invalid {
+		t.Error("baz should be invalid")
+	}
+
+	fi, ok = fs.Get(protocol.DeviceID{42}, "quux")
+	if !ok {
+		t.Fatal("quux should exist")
+	}
+	if fi.Invalid {
+		t.Error("quux should not be invalid")
+	}
+}

lib/db/testdata/v0.14.48-ignoredfiles.db.jsons

@@ -0,0 +1,15 @@
+{"k":"AAAAAAAAAAABYmFy","v":"CgNiYXJKBwoFCAEQ6QdQAg=="}
+{"k":"AAAAAAAAAAABZm9v","v":"CgNmb284AUoHCgUIARDoB1AB"}
+{"k":"AAAAAAAAAAACYmF6","v":"CgNiYXo4AUoHCgUIKhDoBw=="}
+{"k":"AAAAAAAAAAACcXV1eA==","v":"CgRxdXV4SgcKBQgqEOoH"}
+{"k":"AQAAAABiYXI=","v":"CisKBwoFCAEQ6QcSIP//////////////////////////////////////////"}
+{"k":"AQAAAABiYXo=","v":"Ci0KBwoFCCoQ6AcSICoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGAE="}
+{"k":"AQAAAABmb28=","v":"Ci0KBwoFCAEQ6AcSIP//////////////////////////////////////////GAE="}
+{"k":"AQAAAABxdXV4","v":"CisKBwoFCCoQ6gcSICoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"}
+{"k":"BgAAAAAAAAAA","v":"dGVzdA=="}
+{"k":"BwAAAAAAAAAA","v":""}
+{"k":"BwAAAAEAAAAA","v":"//////////////////////////////////////////8="}
+{"k":"BwAAAAIAAAAA","v":"KgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="}
+{"k":"CQAAAAA=","v":"CicIATACigEg//////////////////////////////////////////8KJwgCMAKKASD4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+AolCAGKASAqAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABCoudnCuM+vlxU="}
+{"k":"CwAAAAAAAAAAAAAAAQ==","v":"AAAAAAAAAAABZm9v"}
+{"k":"CwAAAAAAAAAAAAAAAg==","v":"AAAAAAAAAAABYmFy"}

lib/db/util_test.go

@@ -0,0 +1,94 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package db
+
+import (
+	"encoding/json"
+	"io"
+	"os"
+
+	"github.com/syncthing/syncthing/lib/fs"
+	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/storage"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+// writeJSONS serializes the database to a JSON stream that can be checked
+// in to the repo and used for tests.
+func writeJSONS(w io.Writer, db *leveldb.DB) {
+	it := db.NewIterator(&util.Range{}, nil)
+	defer it.Release()
+	enc := json.NewEncoder(w)
+	for it.Next() {
+		enc.Encode(map[string][]byte{
+			"k": it.Key(),
+			"v": it.Value(),
+		})
+	}
+}
+
+// openJSONS reads a JSON stream file into a leveldb.DB
+func openJSONS(file string) (*leveldb.DB, error) {
+	fd, err := os.Open(file)
+	if err != nil {
+		return nil, err
+	}
+	dec := json.NewDecoder(fd)
+
+	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+
+	for {
+		var row map[string][]byte
+
+		err := dec.Decode(&row)
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			return nil, err
+		}
+
+		db.Put(row["k"], row["v"], nil)
+	}
+
+	return db, nil
+}
+
+func generateIgnoredFilesDB() {
+	// This generates a database with files with invalid flags, local and
+	// remote, in the format used in 0.14.48.
+
+	db := OpenMemory()
+	fs := NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), db)
+	fs.Update(protocol.LocalDeviceID, []protocol.FileInfo{
+		{ // invalid (ignored) file
+			Name:    "foo",
+			Type:    protocol.FileInfoTypeFile,
+			Invalid: true,
+			Version: protocol.Vector{Counters: []protocol.Counter{{ID: 1, Value: 1000}}},
+		},
+		{ // regular file
+			Name:    "bar",
+			Type:    protocol.FileInfoTypeFile,
+			Version: protocol.Vector{Counters: []protocol.Counter{{ID: 1, Value: 1001}}},
+		},
+	})
+	fs.Update(protocol.DeviceID{42}, []protocol.FileInfo{
+		{ // invalid file
+			Name:    "baz",
+			Type:    protocol.FileInfoTypeFile,
+			Invalid: true,
+			Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 1000}}},
+		},
+		{ // regular file
+			Name:    "quux",
+			Type:    protocol.FileInfoTypeFile,
+			Version: protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: 1002}}},
+		},
+	})
+	writeJSONS(os.Stdout, db.DB)
+}