2
2
mirror of https://github.com/octoleo/restic.git synced 2024-10-31 19:02:32 +00:00
restic/internal/repository/index_test.go

437 lines
10 KiB
Go
Raw Normal View History

package repository_test
2015-04-26 15:10:31 +00:00
import (
"bytes"
2018-01-08 19:54:53 +00:00
"math/rand"
2015-04-26 15:10:31 +00:00
"testing"
2017-07-23 12:21:03 +00:00
"github.com/restic/restic/internal/repository"
2017-07-24 15:42:25 +00:00
"github.com/restic/restic/internal/restic"
2017-10-02 13:06:39 +00:00
rtest "github.com/restic/restic/internal/test"
2015-04-26 15:10:31 +00:00
)
func TestIndexSerialize(t *testing.T) {
type testEntry struct {
2016-08-31 18:29:54 +00:00
id restic.ID
pack restic.ID
2016-08-31 18:58:57 +00:00
tpe restic.BlobType
2015-04-26 15:10:31 +00:00
offset, length uint
}
tests := []testEntry{}
idx := repository.NewIndex()
2015-04-26 15:10:31 +00:00
// create 50 packs with 20 blobs each
for i := 0; i < 50; i++ {
packID := restic.NewRandomID()
2015-04-26 15:10:31 +00:00
pos := uint(0)
for j := 0; j < 20; j++ {
id := restic.NewRandomID()
2015-04-26 15:10:31 +00:00
length := uint(i*100 + j)
2016-08-31 21:07:50 +00:00
idx.Store(restic.PackedBlob{
Blob: restic.Blob{
Type: restic.DataBlob,
ID: id,
Offset: pos,
Length: length,
},
PackID: packID,
})
2015-04-26 15:10:31 +00:00
tests = append(tests, testEntry{
id: id,
pack: packID,
2016-08-31 21:07:50 +00:00
tpe: restic.DataBlob,
2015-04-26 15:10:31 +00:00
offset: pos,
length: length,
})
pos += length
}
}
wr := bytes.NewBuffer(nil)
err := idx.Encode(wr)
2017-10-02 13:06:39 +00:00
rtest.OK(t, err)
2015-04-26 15:10:31 +00:00
idx2, err := repository.DecodeIndex(wr.Bytes())
2017-10-02 13:06:39 +00:00
rtest.OK(t, err)
rtest.Assert(t, idx2 != nil,
2015-04-26 15:10:31 +00:00
"nil returned for decoded index")
wr2 := bytes.NewBuffer(nil)
err = idx2.Encode(wr2)
2017-10-02 13:06:39 +00:00
rtest.OK(t, err)
2015-04-26 15:10:31 +00:00
for _, testBlob := range tests {
list, err := idx.Lookup(testBlob.id, testBlob.tpe)
2017-10-02 13:06:39 +00:00
rtest.OK(t, err)
2015-04-26 15:10:31 +00:00
if len(list) != 1 {
t.Errorf("expected one result for blob %v, got %v: %v", testBlob.id.Str(), len(list), list)
}
result := list[0]
2017-10-02 13:06:39 +00:00
rtest.Equals(t, testBlob.pack, result.PackID)
rtest.Equals(t, testBlob.tpe, result.Type)
rtest.Equals(t, testBlob.offset, result.Offset)
rtest.Equals(t, testBlob.length, result.Length)
2015-04-26 15:10:31 +00:00
list2, err := idx2.Lookup(testBlob.id, testBlob.tpe)
2017-10-02 13:06:39 +00:00
rtest.OK(t, err)
2015-04-26 15:10:31 +00:00
if len(list2) != 1 {
t.Errorf("expected one result for blob %v, got %v: %v", testBlob.id.Str(), len(list2), list2)
}
result2 := list2[0]
2017-10-02 13:06:39 +00:00
rtest.Equals(t, testBlob.pack, result2.PackID)
rtest.Equals(t, testBlob.tpe, result2.Type)
rtest.Equals(t, testBlob.offset, result2.Offset)
rtest.Equals(t, testBlob.length, result2.Length)
2015-04-26 15:10:31 +00:00
}
2015-10-12 20:34:12 +00:00
// add more blobs to idx
2015-04-26 15:10:31 +00:00
newtests := []testEntry{}
for i := 0; i < 10; i++ {
packID := restic.NewRandomID()
2015-04-26 15:10:31 +00:00
pos := uint(0)
for j := 0; j < 10; j++ {
id := restic.NewRandomID()
2015-04-26 15:10:31 +00:00
length := uint(i*100 + j)
2016-08-31 21:07:50 +00:00
idx.Store(restic.PackedBlob{
Blob: restic.Blob{
Type: restic.DataBlob,
ID: id,
Offset: pos,
Length: length,
},
PackID: packID,
})
2015-04-26 15:10:31 +00:00
newtests = append(newtests, testEntry{
id: id,
pack: packID,
2016-08-31 21:07:50 +00:00
tpe: restic.DataBlob,
2015-04-26 15:10:31 +00:00
offset: pos,
length: length,
})
pos += length
}
}
2015-10-12 20:34:12 +00:00
// serialize idx, unserialize to idx3
2015-04-26 15:10:31 +00:00
wr3 := bytes.NewBuffer(nil)
2015-10-12 20:34:12 +00:00
err = idx.Finalize(wr3)
2017-10-02 13:06:39 +00:00
rtest.OK(t, err)
2015-04-26 15:10:31 +00:00
2017-10-02 13:06:39 +00:00
rtest.Assert(t, idx.Final(),
2015-10-12 20:34:12 +00:00
"index not final after encoding")
id := restic.NewRandomID()
2017-10-02 13:06:39 +00:00
rtest.OK(t, idx.SetID(id))
2015-11-02 17:51:45 +00:00
id2, err := idx.ID()
2017-10-02 13:06:39 +00:00
rtest.Assert(t, id2.Equal(id),
2015-11-02 17:51:45 +00:00
"wrong ID returned: want %v, got %v", id, id2)
idx3, err := repository.DecodeIndex(wr3.Bytes())
2017-10-02 13:06:39 +00:00
rtest.OK(t, err)
rtest.Assert(t, idx3 != nil,
2015-04-26 15:10:31 +00:00
"nil returned for decoded index")
2017-10-02 13:06:39 +00:00
rtest.Assert(t, idx3.Final(),
2015-10-12 20:34:12 +00:00
"decoded index is not final")
2015-04-26 15:10:31 +00:00
// all new blobs must be in the index
for _, testBlob := range newtests {
list, err := idx3.Lookup(testBlob.id, testBlob.tpe)
2017-10-02 13:06:39 +00:00
rtest.OK(t, err)
2015-04-26 15:10:31 +00:00
if len(list) != 1 {
t.Errorf("expected one result for blob %v, got %v: %v", testBlob.id.Str(), len(list), list)
}
blob := list[0]
2017-10-02 13:06:39 +00:00
rtest.Equals(t, testBlob.pack, blob.PackID)
rtest.Equals(t, testBlob.tpe, blob.Type)
rtest.Equals(t, testBlob.offset, blob.Offset)
rtest.Equals(t, testBlob.length, blob.Length)
2015-04-26 15:10:31 +00:00
}
}
func TestIndexSize(t *testing.T) {
idx := repository.NewIndex()
2015-04-26 15:10:31 +00:00
packs := 200
blobs := 100
for i := 0; i < packs; i++ {
packID := restic.NewRandomID()
2015-04-26 15:10:31 +00:00
pos := uint(0)
for j := 0; j < blobs; j++ {
id := restic.NewRandomID()
2015-04-26 15:10:31 +00:00
length := uint(i*100 + j)
2016-08-31 21:07:50 +00:00
idx.Store(restic.PackedBlob{
Blob: restic.Blob{
Type: restic.DataBlob,
ID: id,
Offset: pos,
Length: length,
},
PackID: packID,
})
2015-04-26 15:10:31 +00:00
pos += length
}
}
wr := bytes.NewBuffer(nil)
err := idx.Encode(wr)
2017-10-02 13:06:39 +00:00
rtest.OK(t, err)
2015-04-26 15:10:31 +00:00
t.Logf("Index file size for %d blobs in %d packs is %d", blobs*packs, packs, wr.Len())
}
// example index serialization from doc/Design.rst
2015-04-26 15:10:31 +00:00
var docExample = []byte(`
2015-07-25 22:40:00 +00:00
{
"supersedes": [
"ed54ae36197f4745ebc4b54d10e0f623eaaaedd03013eb7ae90df881b7781452"
],
"packs": [
{
"id": "73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c",
"blobs": [
{
"id": "3ec79977ef0cf5de7b08cd12b874cd0f62bbaf7f07f3497a5b1bbcc8cb39b1ce",
"type": "data",
"offset": 0,
"length": 25
},{
"id": "9ccb846e60d90d4eb915848add7aa7ea1e4bbabfc60e573db9f7bfb2789afbae",
"type": "tree",
"offset": 38,
"length": 100
},
{
"id": "d3dc577b4ffd38cc4b32122cabf8655a0223ed22edfd93b353dc0c3f2b0fdf66",
"type": "data",
"offset": 150,
"length": 123
}
]
}
]
}
`)
var docOldExample = []byte(`
2015-04-26 15:10:31 +00:00
[ {
"id": "73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c",
"blobs": [
2015-07-25 22:40:00 +00:00
{
"id": "3ec79977ef0cf5de7b08cd12b874cd0f62bbaf7f07f3497a5b1bbcc8cb39b1ce",
"type": "data",
"offset": 0,
"length": 25
},{
"id": "9ccb846e60d90d4eb915848add7aa7ea1e4bbabfc60e573db9f7bfb2789afbae",
"type": "tree",
"offset": 38,
"length": 100
},
{
"id": "d3dc577b4ffd38cc4b32122cabf8655a0223ed22edfd93b353dc0c3f2b0fdf66",
"type": "data",
"offset": 150,
"length": 123
}
2015-04-26 15:10:31 +00:00
]
} ]
`)
var exampleTests = []struct {
2016-08-31 18:29:54 +00:00
id, packID restic.ID
2016-08-31 18:58:57 +00:00
tpe restic.BlobType
2015-04-26 15:10:31 +00:00
offset, length uint
}{
{
2016-09-04 12:38:18 +00:00
restic.TestParseID("3ec79977ef0cf5de7b08cd12b874cd0f62bbaf7f07f3497a5b1bbcc8cb39b1ce"),
restic.TestParseID("73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c"),
2016-08-31 21:07:50 +00:00
restic.DataBlob, 0, 25,
2015-04-26 15:10:31 +00:00
}, {
2016-09-04 12:38:18 +00:00
restic.TestParseID("9ccb846e60d90d4eb915848add7aa7ea1e4bbabfc60e573db9f7bfb2789afbae"),
restic.TestParseID("73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c"),
2016-08-31 21:07:50 +00:00
restic.TreeBlob, 38, 100,
2015-04-26 15:10:31 +00:00
}, {
2016-09-04 12:38:18 +00:00
restic.TestParseID("d3dc577b4ffd38cc4b32122cabf8655a0223ed22edfd93b353dc0c3f2b0fdf66"),
restic.TestParseID("73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c"),
2016-08-31 21:07:50 +00:00
restic.DataBlob, 150, 123,
2015-04-26 15:10:31 +00:00
},
}
2015-11-01 21:32:28 +00:00
var exampleLookupTest = struct {
2016-08-31 18:29:54 +00:00
packID restic.ID
2016-08-31 18:58:57 +00:00
blobs map[restic.ID]restic.BlobType
2015-11-01 21:32:28 +00:00
}{
2016-09-04 12:38:18 +00:00
restic.TestParseID("73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c"),
2016-08-31 18:58:57 +00:00
map[restic.ID]restic.BlobType{
2016-09-04 12:38:18 +00:00
restic.TestParseID("3ec79977ef0cf5de7b08cd12b874cd0f62bbaf7f07f3497a5b1bbcc8cb39b1ce"): restic.DataBlob,
restic.TestParseID("9ccb846e60d90d4eb915848add7aa7ea1e4bbabfc60e573db9f7bfb2789afbae"): restic.TreeBlob,
restic.TestParseID("d3dc577b4ffd38cc4b32122cabf8655a0223ed22edfd93b353dc0c3f2b0fdf66"): restic.DataBlob,
2015-11-01 21:32:28 +00:00
},
}
2015-04-26 15:10:31 +00:00
func TestIndexUnserialize(t *testing.T) {
2016-09-04 12:38:18 +00:00
oldIdx := restic.IDs{restic.TestParseID("ed54ae36197f4745ebc4b54d10e0f623eaaaedd03013eb7ae90df881b7781452")}
2015-07-25 22:40:00 +00:00
idx, err := repository.DecodeIndex(docExample)
2017-10-02 13:06:39 +00:00
rtest.OK(t, err)
2015-04-26 15:10:31 +00:00
for _, test := range exampleTests {
list, err := idx.Lookup(test.id, test.tpe)
2017-10-02 13:06:39 +00:00
rtest.OK(t, err)
2015-04-26 15:10:31 +00:00
if len(list) != 1 {
t.Errorf("expected one result for blob %v, got %v: %v", test.id.Str(), len(list), list)
}
blob := list[0]
t.Logf("looking for blob %v/%v, got %v", test.tpe, test.id.Str(), blob)
2017-10-02 13:06:39 +00:00
rtest.Equals(t, test.packID, blob.PackID)
rtest.Equals(t, test.tpe, blob.Type)
rtest.Equals(t, test.offset, blob.Offset)
rtest.Equals(t, test.length, blob.Length)
2015-04-26 15:10:31 +00:00
}
2015-07-25 22:40:00 +00:00
2017-10-02 13:06:39 +00:00
rtest.Equals(t, oldIdx, idx.Supersedes())
2015-11-01 21:32:28 +00:00
blobs := idx.ListPack(exampleLookupTest.packID)
if len(blobs) != len(exampleLookupTest.blobs) {
t.Fatalf("expected %d blobs in pack, got %d", len(exampleLookupTest.blobs), len(blobs))
}
for _, blob := range blobs {
b, ok := exampleLookupTest.blobs[blob.ID]
if !ok {
2015-11-01 21:32:28 +00:00
t.Errorf("unexpected blob %v found", blob.ID.Str())
}
if blob.Type != b {
t.Errorf("unexpected type for blob %v: want %v, got %v", blob.ID.Str(), b, blob.Type)
}
2015-11-01 21:32:28 +00:00
}
2015-07-25 22:40:00 +00:00
}
2017-01-13 20:27:37 +00:00
func BenchmarkDecodeIndex(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, err := repository.DecodeIndex(docExample)
2017-10-02 13:06:39 +00:00
rtest.OK(b, err)
2017-01-13 20:27:37 +00:00
}
}
2015-07-25 22:40:00 +00:00
func TestIndexUnserializeOld(t *testing.T) {
idx, err := repository.DecodeOldIndex(docOldExample)
2017-10-02 13:06:39 +00:00
rtest.OK(t, err)
2015-07-25 22:40:00 +00:00
for _, test := range exampleTests {
list, err := idx.Lookup(test.id, test.tpe)
2017-10-02 13:06:39 +00:00
rtest.OK(t, err)
2015-07-25 22:40:00 +00:00
if len(list) != 1 {
t.Errorf("expected one result for blob %v, got %v: %v", test.id.Str(), len(list), list)
}
blob := list[0]
2017-10-02 13:06:39 +00:00
rtest.Equals(t, test.packID, blob.PackID)
rtest.Equals(t, test.tpe, blob.Type)
rtest.Equals(t, test.offset, blob.Offset)
rtest.Equals(t, test.length, blob.Length)
2015-07-25 22:40:00 +00:00
}
2017-10-02 13:06:39 +00:00
rtest.Equals(t, 0, len(idx.Supersedes()))
}
2015-10-25 14:28:01 +00:00
func TestIndexPacks(t *testing.T) {
idx := repository.NewIndex()
2016-08-31 18:29:54 +00:00
packs := restic.NewIDSet()
2015-10-25 14:28:01 +00:00
for i := 0; i < 20; i++ {
packID := restic.NewRandomID()
2016-08-31 21:07:50 +00:00
idx.Store(restic.PackedBlob{
Blob: restic.Blob{
Type: restic.DataBlob,
ID: restic.NewRandomID(),
2016-08-31 21:07:50 +00:00
Offset: 0,
Length: 23,
},
PackID: packID,
})
2015-10-25 14:28:01 +00:00
packs.Insert(packID)
}
idxPacks := idx.Packs()
2017-10-02 13:06:39 +00:00
rtest.Assert(t, packs.Equals(idxPacks), "packs in index do not match packs added to index")
2015-10-25 14:28:01 +00:00
}
2018-01-08 19:54:53 +00:00
const maxPackSize = 16 * 1024 * 1024
func createRandomIndex() (idx *repository.Index, lookupID restic.ID) {
idx = repository.NewIndex()
// create index with 200k pack files
for i := 0; i < 200000; i++ {
packID := restic.NewRandomID()
offset := 0
for offset < maxPackSize {
size := 2000 + rand.Intn(4*1024*1024)
id := restic.NewRandomID()
idx.Store(restic.PackedBlob{
PackID: packID,
Blob: restic.Blob{
Type: restic.DataBlob,
ID: id,
Length: uint(size),
Offset: uint(offset),
},
})
offset += size
if rand.Float32() < 0.001 && lookupID.IsNull() {
lookupID = id
}
}
}
return idx, lookupID
}
func BenchmarkIndexHasUnknown(b *testing.B) {
idx, _ := createRandomIndex()
lookupID := restic.NewRandomID()
b.ResetTimer()
for i := 0; i < b.N; i++ {
idx.Has(lookupID, restic.DataBlob)
}
}
func BenchmarkIndexHasKnown(b *testing.B) {
idx, lookupID := createRandomIndex()
b.ResetTimer()
for i := 0; i < b.N; i++ {
idx.Has(lookupID, restic.DataBlob)
}
}