From ec2b25565aaa021e15a786519df637df54291374 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 29 Apr 2022 23:17:01 +0200 Subject: [PATCH] repository: test uncompressedLength field and index example --- internal/repository/index_test.go | 160 ++++++++++++++++------- internal/repository/master_index_test.go | 23 ++-- 2 files changed, 125 insertions(+), 58 deletions(-) diff --git a/internal/repository/index_test.go b/internal/repository/index_test.go index c4f0179db..6940afe2f 100644 --- a/internal/repository/index_test.go +++ b/internal/repository/index_test.go @@ -23,11 +23,17 @@ func TestIndexSerialize(t *testing.T) { pos := uint(0) for j := 0; j < 20; j++ { length := uint(i*100 + j) + uncompressedLength := uint(0) + if i >= 25 { + // test a mix of compressed and uncompressed packs + uncompressedLength = 2 * length + } pb := restic.PackedBlob{ Blob: restic.Blob{ - BlobHandle: restic.NewRandomBlobHandle(), - Offset: pos, - Length: length, + BlobHandle: restic.NewRandomBlobHandle(), + Offset: pos, + Length: length, + UncompressedLength: uncompressedLength, }, PackID: packID, } @@ -164,7 +170,7 @@ func TestIndexSize(t *testing.T) { } // example index serialization from doc/Design.rst -var docExample = []byte(` +var docExampleV1 = []byte(` { "supersedes": [ "ed54ae36197f4745ebc4b54d10e0f623eaaaedd03013eb7ae90df881b7781452" @@ -177,12 +183,12 @@ var docExample = []byte(` "id": "3ec79977ef0cf5de7b08cd12b874cd0f62bbaf7f07f3497a5b1bbcc8cb39b1ce", "type": "data", "offset": 0, - "length": 25 + "length": 38 },{ "id": "9ccb846e60d90d4eb915848add7aa7ea1e4bbabfc60e573db9f7bfb2789afbae", "type": "tree", "offset": 38, - "length": 100 + "length": 112 }, { "id": "d3dc577b4ffd38cc4b32122cabf8655a0223ed22edfd93b353dc0c3f2b0fdf66", @@ -196,6 +202,41 @@ var docExample = []byte(` } `) +var docExampleV2 = []byte(` +{ + "supersedes": [ + "ed54ae36197f4745ebc4b54d10e0f623eaaaedd03013eb7ae90df881b7781452" + ], + "packs": [ + { + "id": "73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c", + "blobs": [ + { + "id": "3ec79977ef0cf5de7b08cd12b874cd0f62bbaf7f07f3497a5b1bbcc8cb39b1ce", + "type": "data", + "offset": 0, + "length": 38 + }, + { + "id": "9ccb846e60d90d4eb915848add7aa7ea1e4bbabfc60e573db9f7bfb2789afbae", + "type": "tree", + "offset": 38, + "length": 112, + "uncompressed_length": 511 + }, + { + "id": "d3dc577b4ffd38cc4b32122cabf8655a0223ed22edfd93b353dc0c3f2b0fdf66", + "type": "data", + "offset": 150, + "length": 123, + "uncompressed_length": 234 + } + ] + } + ] + } +`) + var docOldExample = []byte(` [ { "id": "73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c", @@ -204,12 +245,12 @@ var docOldExample = []byte(` "id": "3ec79977ef0cf5de7b08cd12b874cd0f62bbaf7f07f3497a5b1bbcc8cb39b1ce", "type": "data", "offset": 0, - "length": 25 + "length": 38 },{ "id": "9ccb846e60d90d4eb915848add7aa7ea1e4bbabfc60e573db9f7bfb2789afbae", "type": "tree", "offset": 38, - "length": 100 + "length": 112 }, { "id": "d3dc577b4ffd38cc4b32122cabf8655a0223ed22edfd93b353dc0c3f2b0fdf66", @@ -222,22 +263,23 @@ var docOldExample = []byte(` `) var exampleTests = []struct { - id, packID restic.ID - tpe restic.BlobType - offset, length uint + id, packID restic.ID + tpe restic.BlobType + offset, length uint + uncompressedLength uint }{ { restic.TestParseID("3ec79977ef0cf5de7b08cd12b874cd0f62bbaf7f07f3497a5b1bbcc8cb39b1ce"), restic.TestParseID("73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c"), - restic.DataBlob, 0, 25, + restic.DataBlob, 0, 38, 0, }, { restic.TestParseID("9ccb846e60d90d4eb915848add7aa7ea1e4bbabfc60e573db9f7bfb2789afbae"), restic.TestParseID("73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c"), - restic.TreeBlob, 38, 100, + restic.TreeBlob, 38, 112, 511, }, { restic.TestParseID("d3dc577b4ffd38cc4b32122cabf8655a0223ed22edfd93b353dc0c3f2b0fdf66"), restic.TestParseID("73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c"), - restic.DataBlob, 150, 123, + restic.DataBlob, 150, 123, 234, }, } @@ -254,41 +296,56 @@ var exampleLookupTest = struct { } func TestIndexUnserialize(t *testing.T) { - oldIdx := restic.IDs{restic.TestParseID("ed54ae36197f4745ebc4b54d10e0f623eaaaedd03013eb7ae90df881b7781452")} + for _, task := range []struct { + idxBytes []byte + version int + }{ + {docExampleV1, 1}, + {docExampleV2, 2}, + } { + oldIdx := restic.IDs{restic.TestParseID("ed54ae36197f4745ebc4b54d10e0f623eaaaedd03013eb7ae90df881b7781452")} - idx, oldFormat, err := repository.DecodeIndex(docExample, restic.NewRandomID()) - rtest.OK(t, err) - rtest.Assert(t, !oldFormat, "new index format recognized as old format") + idx, oldFormat, err := repository.DecodeIndex(task.idxBytes, restic.NewRandomID()) + rtest.OK(t, err) + rtest.Assert(t, !oldFormat, "new index format recognized as old format") - for _, test := range exampleTests { - list := idx.Lookup(restic.BlobHandle{ID: test.id, Type: test.tpe}, nil) - if len(list) != 1 { - t.Errorf("expected one result for blob %v, got %v: %v", test.id.Str(), len(list), list) + for _, test := range exampleTests { + list := idx.Lookup(restic.BlobHandle{ID: test.id, Type: test.tpe}, nil) + if len(list) != 1 { + t.Errorf("expected one result for blob %v, got %v: %v", test.id.Str(), len(list), list) + } + blob := list[0] + + t.Logf("looking for blob %v/%v, got %v", test.tpe, test.id.Str(), blob) + + rtest.Equals(t, test.packID, blob.PackID) + rtest.Equals(t, test.tpe, blob.Type) + rtest.Equals(t, test.offset, blob.Offset) + rtest.Equals(t, test.length, blob.Length) + if task.version == 1 { + rtest.Equals(t, uint(0), blob.UncompressedLength) + } else if task.version == 2 { + rtest.Equals(t, test.uncompressedLength, blob.UncompressedLength) + } else { + t.Fatal("Invalid index version") + } } - blob := list[0] - t.Logf("looking for blob %v/%v, got %v", test.tpe, test.id.Str(), blob) + rtest.Equals(t, oldIdx, idx.Supersedes()) - rtest.Equals(t, test.packID, blob.PackID) - rtest.Equals(t, test.tpe, blob.Type) - rtest.Equals(t, test.offset, blob.Offset) - rtest.Equals(t, test.length, blob.Length) - } - - rtest.Equals(t, oldIdx, idx.Supersedes()) - - blobs := idx.ListPack(exampleLookupTest.packID) - if len(blobs) != len(exampleLookupTest.blobs) { - t.Fatalf("expected %d blobs in pack, got %d", len(exampleLookupTest.blobs), len(blobs)) - } - - for _, blob := range blobs { - b, ok := exampleLookupTest.blobs[blob.ID] - if !ok { - t.Errorf("unexpected blob %v found", blob.ID.Str()) + blobs := idx.ListPack(exampleLookupTest.packID) + if len(blobs) != len(exampleLookupTest.blobs) { + t.Fatalf("expected %d blobs in pack, got %d", len(exampleLookupTest.blobs), len(blobs)) } - if blob.Type != b { - t.Errorf("unexpected type for blob %v: want %v, got %v", blob.ID.Str(), b, blob.Type) + + for _, blob := range blobs { + b, ok := exampleLookupTest.blobs[blob.ID] + if !ok { + t.Errorf("unexpected blob %v found", blob.ID.Str()) + } + if blob.Type != b { + t.Errorf("unexpected type for blob %v: want %v, got %v", blob.ID.Str(), b, blob.Type) + } } } } @@ -403,8 +460,9 @@ func createRandomIndex(rng *rand.Rand, packfiles int) (idx *repository.Index, lo Type: restic.DataBlob, ID: id, }, - Length: uint(size), - Offset: uint(offset), + Length: uint(size), + UncompressedLength: uint(2 * size), + Offset: uint(offset), }) offset += size @@ -475,11 +533,17 @@ func TestIndexHas(t *testing.T) { pos := uint(0) for j := 0; j < 20; j++ { length := uint(i*100 + j) + uncompressedLength := uint(0) + if i >= 25 { + // test a mix of compressed and uncompressed packs + uncompressedLength = 2 * length + } pb := restic.PackedBlob{ Blob: restic.Blob{ - BlobHandle: restic.NewRandomBlobHandle(), - Offset: pos, - Length: length, + BlobHandle: restic.NewRandomBlobHandle(), + Offset: pos, + Length: length, + UncompressedLength: uncompressedLength, }, PackID: packID, } diff --git a/internal/repository/master_index_test.go b/internal/repository/master_index_test.go index 28a2c1251..79932af07 100644 --- a/internal/repository/master_index_test.go +++ b/internal/repository/master_index_test.go @@ -30,9 +30,10 @@ func TestMasterIndex(t *testing.T) { blob2 := restic.PackedBlob{ PackID: restic.NewRandomID(), Blob: restic.Blob{ - BlobHandle: bhInIdx2, - Length: uint(restic.CiphertextLength(100)), - Offset: 10, + BlobHandle: bhInIdx2, + Length: uint(restic.CiphertextLength(100)), + Offset: 10, + UncompressedLength: 200, }, } @@ -48,9 +49,10 @@ func TestMasterIndex(t *testing.T) { blob12b := restic.PackedBlob{ PackID: restic.NewRandomID(), Blob: restic.Blob{ - BlobHandle: bhInIdx12, - Length: uint(restic.CiphertextLength(123)), - Offset: 50, + BlobHandle: bhInIdx12, + Length: uint(restic.CiphertextLength(123)), + Offset: 50, + UncompressedLength: 80, }, } @@ -86,7 +88,7 @@ func TestMasterIndex(t *testing.T) { size, found = mIdx.LookupSize(bhInIdx2) rtest.Equals(t, true, found) - rtest.Equals(t, uint(100), size) + rtest.Equals(t, uint(200), size) // test idInIdx12 found = mIdx.Has(bhInIdx12) @@ -144,9 +146,10 @@ func TestMasterMergeFinalIndexes(t *testing.T) { blob2 := restic.PackedBlob{ PackID: restic.NewRandomID(), Blob: restic.Blob{ - BlobHandle: bhInIdx2, - Length: 100, - Offset: 10, + BlobHandle: bhInIdx2, + Length: 100, + Offset: 10, + UncompressedLength: 200, }, }