From 99634c09367ac658fa83a4c8959f003daabd3ce6 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 1 May 2022 14:26:57 +0200 Subject: [PATCH 1/5] Return real size from SaveBlob --- cmd/restic/cmd_debug.go | 2 +- internal/archiver/archiver_test.go | 12 ++++----- internal/archiver/blob_saver.go | 6 +++-- internal/archiver/blob_saver_test.go | 6 ++--- internal/checker/checker_test.go | 2 +- internal/repository/fuzz_test.go | 2 +- internal/repository/repack.go | 2 +- internal/repository/repack_test.go | 4 +-- internal/repository/repository.go | 34 ++++++++++++++++---------- internal/repository/repository_test.go | 12 ++++----- internal/restic/repository.go | 2 +- internal/restic/testing.go | 4 +-- internal/restorer/restorer_test.go | 2 +- 13 files changed, 50 insertions(+), 40 deletions(-) diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index 8efb6c8a3..2a9115b19 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -386,7 +386,7 @@ func loadBlobs(ctx context.Context, repo restic.Repository, pack restic.ID, list } } if reuploadBlobs { - _, _, err := repo.SaveBlob(ctx, blob.Type, plaintext, id, true) + _, _, _, err := repo.SaveBlob(ctx, blob.Type, plaintext, id, true) if err != nil { return err } diff --git a/internal/archiver/archiver_test.go b/internal/archiver/archiver_test.go index 13c82a294..e181bc02a 100644 --- a/internal/archiver/archiver_test.go +++ b/internal/archiver/archiver_test.go @@ -415,16 +415,16 @@ type blobCountingRepo struct { saved map[restic.BlobHandle]uint } -func (repo *blobCountingRepo) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicate bool) (restic.ID, bool, error) { - id, exists, err := repo.Repository.SaveBlob(ctx, t, buf, id, false) +func (repo *blobCountingRepo) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicate bool) (restic.ID, bool, int, error) { + id, exists, size, err := repo.Repository.SaveBlob(ctx, t, buf, id, false) if exists { - return id, exists, err + return id, exists, size, err } h := restic.BlobHandle{ID: id, Type: t} repo.m.Lock() repo.saved[h]++ repo.m.Unlock() - return id, exists, err + return id, exists, size, err } func (repo *blobCountingRepo) SaveTree(ctx context.Context, t *restic.Tree) (restic.ID, error) { @@ -1944,10 +1944,10 @@ type failSaveRepo struct { err error } -func (f *failSaveRepo) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicate bool) (restic.ID, bool, error) { +func (f *failSaveRepo) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicate bool) (restic.ID, bool, int, error) { val := atomic.AddInt32(&f.cnt, 1) if val >= f.failAfter { - return restic.ID{}, false, f.err + return restic.ID{}, false, 0, f.err } return f.Repository.SaveBlob(ctx, t, buf, id, storeDuplicate) diff --git a/internal/archiver/blob_saver.go b/internal/archiver/blob_saver.go index 1ee319091..ffd9cb7e4 100644 --- a/internal/archiver/blob_saver.go +++ b/internal/archiver/blob_saver.go @@ -10,7 +10,7 @@ import ( // Saver allows saving a blob. type Saver interface { - SaveBlob(ctx context.Context, t restic.BlobType, data []byte, id restic.ID, storeDuplicate bool) (restic.ID, bool, error) + SaveBlob(ctx context.Context, t restic.BlobType, data []byte, id restic.ID, storeDuplicate bool) (restic.ID, bool, int, error) Index() restic.MasterIndex } @@ -100,10 +100,11 @@ type saveBlobJob struct { type saveBlobResponse struct { id restic.ID known bool + size int } func (s *BlobSaver) saveBlob(ctx context.Context, t restic.BlobType, buf []byte) (saveBlobResponse, error) { - id, known, err := s.repo.SaveBlob(ctx, t, buf, restic.ID{}, false) + id, known, size, err := s.repo.SaveBlob(ctx, t, buf, restic.ID{}, false) if err != nil { return saveBlobResponse{}, err @@ -112,6 +113,7 @@ func (s *BlobSaver) saveBlob(ctx context.Context, t restic.BlobType, buf []byte) return saveBlobResponse{ id: id, known: known, + size: size, }, nil } diff --git a/internal/archiver/blob_saver_test.go b/internal/archiver/blob_saver_test.go index 54aa374cf..69cd4c2e2 100644 --- a/internal/archiver/blob_saver_test.go +++ b/internal/archiver/blob_saver_test.go @@ -21,13 +21,13 @@ type saveFail struct { failAt int32 } -func (b *saveFail) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicates bool) (restic.ID, bool, error) { +func (b *saveFail) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicates bool) (restic.ID, bool, int, error) { val := atomic.AddInt32(&b.cnt, 1) if val == b.failAt { - return restic.ID{}, false, errTest + return restic.ID{}, false, 0, errTest } - return id, false, nil + return id, false, 0, nil } func (b *saveFail) Index() restic.MasterIndex { diff --git a/internal/checker/checker_test.go b/internal/checker/checker_test.go index 1a9a3b8f3..f658613c3 100644 --- a/internal/checker/checker_test.go +++ b/internal/checker/checker_test.go @@ -483,7 +483,7 @@ func TestCheckerBlobTypeConfusion(t *testing.T) { buf, err := repo.LoadBlob(ctx, restic.TreeBlob, id, nil) test.OK(t, err) - _, _, err = repo.SaveBlob(ctx, restic.DataBlob, buf, id, false) + _, _, _, err = repo.SaveBlob(ctx, restic.DataBlob, buf, id, false) test.OK(t, err) malNode := &restic.Node{ diff --git a/internal/repository/fuzz_test.go b/internal/repository/fuzz_test.go index 5af134d84..3847f37f7 100644 --- a/internal/repository/fuzz_test.go +++ b/internal/repository/fuzz_test.go @@ -23,7 +23,7 @@ func FuzzSaveLoadBlob(f *testing.F) { id := restic.Hash(blob) repo, _ := TestRepositoryWithBackend(t, mem.New(), 2) - _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, blob, id, false) + _, _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, blob, id, false) if err != nil { t.Fatal(err) } diff --git a/internal/repository/repack.go b/internal/repository/repack.go index 4d0ca8236..80902c11c 100644 --- a/internal/repository/repack.go +++ b/internal/repository/repack.go @@ -75,7 +75,7 @@ func Repack(ctx context.Context, repo restic.Repository, dstRepo restic.Reposito } // We do want to save already saved blobs! - _, _, err = dstRepo.SaveBlob(wgCtx, blob.Type, buf, blob.ID, true) + _, _, _, err = dstRepo.SaveBlob(wgCtx, blob.Type, buf, blob.ID, true) if err != nil { return err } diff --git a/internal/repository/repack_test.go b/internal/repository/repack_test.go index f8e375a52..248477292 100644 --- a/internal/repository/repack_test.go +++ b/internal/repository/repack_test.go @@ -32,7 +32,7 @@ func createRandomBlobs(t testing.TB, repo restic.Repository, blobs int, pData fl buf := make([]byte, length) rand.Read(buf) - id, exists, err := repo.SaveBlob(context.TODO(), tpe, buf, restic.ID{}, false) + id, exists, _, err := repo.SaveBlob(context.TODO(), tpe, buf, restic.ID{}, false) if err != nil { t.Fatalf("SaveFrom() error %v", err) } @@ -62,7 +62,7 @@ func createRandomWrongBlob(t testing.TB, repo restic.Repository) { // invert first data byte buf[0] ^= 0xff - _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, buf, id, false) + _, _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, buf, id, false) if err != nil { t.Fatalf("SaveFrom() error %v", err) } diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 1312ea754..beaf7fecc 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -378,9 +378,10 @@ func (r *Repository) getZstdDecoder() *zstd.Decoder { } // saveAndEncrypt encrypts data and stores it to the backend as type t. If data -// is small enough, it will be packed together with other small blobs. -// The caller must ensure that the id matches the data. -func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data []byte, id restic.ID) error { +// is small enough, it will be packed together with other small blobs. The +// caller must ensure that the id matches the data. Returned is the size data +// occupies in the repo (compressed or not, including the encryption overhead). +func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data []byte, id restic.ID) (size int, err error) { debug.Log("save id %v (%v, %d bytes)", id, t, len(data)) uncompressedLength := 0 @@ -417,24 +418,29 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data packer, err := pm.findPacker() if err != nil { - return err + return 0, err } // save ciphertext - _, err = packer.Add(t, id, ciphertext, uncompressedLength) + size, err = packer.Add(t, id, ciphertext, uncompressedLength) if err != nil { - return err + return 0, err } // if the pack is not full enough, put back to the list if packer.Size() < minPackSize { debug.Log("pack is not full enough (%d bytes)", packer.Size()) pm.insertPacker(packer) - return nil + return size, nil } // else write the pack to the backend - return r.savePacker(ctx, t, packer) + err = r.savePacker(ctx, t, packer) + if err != nil { + return 0, err + } + + return size, nil } // SaveJSONUnpacked serialises item as JSON and encrypts and saves it in the @@ -815,8 +821,10 @@ func (r *Repository) Close() error { // It takes care that no duplicates are saved; this can be overwritten // by setting storeDuplicate to true. // If id is the null id, it will be computed and returned. -// Also returns if the blob was already known before -func (r *Repository) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicate bool) (newID restic.ID, known bool, err error) { +// Also returns if the blob was already known before. +// If the blob was not known before, it returns the number of bytes the blob +// occupies in the repo (compressed or not, including encryption overhead). +func (r *Repository) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicate bool) (newID restic.ID, known bool, size int, err error) { // compute plaintext hash if not already set if id.IsNull() { @@ -830,10 +838,10 @@ func (r *Repository) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte // only save when needed or explicitly told if !known || storeDuplicate { - err = r.saveAndEncrypt(ctx, t, buf, newID) + size, err = r.saveAndEncrypt(ctx, t, buf, newID) } - return newID, known, err + return newID, known, size, err } // LoadTree loads a tree from the repository. @@ -867,7 +875,7 @@ func (r *Repository) SaveTree(ctx context.Context, t *restic.Tree) (restic.ID, e // adds a newline after each object) buf = append(buf, '\n') - id, _, err := r.SaveBlob(ctx, restic.TreeBlob, buf, restic.ID{}, false) + id, _, _, err := r.SaveBlob(ctx, restic.TreeBlob, buf, restic.ID{}, false) return id, err } diff --git a/internal/repository/repository_test.go b/internal/repository/repository_test.go index 2265ba453..f0e25f520 100644 --- a/internal/repository/repository_test.go +++ b/internal/repository/repository_test.go @@ -44,7 +44,7 @@ func testSave(t *testing.T, version uint) { id := restic.Hash(data) // save - sid, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, data, restic.ID{}, false) + sid, _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, data, restic.ID{}, false) rtest.OK(t, err) rtest.Equals(t, id, sid) @@ -83,7 +83,7 @@ func testSaveFrom(t *testing.T, version uint) { id := restic.Hash(data) // save - id2, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, data, id, false) + id2, _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, data, id, false) rtest.OK(t, err) rtest.Equals(t, id, id2) @@ -125,7 +125,7 @@ func benchmarkSaveAndEncrypt(t *testing.B, version uint) { t.SetBytes(int64(size)) for i := 0; i < t.N; i++ { - _, _, err = repo.SaveBlob(context.TODO(), restic.DataBlob, data, id, true) + _, _, _, err = repo.SaveBlob(context.TODO(), restic.DataBlob, data, id, true) rtest.OK(t, err) } } @@ -187,7 +187,7 @@ func testLoadBlob(t *testing.T, version uint) { _, err := io.ReadFull(rnd, buf) rtest.OK(t, err) - id, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, buf, restic.ID{}, false) + id, _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, buf, restic.ID{}, false) rtest.OK(t, err) rtest.OK(t, repo.Flush(context.Background())) @@ -220,7 +220,7 @@ func benchmarkLoadBlob(b *testing.B, version uint) { _, err := io.ReadFull(rnd, buf) rtest.OK(b, err) - id, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, buf, restic.ID{}, false) + id, _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, buf, restic.ID{}, false) rtest.OK(b, err) rtest.OK(b, repo.Flush(context.Background())) @@ -396,7 +396,7 @@ func saveRandomDataBlobs(t testing.TB, repo restic.Repository, num int, sizeMax _, err := io.ReadFull(rnd, buf) rtest.OK(t, err) - _, _, err = repo.SaveBlob(context.TODO(), restic.DataBlob, buf, restic.ID{}, false) + _, _, _, err = repo.SaveBlob(context.TODO(), restic.DataBlob, buf, restic.ID{}, false) rtest.OK(t, err) } } diff --git a/internal/restic/repository.go b/internal/restic/repository.go index fea151164..35fdbabcb 100644 --- a/internal/restic/repository.go +++ b/internal/restic/repository.go @@ -46,7 +46,7 @@ type Repository interface { LoadUnpacked(ctx context.Context, buf []byte, t FileType, id ID) (data []byte, err error) LoadBlob(context.Context, BlobType, ID, []byte) ([]byte, error) - SaveBlob(context.Context, BlobType, []byte, ID, bool) (ID, bool, error) + SaveBlob(context.Context, BlobType, []byte, ID, bool) (ID, bool, int, error) LoadTree(context.Context, ID) (*Tree, error) SaveTree(context.Context, *Tree) (ID, error) diff --git a/internal/restic/testing.go b/internal/restic/testing.go index 392538506..54621c183 100644 --- a/internal/restic/testing.go +++ b/internal/restic/testing.go @@ -52,7 +52,7 @@ func (fs *fakeFileSystem) saveFile(ctx context.Context, rd io.Reader) (blobs IDs id := Hash(chunk.Data) if !fs.blobIsKnown(BlobHandle{ID: id, Type: DataBlob}) { - _, _, err := fs.repo.SaveBlob(ctx, DataBlob, chunk.Data, id, true) + _, _, _, err := fs.repo.SaveBlob(ctx, DataBlob, chunk.Data, id, true) if err != nil { fs.t.Fatalf("error saving chunk: %v", err) } @@ -138,7 +138,7 @@ func (fs *fakeFileSystem) saveTree(ctx context.Context, seed int64, depth int) I return id } - _, _, err := fs.repo.SaveBlob(ctx, TreeBlob, buf, id, false) + _, _, _, err := fs.repo.SaveBlob(ctx, TreeBlob, buf, id, false) if err != nil { fs.t.Fatal(err) } diff --git a/internal/restorer/restorer_test.go b/internal/restorer/restorer_test.go index a5a3bb5ba..7e7e0c4c6 100644 --- a/internal/restorer/restorer_test.go +++ b/internal/restorer/restorer_test.go @@ -41,7 +41,7 @@ func saveFile(t testing.TB, repo restic.Repository, node File) restic.ID { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - id, _, err := repo.SaveBlob(ctx, restic.DataBlob, []byte(node.Data), restic.ID{}, false) + id, _, _, err := repo.SaveBlob(ctx, restic.DataBlob, []byte(node.Data), restic.ID{}, false) if err != nil { t.Fatal(err) } From 6c4ceaf1e77b8cd8ad10c1f08957271c7faaa921 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 1 May 2022 14:41:36 +0200 Subject: [PATCH 2/5] Print number of bytes added to the repo This includes optional compression and crypto overhead. --- internal/archiver/archiver.go | 15 ++++--- internal/archiver/archiver_test.go | 11 ++--- internal/archiver/blob_saver.go | 8 +++- internal/archiver/file_saver.go | 1 + internal/ui/backup/json.go | 66 +++++++++++++++++------------- internal/ui/backup/text.go | 8 ++-- 6 files changed, 65 insertions(+), 44 deletions(-) diff --git a/internal/archiver/archiver.go b/internal/archiver/archiver.go index 2c7af3ae0..43153030a 100644 --- a/internal/archiver/archiver.go +++ b/internal/archiver/archiver.go @@ -31,18 +31,22 @@ type ErrorFunc func(file string, fi os.FileInfo, err error) error // ItemStats collects some statistics about a particular file or directory. type ItemStats struct { - DataBlobs int // number of new data blobs added for this item - DataSize uint64 // sum of the sizes of all new data blobs - TreeBlobs int // number of new tree blobs added for this item - TreeSize uint64 // sum of the sizes of all new tree blobs + DataBlobs int // number of new data blobs added for this item + DataSize uint64 // sum of the sizes of all new data blobs + DataSizeInRepo uint64 // sum of the bytes added to the repo (including compression and crypto overhead) + TreeBlobs int // number of new tree blobs added for this item + TreeSize uint64 // sum of the sizes of all new tree blobs + TreeSizeInRepo uint64 // sum of the bytes added to the repo (including compression and crypto overhead) } // Add adds other to the current ItemStats. func (s *ItemStats) Add(other ItemStats) { s.DataBlobs += other.DataBlobs s.DataSize += other.DataSize + s.DataSizeInRepo += other.DataSizeInRepo s.TreeBlobs += other.TreeBlobs s.TreeSize += other.TreeSize + s.TreeSizeInRepo += other.TreeSizeInRepo } // Archiver saves a directory structure to the repo. @@ -183,7 +187,8 @@ func (arch *Archiver) saveTree(ctx context.Context, t *restic.Tree) (restic.ID, res.Wait(ctx) if !res.Known() { s.TreeBlobs++ - s.TreeSize += uint64(len(buf)) + s.TreeSize += uint64(res.Length()) + s.TreeSizeInRepo += uint64(res.SizeInRepo()) } // The context was canceled in the meantime, res.ID() might be invalid if ctx.Err() != nil { diff --git a/internal/archiver/archiver_test.go b/internal/archiver/archiver_test.go index e181bc02a..dfc24b27b 100644 --- a/internal/archiver/archiver_test.go +++ b/internal/archiver/archiver_test.go @@ -1019,7 +1019,7 @@ func TestArchiverSaveTree(t *testing.T) { want: TestDir{ "targetfile": TestFile{Content: string("foobar")}, }, - stat: ItemStats{1, 6, 0, 0}, + stat: ItemStats{1, 6, 32 + 6, 0, 0, 0}, }, { src: TestDir{ @@ -1031,7 +1031,7 @@ func TestArchiverSaveTree(t *testing.T) { "targetfile": TestFile{Content: string("foobar")}, "filesymlink": TestSymlink{Target: "targetfile"}, }, - stat: ItemStats{1, 6, 0, 0}, + stat: ItemStats{1, 6, 32 + 6, 0, 0, 0}, }, { src: TestDir{ @@ -1051,7 +1051,7 @@ func TestArchiverSaveTree(t *testing.T) { "symlink": TestSymlink{Target: "subdir"}, }, }, - stat: ItemStats{0, 0, 1, 0x154}, + stat: ItemStats{0, 0, 0, 1, 0x154, 0x16a}, }, { src: TestDir{ @@ -1075,7 +1075,7 @@ func TestArchiverSaveTree(t *testing.T) { }, }, }, - stat: ItemStats{1, 6, 3, 0x47f}, + stat: ItemStats{1, 6, 32 + 6, 3, 0x47f, 0x4c1}, }, } @@ -1140,7 +1140,8 @@ func TestArchiverSaveTree(t *testing.T) { bothZeroOrNeither(t, uint64(test.stat.DataBlobs), uint64(stat.DataBlobs)) bothZeroOrNeither(t, uint64(test.stat.TreeBlobs), uint64(stat.TreeBlobs)) bothZeroOrNeither(t, test.stat.DataSize, stat.DataSize) - bothZeroOrNeither(t, test.stat.TreeSize, stat.TreeSize) + bothZeroOrNeither(t, test.stat.DataSizeInRepo, stat.DataSizeInRepo) + bothZeroOrNeither(t, test.stat.TreeSizeInRepo, stat.TreeSizeInRepo) }) } } diff --git a/internal/archiver/blob_saver.go b/internal/archiver/blob_saver.go index ffd9cb7e4..eca158c20 100644 --- a/internal/archiver/blob_saver.go +++ b/internal/archiver/blob_saver.go @@ -86,11 +86,17 @@ func (s *FutureBlob) Known() bool { return s.res.known } -// Length returns the length of the blob. +// Length returns the raw length of the blob. func (s *FutureBlob) Length() int { return s.length } +// SizeInRepo returns the number of bytes added to the repo (including +// compression and crypto overhead). +func (s *FutureBlob) SizeInRepo() int { + return s.res.size +} + type saveBlobJob struct { restic.BlobType buf *Buffer diff --git a/internal/archiver/file_saver.go b/internal/archiver/file_saver.go index 9da589f19..0ec871e8b 100644 --- a/internal/archiver/file_saver.go +++ b/internal/archiver/file_saver.go @@ -210,6 +210,7 @@ func (s *FileSaver) saveFile(ctx context.Context, chnker *chunker.Chunker, snPat if !res.Known() { stats.DataBlobs++ stats.DataSize += uint64(res.Length()) + stats.DataSizeInRepo += uint64(res.SizeInRepo()) } node.Content = append(node.Content, res.ID()) diff --git a/internal/ui/backup/json.go b/internal/ui/backup/json.go index fcc200b2a..3e8ce5e94 100644 --- a/internal/ui/backup/json.go +++ b/internal/ui/backup/json.go @@ -110,12 +110,14 @@ func (b *JSONProgress) CompleteItem(messageType, item string, previous, current switch messageType { case "dir new": b.print(verboseUpdate{ - MessageType: "verbose_status", - Action: "new", - Item: item, - Duration: d.Seconds(), - DataSize: s.DataSize, - MetadataSize: s.TreeSize, + MessageType: "verbose_status", + Action: "new", + Item: item, + Duration: d.Seconds(), + DataSize: s.DataSize, + DataSizeInRepo: s.DataSizeInRepo, + MetadataSize: s.TreeSize, + MetadataSizeInRepo: s.TreeSizeInRepo, }) case "dir unchanged": b.print(verboseUpdate{ @@ -125,20 +127,23 @@ func (b *JSONProgress) CompleteItem(messageType, item string, previous, current }) case "dir modified": b.print(verboseUpdate{ - MessageType: "verbose_status", - Action: "modified", - Item: item, - Duration: d.Seconds(), - DataSize: s.DataSize, - MetadataSize: s.TreeSize, + MessageType: "verbose_status", + Action: "modified", + Item: item, + Duration: d.Seconds(), + DataSize: s.DataSize, + DataSizeInRepo: s.DataSizeInRepo, + MetadataSize: s.TreeSize, + MetadataSizeInRepo: s.TreeSizeInRepo, }) case "file new": b.print(verboseUpdate{ - MessageType: "verbose_status", - Action: "new", - Item: item, - Duration: d.Seconds(), - DataSize: s.DataSize, + MessageType: "verbose_status", + Action: "new", + Item: item, + Duration: d.Seconds(), + DataSize: s.DataSize, + DataSizeInRepo: s.DataSizeInRepo, }) case "file unchanged": b.print(verboseUpdate{ @@ -148,11 +153,12 @@ func (b *JSONProgress) CompleteItem(messageType, item string, previous, current }) case "file modified": b.print(verboseUpdate{ - MessageType: "verbose_status", - Action: "modified", - Item: item, - Duration: d.Seconds(), - DataSize: s.DataSize, + MessageType: "verbose_status", + Action: "modified", + Item: item, + Duration: d.Seconds(), + DataSize: s.DataSize, + DataSizeInRepo: s.DataSizeInRepo, }) } } @@ -216,13 +222,15 @@ type errorUpdate struct { } type verboseUpdate struct { - MessageType string `json:"message_type"` // "verbose_status" - Action string `json:"action"` - Item string `json:"item"` - Duration float64 `json:"duration"` // in seconds - DataSize uint64 `json:"data_size"` - MetadataSize uint64 `json:"metadata_size"` - TotalFiles uint `json:"total_files"` + MessageType string `json:"message_type"` // "verbose_status" + Action string `json:"action"` + Item string `json:"item"` + Duration float64 `json:"duration"` // in seconds + DataSize uint64 `json:"data_size"` + DataSizeInRepo uint64 `json:"data_size_in_repo"` + MetadataSize uint64 `json:"metadata_size"` + MetadataSizeInRepo uint64 `json:"metadata_size_in_repo"` + TotalFiles uint `json:"total_files"` } type summaryOutput struct { diff --git a/internal/ui/backup/text.go b/internal/ui/backup/text.go index 998805865..85408e467 100644 --- a/internal/ui/backup/text.go +++ b/internal/ui/backup/text.go @@ -138,17 +138,17 @@ func formatBytes(c uint64) string { func (b *TextProgress) CompleteItem(messageType, item string, previous, current *restic.Node, s archiver.ItemStats, d time.Duration) { switch messageType { case "dir new": - b.VV("new %v, saved in %.3fs (%v added, %v metadata)", item, d.Seconds(), formatBytes(s.DataSize), formatBytes(s.TreeSize)) + b.VV("new %v, saved in %.3fs (%v added, %v stored, %v metadata)", item, d.Seconds(), formatBytes(s.DataSize), formatBytes(s.DataSizeInRepo), formatBytes(s.TreeSizeInRepo)) case "dir unchanged": b.VV("unchanged %v", item) case "dir modified": - b.VV("modified %v, saved in %.3fs (%v added, %v metadata)", item, d.Seconds(), formatBytes(s.DataSize), formatBytes(s.TreeSize)) + b.VV("modified %v, saved in %.3fs (%v added, %v stored, %v metadata)", item, d.Seconds(), formatBytes(s.DataSize), formatBytes(s.DataSizeInRepo), formatBytes(s.TreeSizeInRepo)) case "file new": b.VV("new %v, saved in %.3fs (%v added)", item, d.Seconds(), formatBytes(s.DataSize)) case "file unchanged": b.VV("unchanged %v", item) case "file modified": - b.VV("modified %v, saved in %.3fs (%v added)", item, d.Seconds(), formatBytes(s.DataSize)) + b.VV("modified %v, saved in %.3fs (%v added, %v stored)", item, d.Seconds(), formatBytes(s.DataSize), formatBytes(s.DataSizeInRepo)) } } @@ -178,7 +178,7 @@ func (b *TextProgress) Finish(snapshotID restic.ID, start time.Time, summary *Su if dryRun { verb = "Would add" } - b.P("%s to the repo: %-5s\n", verb, formatBytes(summary.ItemStats.DataSize+summary.ItemStats.TreeSize)) + b.P("%s to the repo: %-5s (%-5s stored)\n", verb, formatBytes(summary.ItemStats.DataSize+summary.ItemStats.TreeSize), formatBytes(summary.ItemStats.DataSizeInRepo+summary.ItemStats.TreeSizeInRepo)) b.P("\n") b.P("processed %v files, %v in %s", summary.Files.New+summary.Files.Changed+summary.Files.Unchanged, From 856d5e430328b1921f5d18c95e50dd62bf17ed84 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 4 Jun 2022 23:53:25 +0200 Subject: [PATCH 3/5] stats: return storage size for raw-data mode raw-data summed up the size of the blob plaintexts. However, with compression this makes little sense as the storage size in the repository is lower due to compression. Thus sum up the actual size each blob takes in the repository. --- cmd/restic/cmd_stats.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/restic/cmd_stats.go b/cmd/restic/cmd_stats.go index b8dcac2a3..b8281034b 100644 --- a/cmd/restic/cmd_stats.go +++ b/cmd/restic/cmd_stats.go @@ -130,11 +130,11 @@ func runStats(gopts GlobalOptions, args []string) error { if statsOptions.countMode == countModeRawData { // the blob handles have been collected, but not yet counted for blobHandle := range stats.blobs { - blobSize, found := repo.LookupBlobSize(blobHandle.ID, blobHandle.Type) - if !found { + pbs := repo.Index().Lookup(blobHandle) + if len(pbs) == 0 { return fmt.Errorf("blob %v not found", blobHandle) } - stats.TotalSize += uint64(blobSize) + stats.TotalSize += uint64(pbs[0].Length) stats.TotalBlobCount++ } } From a6e9e0803414aabde80baaaf3616217894e9877b Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 5 Jun 2022 11:39:57 +0200 Subject: [PATCH 4/5] Account for pack header overhead at each entry This will miss the pack header crypto overhead and the length field, which only amount to a few bytes per pack file. --- internal/pack/pack.go | 16 ++++++---------- internal/repository/packer_manager.go | 18 +++++++++--------- internal/repository/packer_manager_test.go | 6 +++--- internal/repository/repository.go | 6 +++--- 4 files changed, 21 insertions(+), 25 deletions(-) diff --git a/internal/pack/pack.go b/internal/pack/pack.go index 2d7a5c3fb..cf0093e70 100644 --- a/internal/pack/pack.go +++ b/internal/pack/pack.go @@ -31,7 +31,7 @@ func NewPacker(k *crypto.Key, wr io.Writer) *Packer { } // Add saves the data read from rd as a new blob to the packer. Returned is the -// number of bytes written to the pack. +// number of bytes written to the pack plus the pack header entry size. func (p *Packer) Add(t restic.BlobType, id restic.ID, data []byte, uncompressedLength int) (int, error) { p.m.Lock() defer p.m.Unlock() @@ -44,6 +44,7 @@ func (p *Packer) Add(t restic.BlobType, id restic.ID, data []byte, uncompressedL c.UncompressedLength = uint(uncompressedLength) p.bytes += uint(n) p.blobs = append(p.blobs, c) + n += CalculateEntrySize(c) return n, errors.Wrap(err, "Write") } @@ -69,13 +70,11 @@ type compressedHeaderEntry struct { } // Finalize writes the header for all added blobs and finalizes the pack. -// Returned are the number of bytes written, including the header. -func (p *Packer) Finalize() (uint, error) { +// Returned are the number of bytes written, not yet reported by Add. +func (p *Packer) Finalize() (int, error) { p.m.Lock() defer p.m.Unlock() - bytesWritten := p.bytes - header, err := p.makeHeader() if err != nil { return 0, err @@ -97,17 +96,14 @@ func (p *Packer) Finalize() (uint, error) { return 0, errors.New("wrong number of bytes written") } - bytesWritten += uint(hdrBytes) - // write length err = binary.Write(p.wr, binary.LittleEndian, uint32(hdrBytes)) if err != nil { return 0, errors.Wrap(err, "binary.Write") } - bytesWritten += uint(binary.Size(uint32(0))) + p.bytes += uint(hdrBytes + binary.Size(uint32(0))) - p.bytes = uint(bytesWritten) - return bytesWritten, nil + return restic.CiphertextLength(0) + binary.Size(uint32(0)), nil } // makeHeader constructs the header for p. diff --git a/internal/repository/packer_manager.go b/internal/repository/packer_manager.go index 7d0306a03..c7acec7f1 100644 --- a/internal/repository/packer_manager.go +++ b/internal/repository/packer_manager.go @@ -106,11 +106,11 @@ func (r *packerManager) insertPacker(p *Packer) { } // savePacker stores p in the backend. -func (r *Repository) savePacker(ctx context.Context, t restic.BlobType, p *Packer) error { +func (r *Repository) savePacker(ctx context.Context, t restic.BlobType, p *Packer) (int, error) { debug.Log("save packer for %v with %d blobs (%d bytes)\n", t, p.Packer.Count(), p.Packer.Size()) - _, err := p.Packer.Finalize() + hdrOverhead, err := p.Packer.Finalize() if err != nil { - return err + return 0, err } id := restic.IDFromHash(p.hw.Sum(nil)) @@ -122,27 +122,27 @@ func (r *Repository) savePacker(ctx context.Context, t restic.BlobType, p *Packe } rd, err := restic.NewFileReader(p.tmpfile, beHash) if err != nil { - return err + return 0, err } err = r.be.Save(ctx, h, rd) if err != nil { debug.Log("Save(%v) error: %v", h, err) - return err + return 0, err } debug.Log("saved as %v", h) err = p.tmpfile.Close() if err != nil { - return errors.Wrap(err, "close tempfile") + return 0, errors.Wrap(err, "close tempfile") } // on windows the tempfile is automatically deleted on close if runtime.GOOS != "windows" { err = fs.RemoveIfExists(p.tmpfile.Name()) if err != nil { - return errors.Wrap(err, "Remove") + return 0, errors.Wrap(err, "Remove") } } @@ -152,9 +152,9 @@ func (r *Repository) savePacker(ctx context.Context, t restic.BlobType, p *Packe // Save index if full if r.noAutoIndexUpdate { - return nil + return hdrOverhead, nil } - return r.idx.SaveFullIndex(ctx, r) + return hdrOverhead, r.idx.SaveFullIndex(ctx, r) } // countPacker returns the number of open (unfinished) packers. diff --git a/internal/repository/packer_manager_test.go b/internal/repository/packer_manager_test.go index c5233ab4e..633e1be35 100644 --- a/internal/repository/packer_manager_test.go +++ b/internal/repository/packer_manager_test.go @@ -74,8 +74,8 @@ func fillPacks(t testing.TB, rnd *rand.Rand, be Saver, pm *packerManager, buf [] if err != nil { t.Fatal(err) } - if n != l { - t.Errorf("Add() returned invalid number of bytes: want %v, got %v", n, l) + if n != l+37 { + t.Errorf("Add() returned invalid number of bytes: want %v, got %v", l, n) } bytes += l @@ -107,7 +107,7 @@ func flushRemainingPacks(t testing.TB, be Saver, pm *packerManager) (bytes int) if err != nil { t.Fatal(err) } - bytes += int(n) + bytes += n packID := restic.IDFromHash(packer.hw.Sum(nil)) var beHash []byte diff --git a/internal/repository/repository.go b/internal/repository/repository.go index beaf7fecc..3ecf853d2 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -435,12 +435,12 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data } // else write the pack to the backend - err = r.savePacker(ctx, t, packer) + hdrSize, err := r.savePacker(ctx, t, packer) if err != nil { return 0, err } - return size, nil + return size + hdrSize, nil } // SaveJSONUnpacked serialises item as JSON and encrypts and saves it in the @@ -551,7 +551,7 @@ func (r *Repository) flushPacks(ctx context.Context) error { debug.Log("manually flushing %d packs", len(p.pm.packers)) for _, packer := range p.pm.packers { - err := r.savePacker(ctx, p.t, packer) + _, err := r.savePacker(ctx, p.t, packer) if err != nil { p.pm.pm.Unlock() return err From 00d7fcff96b44ba1214d0d2b5a83ce83c910da16 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 5 Jun 2022 00:25:19 +0200 Subject: [PATCH 5/5] extend compression feature changelog entry --- changelog/unreleased/issue-21 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/changelog/unreleased/issue-21 b/changelog/unreleased/issue-21 index b15c51d71..795c5331e 100644 --- a/changelog/unreleased/issue-21 +++ b/changelog/unreleased/issue-21 @@ -23,4 +23,7 @@ repository using `init --repository-version 2 --copy-chunker-params --repo2 path Then use the `copy` command to copy all snapshots to the new repository. https://github.com/restic/restic/issues/21 +https://github.com/restic/restic/issues/3779 https://github.com/restic/restic/pull/3666 +https://github.com/restic/restic/pull/3704 +https://github.com/restic/restic/pull/3733