From 9b776dc7abc4a2262a1577ed76c9a8355d857517 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Wed, 1 Mar 2017 13:44:56 +0100 Subject: [PATCH 1/2] Use new Index implementation for rebuild-index --- src/cmds/restic/cmd_rebuild_index.go | 47 +++++++++++++++++- src/restic/repository/index_rebuild.go | 66 -------------------------- src/restic/repository/repack_test.go | 21 +++++++- 3 files changed, 64 insertions(+), 70 deletions(-) delete mode 100644 src/restic/repository/index_rebuild.go diff --git a/src/cmds/restic/cmd_rebuild_index.go b/src/cmds/restic/cmd_rebuild_index.go index e4a6c680a..6f6647daa 100644 --- a/src/cmds/restic/cmd_rebuild_index.go +++ b/src/cmds/restic/cmd_rebuild_index.go @@ -1,7 +1,8 @@ package main import ( - "restic/repository" + "restic" + "restic/index" "github.com/spf13/cobra" ) @@ -34,5 +35,47 @@ func runRebuildIndex(gopts GlobalOptions) error { return err } - return repository.RebuildIndex(repo) + done := make(chan struct{}) + defer close(done) + + Verbosef("counting files in repo\n") + + var packs uint64 + for _ = range repo.List(restic.DataFile, done) { + packs++ + } + + bar := newProgressMax(!gopts.Quiet, packs, "packs") + idx, err := index.New(repo, bar) + if err != nil { + return err + } + + Verbosef("listing old index files\n") + var supersedes restic.IDs + for id := range repo.List(restic.IndexFile, done) { + supersedes = append(supersedes, id) + } + + id, err := idx.Save(repo, supersedes) + if err != nil { + return err + } + + Verbosef("saved new index as %v\n", id.Str()) + + Verbosef("remove %d old index files\n", len(supersedes)) + + for _, id := range supersedes { + err := repo.Backend().Remove(restic.Handle{ + Type: restic.IndexFile, + Name: id.String(), + }) + + if err != nil { + Warnf("error deleting old index %v: %v\n", id.Str(), err) + } + } + + return nil } diff --git a/src/restic/repository/index_rebuild.go b/src/restic/repository/index_rebuild.go deleted file mode 100644 index fcfc3027f..000000000 --- a/src/restic/repository/index_rebuild.go +++ /dev/null @@ -1,66 +0,0 @@ -package repository - -import ( - "fmt" - "os" - "restic" - "restic/debug" - "restic/list" - "restic/worker" -) - -// RebuildIndex lists all packs in the repo, writes a new index and removes all -// old indexes. This operation should only be done with an exclusive lock in -// place. -func RebuildIndex(repo restic.Repository) error { - debug.Log("start rebuilding index") - - done := make(chan struct{}) - defer close(done) - - ch := make(chan worker.Job) - go list.AllPacks(repo, ch, done) - - idx := NewIndex() - for job := range ch { - id := job.Data.(restic.ID) - - if job.Error != nil { - fmt.Fprintf(os.Stderr, "error for pack %v: %v\n", id, job.Error) - continue - } - - res := job.Result.(list.Result) - - for _, entry := range res.Entries() { - pb := restic.PackedBlob{ - Blob: entry, - PackID: res.PackID(), - } - idx.Store(pb) - } - } - - oldIndexes := restic.NewIDSet() - for id := range repo.List(restic.IndexFile, done) { - idx.AddToSupersedes(id) - oldIndexes.Insert(id) - } - - id, err := SaveIndex(repo, idx) - if err != nil { - debug.Log("error saving index: %v", err) - return err - } - debug.Log("new index saved as %v", id.Str()) - - for indexID := range oldIndexes { - h := restic.Handle{Type: restic.IndexFile, Name: indexID.String()} - err := repo.Backend().Remove(h) - if err != nil { - fmt.Fprintf(os.Stderr, "unable to remove index %v: %v\n", indexID.Str(), err) - } - } - - return nil -} diff --git a/src/restic/repository/repack_test.go b/src/restic/repository/repack_test.go index 6d910c97b..619454336 100644 --- a/src/restic/repository/repack_test.go +++ b/src/restic/repository/repack_test.go @@ -4,6 +4,7 @@ import ( "io" "math/rand" "restic" + "restic/index" "restic/repository" "testing" ) @@ -144,8 +145,24 @@ func saveIndex(t *testing.T, repo restic.Repository) { } func rebuildIndex(t *testing.T, repo restic.Repository) { - if err := repository.RebuildIndex(repo); err != nil { - t.Fatalf("error rebuilding index: %v", err) + idx, err := index.New(repo, nil) + if err != nil { + t.Fatal(err) + } + + for id := range repo.List(restic.IndexFile, nil) { + err = repo.Backend().Remove(restic.Handle{ + Type: restic.IndexFile, + Name: id.String(), + }) + if err != nil { + t.Fatal(err) + } + } + + _, err = idx.Save(repo, nil) + if err != nil { + t.Fatal(err) } } From f53d33ba34ae93383f037605d583d18cc3c58ff6 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Thu, 2 Mar 2017 15:45:35 +0100 Subject: [PATCH 2/2] Make ArchiveReader a struct --- src/cmds/restic/cmd_backup.go | 8 +++++- src/restic/archiver/archive_reader.go | 16 ++++++++--- src/restic/archiver/archive_reader_test.go | 32 +++++++++++++++++++--- 3 files changed, 47 insertions(+), 9 deletions(-) diff --git a/src/cmds/restic/cmd_backup.go b/src/cmds/restic/cmd_backup.go index 3d2514ed3..6c259ffa0 100644 --- a/src/cmds/restic/cmd_backup.go +++ b/src/cmds/restic/cmd_backup.go @@ -272,7 +272,13 @@ func readBackupFromStdin(opts BackupOptions, gopts GlobalOptions, args []string) return err } - _, id, err := archiver.ArchiveReader(repo, newArchiveStdinProgress(gopts), os.Stdin, opts.StdinFilename, opts.Tags, opts.Hostname) + r := &archiver.Reader{ + Repository: repo, + Tags: opts.Tags, + Hostname: opts.Hostname, + } + + _, id, err := r.Archive(opts.StdinFilename, os.Stdin, newArchiveStdinProgress(gopts)) if err != nil { return err } diff --git a/src/restic/archiver/archive_reader.go b/src/restic/archiver/archive_reader.go index 43beee69a..6ed72ab96 100644 --- a/src/restic/archiver/archive_reader.go +++ b/src/restic/archiver/archive_reader.go @@ -11,15 +11,22 @@ import ( "github.com/restic/chunker" ) -// ArchiveReader reads from the reader and archives the data. Returned is the -// resulting snapshot and its ID. -func ArchiveReader(repo restic.Repository, p *restic.Progress, rd io.Reader, name string, tags []string, hostname string) (*restic.Snapshot, restic.ID, error) { +// Reader allows saving a stream of data to the repository. +type Reader struct { + restic.Repository + + Tags []string + Hostname string +} + +// Archive reads data from the reader and saves it to the repo. +func (r *Reader) Archive(name string, rd io.Reader, p *restic.Progress) (*restic.Snapshot, restic.ID, error) { if name == "" { return nil, restic.ID{}, errors.New("no filename given") } debug.Log("start archiving %s", name) - sn, err := restic.NewSnapshot([]string{name}, tags, hostname) + sn, err := restic.NewSnapshot([]string{name}, r.Tags, r.Hostname) if err != nil { return nil, restic.ID{}, err } @@ -27,6 +34,7 @@ func ArchiveReader(repo restic.Repository, p *restic.Progress, rd io.Reader, nam p.Start() defer p.Done() + repo := r.Repository chnker := chunker.New(rd, repo.Config().ChunkerPolynomial) ids := restic.IDs{} diff --git a/src/restic/archiver/archive_reader_test.go b/src/restic/archiver/archive_reader_test.go index bdcc2a1e8..a8ab18668 100644 --- a/src/restic/archiver/archive_reader_test.go +++ b/src/restic/archiver/archive_reader_test.go @@ -79,7 +79,13 @@ func TestArchiveReader(t *testing.T) { f := fakeFile(t, seed, size) - sn, id, err := ArchiveReader(repo, nil, f, "fakefile", []string{"test"}, "localhost") + r := &Reader{ + Repository: repo, + Hostname: "localhost", + Tags: []string{"test"}, + } + + sn, id, err := r.Archive("fakefile", f, nil) if err != nil { t.Fatalf("ArchiveReader() returned error %v", err) } @@ -99,7 +105,13 @@ func TestArchiveReaderNull(t *testing.T) { repo, cleanup := repository.TestRepository(t) defer cleanup() - sn, id, err := ArchiveReader(repo, nil, bytes.NewReader(nil), "fakefile", nil, "localhost") + r := &Reader{ + Repository: repo, + Hostname: "localhost", + Tags: []string{"test"}, + } + + sn, id, err := r.Archive("fakefile", bytes.NewReader(nil), nil) if err != nil { t.Fatalf("ArchiveReader() returned error %v", err) } @@ -134,7 +146,13 @@ func TestArchiveReaderError(t *testing.T) { repo, cleanup := repository.TestRepository(t) defer cleanup() - sn, id, err := ArchiveReader(repo, nil, errReader("error returned by reading stdin"), "fakefile", nil, "localhost") + r := &Reader{ + Repository: repo, + Hostname: "localhost", + Tags: []string{"test"}, + } + + sn, id, err := r.Archive("fakefile", errReader("error returned by reading stdin"), nil) if err == nil { t.Errorf("expected error not returned") } @@ -167,11 +185,17 @@ func BenchmarkArchiveReader(t *testing.B) { t.Fatal(err) } + r := &Reader{ + Repository: repo, + Hostname: "localhost", + Tags: []string{"test"}, + } + t.SetBytes(size) t.ResetTimer() for i := 0; i < t.N; i++ { - _, _, err := ArchiveReader(repo, nil, bytes.NewReader(buf), "fakefile", []string{"test"}, "localhost") + _, _, err := r.Archive("fakefile", bytes.NewReader(buf), nil) if err != nil { t.Fatal(err) }