mirror of
https://github.com/octoleo/restic.git
synced 2024-12-22 10:58:55 +00:00
implement progress bar for index loading
This commit is contained in:
parent
eac1c4a8d0
commit
ed65a7dbca
@ -546,7 +546,10 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
|
|||||||
if !gopts.JSON {
|
if !gopts.JSON {
|
||||||
progressPrinter.V("load index files")
|
progressPrinter.V("load index files")
|
||||||
}
|
}
|
||||||
err = repo.LoadIndex(ctx)
|
|
||||||
|
bar := newTerminalProgressMax(!gopts.Quiet, 0, "index files loaded", term)
|
||||||
|
|
||||||
|
err = repo.LoadIndex(ctx, bar)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -252,7 +252,7 @@ func TestBackupTreeLoadError(t *testing.T) {
|
|||||||
|
|
||||||
r, err := OpenRepository(context.TODO(), env.gopts)
|
r, err := OpenRepository(context.TODO(), env.gopts)
|
||||||
rtest.OK(t, err)
|
rtest.OK(t, err)
|
||||||
rtest.OK(t, r.LoadIndex(context.TODO()))
|
rtest.OK(t, r.LoadIndex(context.TODO(), nil))
|
||||||
treePacks := restic.NewIDSet()
|
treePacks := restic.NewIDSet()
|
||||||
r.Index().Each(context.TODO(), func(pb restic.PackedBlob) {
|
r.Index().Each(context.TODO(), func(pb restic.PackedBlob) {
|
||||||
if pb.Type == restic.TreeBlob {
|
if pb.Type == restic.TreeBlob {
|
||||||
|
@ -169,7 +169,8 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
|
|||||||
return err
|
return err
|
||||||
|
|
||||||
case "blob":
|
case "blob":
|
||||||
err = repo.LoadIndex(ctx)
|
bar := newProgressMax(!gopts.Quiet, 0, "index files loaded")
|
||||||
|
err = repo.LoadIndex(ctx, bar)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@ -197,7 +198,8 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
|
|||||||
return errors.Fatalf("could not find snapshot: %v\n", err)
|
return errors.Fatalf("could not find snapshot: %v\n", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repo.LoadIndex(ctx)
|
bar := newProgressMax(!gopts.Quiet, 0, "index files loaded")
|
||||||
|
err = repo.LoadIndex(ctx, bar)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -99,12 +99,13 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []
|
|||||||
}
|
}
|
||||||
|
|
||||||
debug.Log("Loading source index")
|
debug.Log("Loading source index")
|
||||||
if err := srcRepo.LoadIndex(ctx); err != nil {
|
bar := newProgressMax(!gopts.Quiet, 0, "index files loaded")
|
||||||
|
if err := srcRepo.LoadIndex(ctx, bar); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
bar = newProgressMax(!gopts.Quiet, 0, "index files loaded")
|
||||||
debug.Log("Loading destination index")
|
debug.Log("Loading destination index")
|
||||||
if err := dstRepo.LoadIndex(ctx); err != nil {
|
if err := dstRepo.LoadIndex(ctx, bar); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -469,7 +469,8 @@ func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) er
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repo.LoadIndex(ctx)
|
bar := newProgressMax(!gopts.Quiet, 0, "index files loaded")
|
||||||
|
err = repo.LoadIndex(ctx, bar)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -363,8 +363,8 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []
|
|||||||
if !gopts.JSON {
|
if !gopts.JSON {
|
||||||
Verbosef("comparing snapshot %v to %v:\n\n", sn1.ID().Str(), sn2.ID().Str())
|
Verbosef("comparing snapshot %v to %v:\n\n", sn1.ID().Str(), sn2.ID().Str())
|
||||||
}
|
}
|
||||||
|
bar := newProgressMax(!gopts.Quiet, 0, "index files loaded")
|
||||||
if err = repo.LoadIndex(ctx); err != nil {
|
if err = repo.LoadIndex(ctx, bar); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -152,7 +152,8 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args []
|
|||||||
return errors.Fatalf("failed to find snapshot: %v", err)
|
return errors.Fatalf("failed to find snapshot: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repo.LoadIndex(ctx)
|
bar := newProgressMax(!gopts.Quiet, 0, "index files loaded")
|
||||||
|
err = repo.LoadIndex(ctx, bar)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -589,8 +589,8 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args []
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
bar := newProgressMax(!gopts.Quiet, 0, "index files loaded")
|
||||||
if err = repo.LoadIndex(ctx); err != nil {
|
if err = repo.LoadIndex(ctx, bar); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -173,7 +173,8 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = repo.LoadIndex(ctx); err != nil {
|
bar := newProgressMax(!gopts.Quiet, 0, "index files loaded")
|
||||||
|
if err = repo.LoadIndex(ctx, bar); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -130,7 +130,8 @@ func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repo.LoadIndex(ctx)
|
bar := newProgressMax(!gopts.Quiet, 0, "index files loaded")
|
||||||
|
err = repo.LoadIndex(ctx, bar)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -187,7 +187,8 @@ func runPruneWithRepo(ctx context.Context, opts PruneOptions, gopts GlobalOption
|
|||||||
|
|
||||||
Verbosef("loading indexes...\n")
|
Verbosef("loading indexes...\n")
|
||||||
// loading the index before the snapshots is ok, as we use an exclusive lock here
|
// loading the index before the snapshots is ok, as we use an exclusive lock here
|
||||||
err := repo.LoadIndex(ctx)
|
bar := newProgressMax(!gopts.Quiet, 0, "index files loaded")
|
||||||
|
err := repo.LoadIndex(ctx, bar)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -58,7 +58,8 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
Verbosef("load index files\n")
|
Verbosef("load index files\n")
|
||||||
if err = repo.LoadIndex(ctx); err != nil {
|
bar := newProgressMax(!gopts.Quiet, 0, "index files loaded")
|
||||||
|
if err = repo.LoadIndex(ctx, bar); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -73,7 +74,7 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error {
|
|||||||
})
|
})
|
||||||
|
|
||||||
Verbosef("load %d trees\n", len(trees))
|
Verbosef("load %d trees\n", len(trees))
|
||||||
bar := newProgressMax(!gopts.Quiet, uint64(len(trees)), "trees loaded")
|
bar = newProgressMax(!gopts.Quiet, uint64(len(trees)), "trees loaded")
|
||||||
for id := range trees {
|
for id := range trees {
|
||||||
tree, err := restic.LoadTree(ctx, repo, id)
|
tree, err := restic.LoadTree(ctx, repo, id)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -89,7 +89,8 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := repo.LoadIndex(ctx); err != nil {
|
bar := newProgressMax(!gopts.Quiet, 0, "index files loaded")
|
||||||
|
if err := repo.LoadIndex(ctx, bar); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -173,7 +173,8 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
|
|||||||
return errors.Fatalf("failed to find snapshot: %v", err)
|
return errors.Fatalf("failed to find snapshot: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repo.LoadIndex(ctx)
|
bar := newTerminalProgressMax(!gopts.Quiet, 0, "index files loaded", term)
|
||||||
|
err = repo.LoadIndex(ctx, bar)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -212,7 +212,8 @@ func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, a
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = repo.LoadIndex(ctx); err != nil {
|
bar := newProgressMax(!gopts.Quiet, 0, "index files loaded")
|
||||||
|
if err = repo.LoadIndex(ctx, bar); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -98,8 +98,8 @@ func runStats(ctx context.Context, opts StatsOptions, gopts GlobalOptions, args
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
bar := newProgressMax(!gopts.Quiet, 0, "index files loaded")
|
||||||
if err = repo.LoadIndex(ctx); err != nil {
|
if err = repo.LoadIndex(ctx, bar); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -258,7 +258,7 @@ func removePacksExcept(gopts GlobalOptions, t testing.TB, keep restic.IDSet, rem
|
|||||||
rtest.OK(t, err)
|
rtest.OK(t, err)
|
||||||
|
|
||||||
// Get all tree packs
|
// Get all tree packs
|
||||||
rtest.OK(t, r.LoadIndex(context.TODO()))
|
rtest.OK(t, r.LoadIndex(context.TODO(), nil))
|
||||||
|
|
||||||
treePacks := restic.NewIDSet()
|
treePacks := restic.NewIDSet()
|
||||||
r.Index().Each(context.TODO(), func(pb restic.PackedBlob) {
|
r.Index().Each(context.TODO(), func(pb restic.PackedBlob) {
|
||||||
|
@ -29,13 +29,12 @@ func calculateProgressInterval(show bool, json bool) time.Duration {
|
|||||||
return interval
|
return interval
|
||||||
}
|
}
|
||||||
|
|
||||||
// newProgressMax returns a progress.Counter that prints to stdout.
|
// newTerminalProgressMax returns a progress.Counter that prints to stdout or terminal if provided.
|
||||||
func newProgressMax(show bool, max uint64, description string) *progress.Counter {
|
func newGenericProgressMax(show bool, max uint64, description string, print func(status string)) *progress.Counter {
|
||||||
if !show {
|
if !show {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
interval := calculateProgressInterval(show, false)
|
interval := calculateProgressInterval(show, false)
|
||||||
canUpdateStatus := stdoutCanUpdateStatus()
|
|
||||||
|
|
||||||
return progress.NewCounter(interval, max, func(v uint64, max uint64, d time.Duration, final bool) {
|
return progress.NewCounter(interval, max, func(v uint64, max uint64, d time.Duration, final bool) {
|
||||||
var status string
|
var status string
|
||||||
@ -47,14 +46,28 @@ func newProgressMax(show bool, max uint64, description string) *progress.Counter
|
|||||||
ui.FormatDuration(d), ui.FormatPercent(v, max), v, max, description)
|
ui.FormatDuration(d), ui.FormatPercent(v, max), v, max, description)
|
||||||
}
|
}
|
||||||
|
|
||||||
printProgress(status, canUpdateStatus)
|
print(status)
|
||||||
if final {
|
if final {
|
||||||
fmt.Print("\n")
|
fmt.Print("\n")
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func printProgress(status string, canUpdateStatus bool) {
|
func newTerminalProgressMax(show bool, max uint64, description string, term *termstatus.Terminal) *progress.Counter {
|
||||||
|
return newGenericProgressMax(show, max, description, func(status string) {
|
||||||
|
term.SetStatus([]string{status})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// newProgressMax calls newTerminalProgress without a terminal (print to stdout)
|
||||||
|
func newProgressMax(show bool, max uint64, description string) *progress.Counter {
|
||||||
|
return newGenericProgressMax(show, max, description, printProgress)
|
||||||
|
}
|
||||||
|
|
||||||
|
func printProgress(status string) {
|
||||||
|
|
||||||
|
canUpdateStatus := stdoutCanUpdateStatus()
|
||||||
|
|
||||||
w := stdoutTerminalWidth()
|
w := stdoutTerminalWidth()
|
||||||
if w > 0 {
|
if w > 0 {
|
||||||
if w < 3 {
|
if w < 3 {
|
||||||
|
@ -328,7 +328,7 @@ func (d *SnapshotsDirStructure) updateSnapshots(ctx context.Context) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
err = d.root.repo.LoadIndex(ctx)
|
err = d.root.repo.LoadIndex(ctx, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -357,7 +357,7 @@ func TestIndexSave(t *testing.T) {
|
|||||||
func testIndexSave(t *testing.T, version uint) {
|
func testIndexSave(t *testing.T, version uint) {
|
||||||
repo := createFilledRepo(t, 3, version)
|
repo := createFilledRepo(t, 3, version)
|
||||||
|
|
||||||
err := repo.LoadIndex(context.TODO())
|
err := repo.LoadIndex(context.TODO(), nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
@ -213,7 +213,7 @@ func reloadIndex(t *testing.T, repo restic.Repository) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := repo.LoadIndex(context.TODO()); err != nil {
|
if err := repo.LoadIndex(context.TODO(), nil); err != nil {
|
||||||
t.Fatalf("error loading new index: %v", err)
|
t.Fatalf("error loading new index: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -581,15 +581,30 @@ func (r *Repository) SetIndex(i restic.MasterIndex) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// LoadIndex loads all index files from the backend in parallel and stores them
|
// LoadIndex loads all index files from the backend in parallel and stores them
|
||||||
// in the master index. The first error that occurred is returned.
|
func (r *Repository) LoadIndex(ctx context.Context, p *progress.Counter) error {
|
||||||
func (r *Repository) LoadIndex(ctx context.Context) error {
|
|
||||||
debug.Log("Loading index")
|
debug.Log("Loading index")
|
||||||
|
|
||||||
|
if p != nil {
|
||||||
|
var numIndexFiles uint64
|
||||||
|
err := r.List(ctx, restic.IndexFile, func(id restic.ID, size int64) error {
|
||||||
|
numIndexFiles++
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
p.SetMax(numIndexFiles)
|
||||||
|
defer p.Done()
|
||||||
|
}
|
||||||
|
|
||||||
err := index.ForAllIndexes(ctx, r, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
|
err := index.ForAllIndexes(ctx, r, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
r.idx.Insert(idx)
|
r.idx.Insert(idx)
|
||||||
|
if p != nil {
|
||||||
|
p.Add(1)
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
|
|
||||||
|
@ -255,7 +255,7 @@ func TestRepositoryLoadIndex(t *testing.T) {
|
|||||||
defer cleanup()
|
defer cleanup()
|
||||||
|
|
||||||
repo := repository.TestOpenLocal(t, repodir)
|
repo := repository.TestOpenLocal(t, repodir)
|
||||||
rtest.OK(t, repo.LoadIndex(context.TODO()))
|
rtest.OK(t, repo.LoadIndex(context.TODO(), nil))
|
||||||
}
|
}
|
||||||
|
|
||||||
// loadIndex loads the index id from backend and returns it.
|
// loadIndex loads the index id from backend and returns it.
|
||||||
@ -324,7 +324,7 @@ func TestRepositoryLoadUnpackedRetryBroken(t *testing.T) {
|
|||||||
err = repo.SearchKey(context.TODO(), rtest.TestPassword, 10, "")
|
err = repo.SearchKey(context.TODO(), rtest.TestPassword, 10, "")
|
||||||
rtest.OK(t, err)
|
rtest.OK(t, err)
|
||||||
|
|
||||||
rtest.OK(t, repo.LoadIndex(context.TODO()))
|
rtest.OK(t, repo.LoadIndex(context.TODO(), nil))
|
||||||
}
|
}
|
||||||
|
|
||||||
func BenchmarkLoadIndex(b *testing.B) {
|
func BenchmarkLoadIndex(b *testing.B) {
|
||||||
|
@ -24,7 +24,7 @@ type Repository interface {
|
|||||||
Key() *crypto.Key
|
Key() *crypto.Key
|
||||||
|
|
||||||
Index() MasterIndex
|
Index() MasterIndex
|
||||||
LoadIndex(context.Context) error
|
LoadIndex(context.Context, *progress.Counter) error
|
||||||
SetIndex(MasterIndex) error
|
SetIndex(MasterIndex) error
|
||||||
LookupBlobSize(ID, BlobType) (uint, bool)
|
LookupBlobSize(ID, BlobType) (uint, bool)
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user