octoleo
/
restic
mirror of https://github.com/octoleo/restic.git
2
2
Fork 0
Code Issues Releases Wiki Activity

implement progress bar for index loading

This commit is contained in:
arjunajesh 2023-07-15 22:48:30 -04:00 committed by Michael Eischer
parent eac1c4a8d0
commit ed65a7dbca
24 changed files with 80 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
cmd/restic/cmd_backup.go
View File

@ -546,7 +546,10 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
if !gopts.JSON {
progressPrinter.V("load index files")
}
err = repo.LoadIndex(ctx)
bar := newTerminalProgressMax(!gopts.Quiet, 0, "index files loaded", term)
err = repo.LoadIndex(ctx, bar)
if err != nil {
return err
}

2
cmd/restic/cmd_backup_integration_test.go
View File

@ -252,7 +252,7 @@ func TestBackupTreeLoadError(t *testing.T) {
r, err := OpenRepository(context.TODO(), env.gopts)
rtest.OK(t, err)
rtest.OK(t, r.LoadIndex(context.TODO()))
rtest.OK(t, r.LoadIndex(context.TODO(), nil))
treePacks := restic.NewIDSet()
r.Index().Each(context.TODO(), func(pb restic.PackedBlob) {
if pb.Type == restic.TreeBlob {

6
cmd/restic/cmd_cat.go
View File

@ -169,7 +169,8 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
return err
case "blob":
err = repo.LoadIndex(ctx)
bar := newProgressMax(!gopts.Quiet, 0, "index files loaded")
err = repo.LoadIndex(ctx, bar)
if err != nil {
return err
}
@ -197,7 +198,8 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
return errors.Fatalf("could not find snapshot: %v\n", err)
}
err = repo.LoadIndex(ctx)
bar := newProgressMax(!gopts.Quiet, 0, "index files loaded")
err = repo.LoadIndex(ctx, bar)
if err != nil {
return err
}

7
cmd/restic/cmd_copy.go
View File

@ -99,12 +99,13 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []
}
debug.Log("Loading source index")
if err := srcRepo.LoadIndex(ctx); err != nil {
bar := newProgressMax(!gopts.Quiet, 0, "index files loaded")
if err := srcRepo.LoadIndex(ctx, bar); err != nil {
return err
}
bar = newProgressMax(!gopts.Quiet, 0, "index files loaded")
debug.Log("Loading destination index")
if err := dstRepo.LoadIndex(ctx); err != nil {
if err := dstRepo.LoadIndex(ctx, bar); err != nil {
return err
}

3
cmd/restic/cmd_debug.go
View File

@ -469,7 +469,8 @@ func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) er
}
}
err = repo.LoadIndex(ctx)
bar := newProgressMax(!gopts.Quiet, 0, "index files loaded")
err = repo.LoadIndex(ctx, bar)
if err != nil {
return err
}

4
cmd/restic/cmd_diff.go
View File

@ -363,8 +363,8 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []
if !gopts.JSON {
Verbosef("comparing snapshot %v to %v:\n\n", sn1.ID().Str(), sn2.ID().Str())
}
if err = repo.LoadIndex(ctx); err != nil {
bar := newProgressMax(!gopts.Quiet, 0, "index files loaded")
if err = repo.LoadIndex(ctx, bar); err != nil {
return err
}

3
cmd/restic/cmd_dump.go
View File

@ -152,7 +152,8 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args []
return errors.Fatalf("failed to find snapshot: %v", err)
}
err = repo.LoadIndex(ctx)
bar := newProgressMax(!gopts.Quiet, 0, "index files loaded")
err = repo.LoadIndex(ctx, bar)
if err != nil {
return err
}

4
cmd/restic/cmd_find.go
View File

@ -589,8 +589,8 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args []
if err != nil {
return err
}
if err = repo.LoadIndex(ctx); err != nil {
bar := newProgressMax(!gopts.Quiet, 0, "index files loaded")
if err = repo.LoadIndex(ctx, bar); err != nil {
return err
}

3
cmd/restic/cmd_ls.go
View File

@ -173,7 +173,8 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
return err
}
if err = repo.LoadIndex(ctx); err != nil {
bar := newProgressMax(!gopts.Quiet, 0, "index files loaded")
if err = repo.LoadIndex(ctx, bar); err != nil {
return err
}

3
cmd/restic/cmd_mount.go
View File

@ -130,7 +130,8 @@ func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args
}
}
err = repo.LoadIndex(ctx)
bar := newProgressMax(!gopts.Quiet, 0, "index files loaded")
err = repo.LoadIndex(ctx, bar)
if err != nil {
return err
}

3
cmd/restic/cmd_prune.go
View File

@ -187,7 +187,8 @@ func runPruneWithRepo(ctx context.Context, opts PruneOptions, gopts GlobalOption
Verbosef("loading indexes...\n")
// loading the index before the snapshots is ok, as we use an exclusive lock here
err := repo.LoadIndex(ctx)
bar := newProgressMax(!gopts.Quiet, 0, "index files loaded")
err := repo.LoadIndex(ctx, bar)
if err != nil {
return err
}

5
cmd/restic/cmd_recover.go
View File

@ -58,7 +58,8 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error {
}
Verbosef("load index files\n")
if err = repo.LoadIndex(ctx); err != nil {
bar := newProgressMax(!gopts.Quiet, 0, "index files loaded")
if err = repo.LoadIndex(ctx, bar); err != nil {
return err
}
@ -73,7 +74,7 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error {
})
Verbosef("load %d trees\n", len(trees))
bar := newProgressMax(!gopts.Quiet, uint64(len(trees)), "trees loaded")
bar = newProgressMax(!gopts.Quiet, uint64(len(trees)), "trees loaded")
for id := range trees {
tree, err := restic.LoadTree(ctx, repo, id)
if err != nil {

3
cmd/restic/cmd_repair_snapshots.go
View File

@ -89,7 +89,8 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt
return err
}
if err := repo.LoadIndex(ctx); err != nil {
bar := newProgressMax(!gopts.Quiet, 0, "index files loaded")
if err := repo.LoadIndex(ctx, bar); err != nil {
return err
}

3
cmd/restic/cmd_restore.go
View File

@ -173,7 +173,8 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
return errors.Fatalf("failed to find snapshot: %v", err)
}
err = repo.LoadIndex(ctx)
bar := newTerminalProgressMax(!gopts.Quiet, 0, "index files loaded", term)
err = repo.LoadIndex(ctx, bar)
if err != nil {
return err
}

3
cmd/restic/cmd_rewrite.go
View File

@ -212,7 +212,8 @@ func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, a
return err
}
if err = repo.LoadIndex(ctx); err != nil {
bar := newProgressMax(!gopts.Quiet, 0, "index files loaded")
if err = repo.LoadIndex(ctx, bar); err != nil {
return err
}

4
cmd/restic/cmd_stats.go
View File

@ -98,8 +98,8 @@ func runStats(ctx context.Context, opts StatsOptions, gopts GlobalOptions, args
if err != nil {
return err
}
if err = repo.LoadIndex(ctx); err != nil {
bar := newProgressMax(!gopts.Quiet, 0, "index files loaded")
if err = repo.LoadIndex(ctx, bar); err != nil {
return err
}

2
cmd/restic/integration_helpers_test.go
View File

@ -258,7 +258,7 @@ func removePacksExcept(gopts GlobalOptions, t testing.TB, keep restic.IDSet, rem
rtest.OK(t, err)
// Get all tree packs
rtest.OK(t, r.LoadIndex(context.TODO()))
rtest.OK(t, r.LoadIndex(context.TODO(), nil))
treePacks := restic.NewIDSet()
r.Index().Each(context.TODO(), func(pb restic.PackedBlob) {

23
cmd/restic/progress.go
View File

@ -29,13 +29,12 @@ func calculateProgressInterval(show bool, json bool) time.Duration {
return interval
}
// newProgressMax returns a progress.Counter that prints to stdout.
func newProgressMax(show bool, max uint64, description string) *progress.Counter {
// newTerminalProgressMax returns a progress.Counter that prints to stdout or terminal if provided.
func newGenericProgressMax(show bool, max uint64, description string, print func(status string)) *progress.Counter {
if !show {
return nil
}
interval := calculateProgressInterval(show, false)
canUpdateStatus := stdoutCanUpdateStatus()
return progress.NewCounter(interval, max, func(v uint64, max uint64, d time.Duration, final bool) {
var status string
@ -47,14 +46,28 @@ func newProgressMax(show bool, max uint64, description string) *progress.Counter
ui.FormatDuration(d), ui.FormatPercent(v, max), v, max, description)
}
printProgress(status, canUpdateStatus)
print(status)
if final {
fmt.Print("\n")
}
})
}
func printProgress(status string, canUpdateStatus bool) {
func newTerminalProgressMax(show bool, max uint64, description string, term *termstatus.Terminal) *progress.Counter {
return newGenericProgressMax(show, max, description, func(status string) {
term.SetStatus([]string{status})
})
}
// newProgressMax calls newTerminalProgress without a terminal (print to stdout)
func newProgressMax(show bool, max uint64, description string) *progress.Counter {
return newGenericProgressMax(show, max, description, printProgress)
}
func printProgress(status string) {
canUpdateStatus := stdoutCanUpdateStatus()
w := stdoutTerminalWidth()
if w > 0 {
if w < 3 {

2
internal/fuse/snapshots_dirstruct.go
View File

@ -328,7 +328,7 @@ func (d *SnapshotsDirStructure) updateSnapshots(ctx context.Context) error {
return nil
}
err = d.root.repo.LoadIndex(ctx)
err = d.root.repo.LoadIndex(ctx, nil)
if err != nil {
return err
}

2
internal/index/master_index_test.go
View File

@ -357,7 +357,7 @@ func TestIndexSave(t *testing.T) {
func testIndexSave(t *testing.T, version uint) {
repo := createFilledRepo(t, 3, version)
err := repo.LoadIndex(context.TODO())
err := repo.LoadIndex(context.TODO(), nil)
if err != nil {
t.Fatal(err)
}

2
internal/repository/repack_test.go
View File

@ -213,7 +213,7 @@ func reloadIndex(t *testing.T, repo restic.Repository) {
t.Fatal(err)
}
if err := repo.LoadIndex(context.TODO()); err != nil {
if err := repo.LoadIndex(context.TODO(), nil); err != nil {
t.Fatalf("error loading new index: %v", err)
}
}

19
internal/repository/repository.go
View File

@ -581,15 +581,30 @@ func (r *Repository) SetIndex(i restic.MasterIndex) error {
}
// LoadIndex loads all index files from the backend in parallel and stores them
// in the master index. The first error that occurred is returned.
func (r *Repository) LoadIndex(ctx context.Context) error {
func (r *Repository) LoadIndex(ctx context.Context, p *progress.Counter) error {
debug.Log("Loading index")
if p != nil {
var numIndexFiles uint64
err := r.List(ctx, restic.IndexFile, func(id restic.ID, size int64) error {
numIndexFiles++
return nil
})
if err != nil {
return err
}
p.SetMax(numIndexFiles)
defer p.Done()
}
err := index.ForAllIndexes(ctx, r, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
if err != nil {
return err
}
r.idx.Insert(idx)
if p != nil {
p.Add(1)
}
return nil
})

4
internal/repository/repository_test.go
View File

@ -255,7 +255,7 @@ func TestRepositoryLoadIndex(t *testing.T) {
defer cleanup()
repo := repository.TestOpenLocal(t, repodir)
rtest.OK(t, repo.LoadIndex(context.TODO()))
rtest.OK(t, repo.LoadIndex(context.TODO(), nil))
}
// loadIndex loads the index id from backend and returns it.
@ -324,7 +324,7 @@ func TestRepositoryLoadUnpackedRetryBroken(t *testing.T) {
err = repo.SearchKey(context.TODO(), rtest.TestPassword, 10, "")
rtest.OK(t, err)
rtest.OK(t, repo.LoadIndex(context.TODO()))
rtest.OK(t, repo.LoadIndex(context.TODO(), nil))
}
func BenchmarkLoadIndex(b *testing.B) {

2
internal/restic/repository.go
View File

@ -24,7 +24,7 @@ type Repository interface {
Key() *crypto.Key
Index() MasterIndex
LoadIndex(context.Context) error
LoadIndex(context.Context, *progress.Counter) error
SetIndex(MasterIndex) error
LookupBlobSize(ID, BlobType) (uint, bool)