From 086993bae11d4d639a1e1904e3b245f62aeb91ab Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 29 Sep 2018 14:16:01 +0200 Subject: [PATCH] debug: check packs not in index, implement repair --- cmd/restic/cmd_debug.go | 125 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 117 insertions(+), 8 deletions(-) diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index 80b94b980..934628771 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -8,10 +8,14 @@ import ( "fmt" "io" "os" + "runtime" "sort" + "time" "github.com/spf13/cobra" + "golang.org/x/sync/errgroup" + "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/pack" "github.com/restic/restic/internal/repository" @@ -41,10 +45,13 @@ Exit status is 0 if the command was successful, and non-zero if there was any er }, } +var tryRepair bool + func init() { cmdRoot.AddCommand(cmdDebug) cmdDebug.AddCommand(cmdDebugDump) cmdDebug.AddCommand(cmdDebugExamine) + cmdDebugExamine.Flags().BoolVar(&tryRepair, "try-repair", false, "try to repair broken blobs with single bit flips") } func prettyPrintJSON(wr io.Writer, item interface{}) error { @@ -178,7 +185,87 @@ var cmdDebugExamine = &cobra.Command{ }, } -func loadBlobs(ctx context.Context, repo restic.Repository, pack string, list []restic.PackedBlob) error { +func tryRepairWithBitflip(ctx context.Context, key *crypto.Key, input []byte) { + fmt.Printf(" trying to repair blob with single bit flip\n") + + ch := make(chan int) + var wg errgroup.Group + done := make(chan struct{}) + + fmt.Printf(" spinning up %d worker functions\n", runtime.NumCPU()) + for i := 0; i < runtime.NumCPU(); i++ { + wg.Go(func() error { + // make a local copy of the buffer + buf := make([]byte, len(input)) + copy(buf, input) + + for { + select { + case <-done: + return nil + case i := <-ch: + for j := 0; j < 7; j++ { + // flip bit + buf[i] ^= (1 << uint(j)) + + nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():] + plaintext, err := key.Open(plaintext[:0], nonce, plaintext, nil) + if err == nil { + fmt.Printf("\n") + fmt.Printf(" blob could be repaired by flipping bit %v in byte %v\n", j, i) + fmt.Printf(" hash is %v\n", restic.Hash(plaintext)) + close(done) + return nil + } + + // flip bit back + buf[i] ^= (1 << uint(j)) + } + } + } + }) + } + + start := time.Now() + info := time.Now() +outer: + for i := range input { + select { + case ch <- i: + case <-done: + fmt.Printf(" done after %v\n", time.Since(start)) + break outer + } + + if time.Since(info) > time.Second { + secs := time.Since(start).Seconds() + gps := float64(i) / secs + remaining := len(input) - i + eta := time.Duration(float64(remaining)/gps) * time.Second + + fmt.Printf("\r%d byte of %d done (%.2f%%), %.2f guesses per second, ETA %v", + i, len(input), float32(i)/float32(len(input)), + gps, eta) + info = time.Now() + } + } + + var found bool + select { + case <-done: + found = true + default: + close(done) + } + + wg.Wait() + + if !found { + fmt.Printf("\n blob could not be repaired by single bit flip\n") + } +} + +func loadBlobs(ctx context.Context, repo restic.Repository, pack string, list []restic.Blob) error { be := repo.Backend() for _, blob := range list { fmt.Printf(" loading blob %v at %v (length %v)\n", blob.ID, blob.Offset, blob.Length) @@ -202,15 +289,18 @@ func loadBlobs(ctx context.Context, repo restic.Repository, pack string, list [] key := repo.Key() - nonce, buf := buf[:key.NonceSize()], buf[key.NonceSize():] - buf, err = key.Open(buf[:0], nonce, buf, nil) + nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():] + plaintext, err = key.Open(plaintext[:0], nonce, plaintext, nil) if err != nil { fmt.Fprintf(os.Stderr, "error decrypting blob: %v\n", err) + if tryRepair { + tryRepairWithBitflip(ctx, key, buf) + } continue } - id := restic.Hash(buf) - fmt.Printf(" successfully decrypted blob (length %v), hash is %v\n", len(buf), id) + id := restic.Hash(plaintext) + fmt.Printf(" successfully decrypted blob (length %v), hash is %v\n", len(plaintext), id) if !id.Equal(blob.ID) { fmt.Printf(" IDs do not match, want %v, got %v\n", blob.ID, id) } else { @@ -240,6 +330,7 @@ func runDebugExamine(gopts GlobalOptions, args []string) error { return err } + blobsLoaded := false for _, name := range args { fmt.Printf("examine %v\n", name) id, err := restic.ParseID(name) @@ -257,6 +348,8 @@ func runDebugExamine(gopts GlobalOptions, args []string) error { } fmt.Printf(" file size is %v\n", fi.Size) + fmt.Printf(" ========================================\n") + fmt.Printf(" looking for info in the indexes\n") // examine all data the indexes have for the pack file for _, idx := range repo.Index().(*repository.MasterIndex).All() { @@ -267,7 +360,6 @@ func runDebugExamine(gopts GlobalOptions, args []string) error { blobs := idx.ListPack(id) if len(blobs) == 0 { - fmt.Printf(" index %v does not contain the file\n", idxIDs) continue } @@ -300,13 +392,23 @@ func runDebugExamine(gopts GlobalOptions, args []string) error { fmt.Printf(" file sizes match\n") } - err = loadBlobs(gopts.ctx, repo, name, blobs) + // convert list of blobs to []restic.Blob + var list []restic.Blob + for _, b := range blobs { + list = append(list, b.Blob) + } + + err = loadBlobs(gopts.ctx, repo, name, list) if err != nil { fmt.Fprintf(os.Stderr, "error: %v\n", err) + } else { + blobsLoaded = true } } - // inspect the pack file itself + fmt.Printf(" ========================================\n") + fmt.Printf(" inspect the pack itself\n") + blobs, _, err := pack.List(repo.Key(), restic.ReaderAt(gopts.ctx, repo.Backend(), h), fi.Size) if err != nil { fmt.Fprintf(os.Stderr, "error for pack %v: %v\n", id.Str(), err) @@ -339,6 +441,13 @@ func runDebugExamine(gopts GlobalOptions, args []string) error { } else { fmt.Printf(" file sizes match\n") } + + if !blobsLoaded { + err = loadBlobs(gopts.ctx, repo, name, blobs) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + } + } } return nil }