mirror of
https://github.com/octoleo/restic.git
synced 2024-12-22 19:08:55 +00:00
Merge pull request #3730 from MichaelEischer/stricter-check
Let `check` warn about legacy variants of the repo format version 1
This commit is contained in:
commit
049f4c4144
12
changelog/unreleased/issue-3295
Normal file
12
changelog/unreleased/issue-3295
Normal file
@ -0,0 +1,12 @@
|
||||
Change: Deprecate `check --check-unused` and add further checks
|
||||
|
||||
Since restic 0.12.0, it is expected to still have unused blobs after running
|
||||
`prune`. This made the `check --check-unused` rather useless and tended to
|
||||
confuse users. The options has been deprecated and is now ignored.
|
||||
|
||||
`check` now also warns if a repository is using either the legacy S3 layout or
|
||||
mixed pack files with both tree and data blobs. The latter is known to cause
|
||||
performance problems.
|
||||
|
||||
https://github.com/restic/restic/issues/3295
|
||||
https://github.com/restic/restic/pull/3730
|
@ -57,7 +57,13 @@ func init() {
|
||||
f := cmdCheck.Flags()
|
||||
f.BoolVar(&checkOptions.ReadData, "read-data", false, "read all data blobs")
|
||||
f.StringVar(&checkOptions.ReadDataSubset, "read-data-subset", "", "read a `subset` of data packs, specified as 'n/t' for specific part, or either 'x%' or 'x.y%' or a size in bytes with suffixes k/K, m/M, g/G, t/T for a random subset")
|
||||
f.BoolVar(&checkOptions.CheckUnused, "check-unused", false, "find unused blobs")
|
||||
var ignored bool
|
||||
f.BoolVar(&ignored, "check-unused", false, "find unused blobs")
|
||||
err := f.MarkDeprecated("check-unused", "`--check-unused` is deprecated and will be ignored")
|
||||
if err != nil {
|
||||
// MarkDeprecated only returns an error when the flag is not found
|
||||
panic(err)
|
||||
}
|
||||
f.BoolVar(&checkOptions.WithCache, "with-cache", false, "use the cache")
|
||||
}
|
||||
|
||||
@ -221,11 +227,15 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
|
||||
|
||||
errorsFound := false
|
||||
suggestIndexRebuild := false
|
||||
mixedFound := false
|
||||
for _, hint := range hints {
|
||||
switch hint.(type) {
|
||||
case *checker.ErrDuplicatePacks, *checker.ErrOldIndexFormat:
|
||||
Printf("%v\n", hint)
|
||||
suggestIndexRebuild = true
|
||||
case *checker.ErrMixedPack:
|
||||
Printf("%v\n", hint)
|
||||
mixedFound = true
|
||||
default:
|
||||
Warnf("error: %v\n", hint)
|
||||
errorsFound = true
|
||||
@ -235,6 +245,9 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
|
||||
if suggestIndexRebuild {
|
||||
Printf("This is non-critical, you can run `restic rebuild-index' to correct this\n")
|
||||
}
|
||||
if mixedFound {
|
||||
Printf("Mixed packs with tree and data blobs are non-critical, you can run `restic prune` to correct this.\n")
|
||||
}
|
||||
|
||||
if len(errs) > 0 {
|
||||
for _, err := range errs {
|
||||
@ -253,10 +266,12 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
|
||||
if checker.IsOrphanedPack(err) {
|
||||
orphanedPacks++
|
||||
Verbosef("%v\n", err)
|
||||
continue
|
||||
} else if _, ok := err.(*checker.ErrLegacyLayout); ok {
|
||||
Verbosef("repository still uses the S3 legacy layout\nPlease run `restic migrate s3legacy` to correct this.\n")
|
||||
} else {
|
||||
errorsFound = true
|
||||
Warnf("%v\n", err)
|
||||
}
|
||||
errorsFound = true
|
||||
Warnf("error: %v\n", err)
|
||||
}
|
||||
|
||||
if orphanedPacks > 0 {
|
||||
|
@ -13,6 +13,8 @@ import (
|
||||
|
||||
"github.com/minio/sha256-simd"
|
||||
"github.com/restic/restic/internal/backend"
|
||||
"github.com/restic/restic/internal/backend/s3"
|
||||
"github.com/restic/restic/internal/cache"
|
||||
"github.com/restic/restic/internal/debug"
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/hashing"
|
||||
@ -56,6 +58,13 @@ func New(repo restic.Repository, trackUnused bool) *Checker {
|
||||
return c
|
||||
}
|
||||
|
||||
// ErrLegacyLayout is returned when the repository uses the S3 legacy layout.
|
||||
type ErrLegacyLayout struct{}
|
||||
|
||||
func (e *ErrLegacyLayout) Error() string {
|
||||
return "repository uses S3 legacy layout"
|
||||
}
|
||||
|
||||
// ErrDuplicatePacks is returned when a pack is found in more than one index.
|
||||
type ErrDuplicatePacks struct {
|
||||
PackID restic.ID
|
||||
@ -66,6 +75,15 @@ func (e *ErrDuplicatePacks) Error() string {
|
||||
return fmt.Sprintf("pack %v contained in several indexes: %v", e.PackID, e.Indexes)
|
||||
}
|
||||
|
||||
// ErrMixedPack is returned when a pack is found that contains both tree and data blobs.
|
||||
type ErrMixedPack struct {
|
||||
PackID restic.ID
|
||||
}
|
||||
|
||||
func (e *ErrMixedPack) Error() string {
|
||||
return fmt.Sprintf("pack %v contains a mix of tree and data blobs", e.PackID.Str())
|
||||
}
|
||||
|
||||
// ErrOldIndexFormat is returned when an index with the old format is
|
||||
// found.
|
||||
type ErrOldIndexFormat struct {
|
||||
@ -141,6 +159,11 @@ func (c *Checker) LoadIndex(ctx context.Context) (hints []error, errs []error) {
|
||||
Indexes: packToIndex[packID],
|
||||
})
|
||||
}
|
||||
if c.masterIndex.IsMixedPack(packID) {
|
||||
hints = append(hints, &ErrMixedPack{
|
||||
PackID: packID,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
err = c.repo.SetIndex(c.masterIndex)
|
||||
@ -170,12 +193,30 @@ func IsOrphanedPack(err error) bool {
|
||||
return errors.As(err, &e) && e.Orphaned
|
||||
}
|
||||
|
||||
func isS3Legacy(b restic.Backend) bool {
|
||||
// unwrap cache
|
||||
if be, ok := b.(*cache.Backend); ok {
|
||||
b = be.Backend
|
||||
}
|
||||
|
||||
be, ok := b.(*s3.Backend)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
return be.Layout.Name() == "s3legacy"
|
||||
}
|
||||
|
||||
// Packs checks that all packs referenced in the index are still available and
|
||||
// there are no packs that aren't in an index. errChan is closed after all
|
||||
// packs have been checked.
|
||||
func (c *Checker) Packs(ctx context.Context, errChan chan<- error) {
|
||||
defer close(errChan)
|
||||
|
||||
if isS3Legacy(c.repo.Backend()) {
|
||||
errChan <- &ErrLegacyLayout{}
|
||||
}
|
||||
|
||||
debug.Log("checking for %d packs", len(c.packs))
|
||||
|
||||
debug.Log("listing repository packs")
|
||||
|
@ -63,6 +63,14 @@ func checkData(chkr *checker.Checker) []error {
|
||||
)
|
||||
}
|
||||
|
||||
func assertOnlyMixedPackHints(t *testing.T, hints []error) {
|
||||
for _, err := range hints {
|
||||
if _, ok := err.(*checker.ErrMixedPack); !ok {
|
||||
t.Fatalf("expected mixed pack hint, got %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckRepo(t *testing.T) {
|
||||
repodir, cleanup := test.Env(t, checkerTestData)
|
||||
defer cleanup()
|
||||
@ -74,9 +82,9 @@ func TestCheckRepo(t *testing.T) {
|
||||
if len(errs) > 0 {
|
||||
t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
|
||||
}
|
||||
|
||||
if len(hints) > 0 {
|
||||
t.Errorf("expected no hints, got %v: %v", len(hints), hints)
|
||||
assertOnlyMixedPackHints(t, hints)
|
||||
if len(hints) == 0 {
|
||||
t.Fatal("expected mixed pack warnings, got none")
|
||||
}
|
||||
|
||||
test.OKs(t, checkPacks(chkr))
|
||||
@ -100,10 +108,7 @@ func TestMissingPack(t *testing.T) {
|
||||
if len(errs) > 0 {
|
||||
t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
|
||||
}
|
||||
|
||||
if len(hints) > 0 {
|
||||
t.Errorf("expected no hints, got %v: %v", len(hints), hints)
|
||||
}
|
||||
assertOnlyMixedPackHints(t, hints)
|
||||
|
||||
errs = checkPacks(chkr)
|
||||
|
||||
@ -136,10 +141,7 @@ func TestUnreferencedPack(t *testing.T) {
|
||||
if len(errs) > 0 {
|
||||
t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
|
||||
}
|
||||
|
||||
if len(hints) > 0 {
|
||||
t.Errorf("expected no hints, got %v: %v", len(hints), hints)
|
||||
}
|
||||
assertOnlyMixedPackHints(t, hints)
|
||||
|
||||
errs = checkPacks(chkr)
|
||||
|
||||
@ -181,10 +183,7 @@ func TestUnreferencedBlobs(t *testing.T) {
|
||||
if len(errs) > 0 {
|
||||
t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
|
||||
}
|
||||
|
||||
if len(hints) > 0 {
|
||||
t.Errorf("expected no hints, got %v: %v", len(hints), hints)
|
||||
}
|
||||
assertOnlyMixedPackHints(t, hints)
|
||||
|
||||
test.OKs(t, checkPacks(chkr))
|
||||
test.OKs(t, checkStruct(chkr))
|
||||
@ -269,9 +268,7 @@ func TestModifiedIndex(t *testing.T) {
|
||||
t.Logf("found expected error %v", err)
|
||||
}
|
||||
|
||||
if len(hints) > 0 {
|
||||
t.Errorf("expected no hints, got %v: %v", len(hints), hints)
|
||||
}
|
||||
assertOnlyMixedPackHints(t, hints)
|
||||
}
|
||||
|
||||
var checkerDuplicateIndexTestData = filepath.Join("testdata", "duplicate-packs-in-index-test-repo.tar.gz")
|
||||
@ -421,10 +418,7 @@ func TestCheckerNoDuplicateTreeDecodes(t *testing.T) {
|
||||
if len(errs) > 0 {
|
||||
t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
|
||||
}
|
||||
|
||||
if len(hints) > 0 {
|
||||
t.Errorf("expected no hints, got %v: %v", len(hints), hints)
|
||||
}
|
||||
assertOnlyMixedPackHints(t, hints)
|
||||
|
||||
test.OKs(t, checkPacks(chkr))
|
||||
test.OKs(t, checkStruct(chkr))
|
||||
@ -572,8 +566,10 @@ func loadBenchRepository(t *testing.B) (*checker.Checker, restic.Repository, fun
|
||||
t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
|
||||
}
|
||||
|
||||
if len(hints) > 0 {
|
||||
t.Errorf("expected no hints, got %v: %v", len(hints), hints)
|
||||
for _, err := range hints {
|
||||
if _, ok := err.(*checker.ErrMixedPack); !ok {
|
||||
t.Fatalf("expected mixed pack hint, got %v", err)
|
||||
}
|
||||
}
|
||||
return chkr, repo, cleanup
|
||||
}
|
||||
|
@ -8,6 +8,7 @@ import (
|
||||
|
||||
"github.com/restic/restic/internal/backend"
|
||||
"github.com/restic/restic/internal/backend/s3"
|
||||
"github.com/restic/restic/internal/cache"
|
||||
"github.com/restic/restic/internal/debug"
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
@ -21,10 +22,25 @@ func init() {
|
||||
// "default" layout.
|
||||
type S3Layout struct{}
|
||||
|
||||
func toS3Backend(repo restic.Repository) *s3.Backend {
|
||||
b := repo.Backend()
|
||||
// unwrap cache
|
||||
if be, ok := b.(*cache.Backend); ok {
|
||||
b = be.Backend
|
||||
}
|
||||
|
||||
be, ok := b.(*s3.Backend)
|
||||
if !ok {
|
||||
debug.Log("backend is not s3")
|
||||
return nil
|
||||
}
|
||||
return be
|
||||
}
|
||||
|
||||
// Check tests whether the migration can be applied.
|
||||
func (m *S3Layout) Check(ctx context.Context, repo restic.Repository) (bool, error) {
|
||||
be, ok := repo.Backend().(*s3.Backend)
|
||||
if !ok {
|
||||
be := toS3Backend(repo)
|
||||
if be == nil {
|
||||
debug.Log("backend is not s3")
|
||||
return false, nil
|
||||
}
|
||||
@ -75,8 +91,8 @@ func (m *S3Layout) moveFiles(ctx context.Context, be *s3.Backend, l backend.Layo
|
||||
|
||||
// Apply runs the migration.
|
||||
func (m *S3Layout) Apply(ctx context.Context, repo restic.Repository) error {
|
||||
be, ok := repo.Backend().(*s3.Backend)
|
||||
if !ok {
|
||||
be := toS3Backend(repo)
|
||||
if be == nil {
|
||||
debug.Log("backend is not s3")
|
||||
return errors.New("backend is not s3")
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user