From 8b11b8638349a596592e86e611187be42651a71b Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Wed, 13 Apr 2022 20:34:05 +0200 Subject: [PATCH] Add option global --compression --- cmd/restic/cmd_init.go | 2 +- cmd/restic/global.go | 4 +- internal/checker/checker_test.go | 2 +- internal/repository/repository.go | 69 +++++++++++++++++++++++++++++-- internal/repository/testing.go | 4 +- 5 files changed, 72 insertions(+), 9 deletions(-) diff --git a/cmd/restic/cmd_init.go b/cmd/restic/cmd_init.go index f5e43bbc9..058f1ed07 100644 --- a/cmd/restic/cmd_init.go +++ b/cmd/restic/cmd_init.go @@ -86,7 +86,7 @@ func runInit(opts InitOptions, gopts GlobalOptions, args []string) error { return errors.Fatalf("create repository at %s failed: %v\n", location.StripPassword(gopts.Repo), err) } - s := repository.New(be) + s := repository.New(be, repository.Options{Compression: gopts.Compression}) err = s.Init(gopts.ctx, version, gopts.password, chunkerPolynomial) if err != nil { diff --git a/cmd/restic/global.go b/cmd/restic/global.go index ca4b036c9..65dbbb6be 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -64,6 +64,7 @@ type GlobalOptions struct { InsecureTLS bool TLSClientCert string CleanupCache bool + Compression repository.CompressionMode LimitUploadKb int LimitDownloadKb int @@ -120,6 +121,7 @@ func init() { f.StringVar(&globalOptions.TLSClientCert, "tls-client-cert", "", "path to a `file` containing PEM encoded TLS client certificate and private key") f.BoolVar(&globalOptions.InsecureTLS, "insecure-tls", false, "skip TLS certificate verification when connecting to the repo (insecure)") f.BoolVar(&globalOptions.CleanupCache, "cleanup-cache", false, "auto remove old cache directories") + f.Var(&globalOptions.Compression, "compression", "compression mode (only available for repo format version 2), one of (auto|off|max)") f.IntVar(&globalOptions.LimitUploadKb, "limit-upload", 0, "limits uploads to a maximum rate in KiB/s. (default: unlimited)") f.IntVar(&globalOptions.LimitDownloadKb, "limit-download", 0, "limits downloads to a maximum rate in KiB/s. (default: unlimited)") f.StringSliceVarP(&globalOptions.Options, "option", "o", []string{}, "set extended option (`key=value`, can be specified multiple times)") @@ -435,7 +437,7 @@ func OpenRepository(opts GlobalOptions) (*repository.Repository, error) { } } - s := repository.New(be) + s := repository.New(be, repository.Options{Compression: opts.Compression}) passwordTriesLeft := 1 if stdinIsTerminal() && opts.password == "" { diff --git a/internal/checker/checker_test.go b/internal/checker/checker_test.go index 1330211eb..2a4384b15 100644 --- a/internal/checker/checker_test.go +++ b/internal/checker/checker_test.go @@ -350,7 +350,7 @@ func TestCheckerModifiedData(t *testing.T) { t.Logf("archived as %v", sn.ID().Str()) beError := &errorBackend{Backend: repo.Backend()} - checkRepo := repository.New(beError) + checkRepo := repository.New(beError, repository.Options{}) test.OK(t, checkRepo.SearchKey(context.TODO(), test.TestPassword, 5, "")) chkr := checker.New(checkRepo, false) diff --git a/internal/repository/repository.go b/internal/repository/repository.go index bbbc1af68..b6c910cc7 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -37,6 +37,8 @@ type Repository struct { idx *MasterIndex Cache *cache.Cache + opts Options + noAutoIndexUpdate bool treePM *packerManager @@ -48,10 +50,58 @@ type Repository struct { dec *zstd.Decoder } +type Options struct { + Compression CompressionMode +} + +// CompressionMode configures if data should be compressed. +type CompressionMode uint + +// Constants for the different compression levels. +const ( + CompressionAuto CompressionMode = 0 + CompressionOff CompressionMode = 1 + CompressionMax CompressionMode = 2 +) + +// Set implements the method needed for pflag command flag parsing. +func (c *CompressionMode) Set(s string) error { + switch s { + case "auto": + *c = CompressionAuto + case "off": + *c = CompressionOff + case "max": + *c = CompressionMax + default: + return fmt.Errorf("invalid compression mode %q, must be one of (auto|off|max)", s) + } + + return nil +} + +func (c *CompressionMode) String() string { + switch *c { + case CompressionAuto: + return "auto" + case CompressionOff: + return "off" + case CompressionMax: + return "max" + default: + return "invalid" + } + +} +func (c *CompressionMode) Type() string { + return "mode" +} + // New returns a new repository with backend be. -func New(be restic.Backend) *Repository { +func New(be restic.Backend, opts Options) *Repository { repo := &Repository{ be: be, + opts: opts, idx: NewMasterIndex(), dataPM: newPackerManager(be, nil), treePM: newPackerManager(be, nil), @@ -274,7 +324,12 @@ func (r *Repository) LookupBlobSize(id restic.ID, tpe restic.BlobType) (uint, bo func (r *Repository) getZstdEncoder() *zstd.Encoder { r.allocEnc.Do(func() { - enc, err := zstd.NewWriter(nil) + level := zstd.SpeedDefault + if r.opts.Compression == CompressionMax { + level = zstd.SpeedBestCompression + } + + enc, err := zstd.NewWriter(nil, zstd.WithEncoderLevel(level)) if err != nil { panic(err) } @@ -302,8 +357,14 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data uncompressedLength := 0 if r.cfg.Version > 1 { - uncompressedLength = len(data) - data = r.getZstdEncoder().EncodeAll(data, nil) + + // we have a repo v2, so compression is available. if the user opts to + // not compress, we won't compress any data, but everything else is + // compressed. + if r.opts.Compression != CompressionOff || t != restic.DataBlob { + uncompressedLength = len(data) + data = r.getZstdEncoder().EncodeAll(data, nil) + } } nonce := crypto.NewRandomNonce() diff --git a/internal/repository/testing.go b/internal/repository/testing.go index d752e107e..fbe334467 100644 --- a/internal/repository/testing.go +++ b/internal/repository/testing.go @@ -51,7 +51,7 @@ func TestRepositoryWithBackend(t testing.TB, be restic.Backend) (r restic.Reposi be, beCleanup = TestBackend(t) } - repo := New(be) + repo := New(be, Options{}) cfg := restic.TestCreateConfig(t, TestChunkerPol) err := repo.init(context.TODO(), test.TestPassword, cfg) @@ -98,7 +98,7 @@ func TestOpenLocal(t testing.TB, dir string) (r restic.Repository) { t.Fatal(err) } - repo := New(be) + repo := New(be, Options{}) err = repo.SearchKey(context.TODO(), test.TestPassword, 10, "") if err != nil { t.Fatal(err)