2
2
mirror of https://github.com/octoleo/restic.git synced 2024-12-22 19:08:55 +00:00

repository: Implement index/snapshot/lock compression

The config file is not compressed as it should remain readable by older
restic versions such that these can return a proper error.

As the old format for unpacked data does not include a version header,
make use of a trick: The old data is always encoded as JSON. Thus it can
only start with '{' or '['. For any other value the first byte indicates
a versioned format. The version is set to 2 for now. Then the zstd
compressed data follows.
This commit is contained in:
Michael Eischer 2022-02-13 00:12:40 +01:00 committed by Alexander Neumann
parent 0957b74887
commit 4b957e7373
2 changed files with 59 additions and 1 deletions

2
go.mod
View File

@ -21,7 +21,7 @@ require (
github.com/hashicorp/golang-lru v0.5.4 github.com/hashicorp/golang-lru v0.5.4
github.com/json-iterator/go v1.1.12 // indirect github.com/json-iterator/go v1.1.12 // indirect
github.com/juju/ratelimit v1.0.1 github.com/juju/ratelimit v1.0.1
github.com/klauspost/compress v1.15.1 // indirect github.com/klauspost/compress v1.15.1
github.com/klauspost/cpuid/v2 v2.0.12 // indirect github.com/klauspost/cpuid/v2 v2.0.12 // indirect
github.com/kurin/blazer v0.5.4-0.20211030221322-ba894c124ac6 github.com/kurin/blazer v0.5.4-0.20211030221322-ba894c124ac6
github.com/minio/md5-simd v1.1.2 // indirect github.com/minio/md5-simd v1.1.2 // indirect

View File

@ -12,6 +12,7 @@ import (
"sync" "sync"
"github.com/cenkalti/backoff/v4" "github.com/cenkalti/backoff/v4"
"github.com/klauspost/compress/zstd"
"github.com/restic/chunker" "github.com/restic/chunker"
"github.com/restic/restic/internal/backend/dryrun" "github.com/restic/restic/internal/backend/dryrun"
"github.com/restic/restic/internal/cache" "github.com/restic/restic/internal/cache"
@ -40,6 +41,9 @@ type Repository struct {
treePM *packerManager treePM *packerManager
dataPM *packerManager dataPM *packerManager
enc *zstd.Encoder
dec *zstd.Decoder
} }
// New returns a new repository with backend be. // New returns a new repository with backend be.
@ -51,6 +55,16 @@ func New(be restic.Backend) *Repository {
treePM: newPackerManager(be, nil), treePM: newPackerManager(be, nil),
} }
enc, err := zstd.NewWriter(nil)
if err != nil {
panic(err)
}
repo.enc = enc
dec, err := zstd.NewReader(nil)
if err != nil {
panic(err)
}
repo.dec = dec
return repo return repo
} }
@ -125,6 +139,9 @@ func (r *Repository) LoadUnpacked(ctx context.Context, buf []byte, t restic.File
if err != nil { if err != nil {
return nil, err return nil, err
} }
if t != restic.ConfigFile {
return r.decompressUnpacked(plaintext)
}
return plaintext, nil return plaintext, nil
} }
@ -312,9 +329,50 @@ func (r *Repository) SaveJSONUnpacked(ctx context.Context, t restic.FileType, it
return r.SaveUnpacked(ctx, t, plaintext) return r.SaveUnpacked(ctx, t, plaintext)
} }
func (r *Repository) compressUnpacked(p []byte) ([]byte, error) {
// compression is only available starting from version 2
if r.cfg.Version < 2 {
return p, nil
}
// version byte
out := []byte{2}
out = r.enc.EncodeAll(p, out)
return out, nil
}
func (r *Repository) decompressUnpacked(p []byte) ([]byte, error) {
// compression is only available starting from version 2
if r.cfg.Version < 2 {
return p, nil
}
if len(p) < 1 {
// too short for version header
return p, nil
}
if p[0] == '[' || p[0] == '{' {
// probably raw JSON
return p, nil
}
// version
if p[0] != 2 {
return nil, errors.New("not supported encoding format")
}
return r.dec.DecodeAll(p[1:], nil)
}
// SaveUnpacked encrypts data and stores it in the backend. Returned is the // SaveUnpacked encrypts data and stores it in the backend. Returned is the
// storage hash. // storage hash.
func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, p []byte) (id restic.ID, err error) { func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, p []byte) (id restic.ID, err error) {
if t != restic.ConfigFile {
p, err = r.compressUnpacked(p)
if err != nil {
return restic.ID{}, err
}
}
ciphertext := restic.NewBlobBuffer(len(p)) ciphertext := restic.NewBlobBuffer(len(p))
ciphertext = ciphertext[:0] ciphertext = ciphertext[:0]
nonce := crypto.NewRandomNonce() nonce := crypto.NewRandomNonce()