mirror of https://github.com/octoleo/restic.git
rename option to --pack-size
This commit is contained in:
Michael Eischer
parent
d7e2892048
commit
1b076cda97
|
|
@ -1,7 +1,7 @@
|
|||
Enhancement: Allow pack size customization
|
||||
|
||||
Restic now uses a target pack size of 16 MiB by default. It can be customized
|
||||
using the `--packsize size` option. Supported pack sizes range between 4 and
|
||||
using the `--pack-size size` option. Supported pack sizes range between 4 and
|
||||
128 MiB.
|
||||
|
||||
It is possible to migrate an existing repository to _larger_ pack files using
|
||||
|
|
|
|||
|
|
@ -88,7 +88,7 @@ func runInit(opts InitOptions, gopts GlobalOptions, args []string) error {
|
|||
|
||||
s, err := repository.New(be, repository.Options{
|
||||
Compression: gopts.Compression,
|
||||
PackSize: gopts.MinPackSize * 1024 * 1024,
|
||||
PackSize: gopts.PackSize * 1024 * 1024,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
|
|||
|
|
@ -425,7 +425,7 @@ func decidePackAction(ctx context.Context, opts PruneOptions, gopts GlobalOption
|
|||
|
||||
var repackCandidates []packInfoWithID
|
||||
repoVersion := repo.Config().Version
|
||||
minPackSize := repo.MinPackSize()
|
||||
targetPackSize := repo.PackSize()
|
||||
|
||||
// loop over all packs and decide what to do
|
||||
bar := newProgressMax(!gopts.Quiet, uint64(len(indexPack)), "packs processed")
|
||||
|
|
@ -467,7 +467,7 @@ func decidePackAction(ctx context.Context, opts PruneOptions, gopts GlobalOption
|
|||
// use a flag that pack must be compressed
|
||||
p.uncompressed = mustCompress
|
||||
|
||||
packIsLargeEnough := !opts.RepackSmall || packSize >= int64(minPackSize)
|
||||
packIsLargeEnough := !opts.RepackSmall || packSize >= int64(targetPackSize)
|
||||
|
||||
// decide what to do
|
||||
switch {
|
||||
|
|
@ -539,9 +539,9 @@ func decidePackAction(ctx context.Context, opts PruneOptions, gopts GlobalOption
|
|||
return true
|
||||
case pj.tpe != restic.DataBlob && pi.tpe == restic.DataBlob:
|
||||
return false
|
||||
case opts.RepackSmall && pi.unusedSize+pi.usedSize < uint64(minPackSize) && pj.unusedSize+pj.usedSize >= uint64(minPackSize):
|
||||
case opts.RepackSmall && pi.unusedSize+pi.usedSize < uint64(targetPackSize) && pj.unusedSize+pj.usedSize >= uint64(targetPackSize):
|
||||
return true
|
||||
case opts.RepackSmall && pj.unusedSize+pj.usedSize < uint64(minPackSize) && pi.unusedSize+pi.usedSize >= uint64(minPackSize):
|
||||
case opts.RepackSmall && pj.unusedSize+pj.usedSize < uint64(targetPackSize) && pi.unusedSize+pi.usedSize >= uint64(targetPackSize):
|
||||
return false
|
||||
}
|
||||
return pi.unusedSize*pj.usedSize > pj.unusedSize*pi.usedSize
|
||||
|
|
@ -561,7 +561,7 @@ func decidePackAction(ctx context.Context, opts PruneOptions, gopts GlobalOption
|
|||
for _, p := range repackCandidates {
|
||||
reachedUnusedSizeAfter := (stats.size.unused-stats.size.remove-stats.size.repackrm < maxUnusedSizeAfter)
|
||||
reachedRepackSize := stats.size.repack+p.unusedSize+p.usedSize >= opts.MaxRepackBytes
|
||||
packIsLargeEnough := !opts.RepackSmall || p.unusedSize+p.usedSize >= uint64(minPackSize)
|
||||
packIsLargeEnough := !opts.RepackSmall || p.unusedSize+p.usedSize >= uint64(targetPackSize)
|
||||
|
||||
switch {
|
||||
case reachedRepackSize:
|
||||
|
|
|
|||
|
|
@ -63,7 +63,7 @@ type GlobalOptions struct {
|
|||
NoCache bool
|
||||
CleanupCache bool
|
||||
Compression repository.CompressionMode
|
||||
MinPackSize uint
|
||||
PackSize uint
|
||||
|
||||
backend.TransportOptions
|
||||
limiter.Limits
|
||||
|
|
@ -104,8 +104,8 @@ func init() {
|
|||
return nil
|
||||
})
|
||||
|
||||
// parse min pack size from env, on error the default value will be used
|
||||
minPackSize, _ := strconv.ParseUint(os.Getenv("RESTIC_MIN_PACKSIZE"), 10, 32)
|
||||
// parse target pack size from env, on error the default value will be used
|
||||
targetPackSize, _ := strconv.ParseUint(os.Getenv("RESTIC_PACK_SIZE"), 10, 32)
|
||||
|
||||
f := cmdRoot.PersistentFlags()
|
||||
f.StringVarP(&globalOptions.Repo, "repo", "r", os.Getenv("RESTIC_REPOSITORY"), "`repository` to backup to or restore from (default: $RESTIC_REPOSITORY)")
|
||||
|
|
@ -126,7 +126,7 @@ func init() {
|
|||
f.Var(&globalOptions.Compression, "compression", "compression mode (only available for repository format version 2), one of (auto|off|max)")
|
||||
f.IntVar(&globalOptions.Limits.UploadKb, "limit-upload", 0, "limits uploads to a maximum rate in KiB/s. (default: unlimited)")
|
||||
f.IntVar(&globalOptions.Limits.DownloadKb, "limit-download", 0, "limits downloads to a maximum rate in KiB/s. (default: unlimited)")
|
||||
f.UintVar(&globalOptions.MinPackSize, "min-packsize", uint(minPackSize), "set min pack size in MiB. (default: $RESTIC_MIN_PACKSIZE)")
|
||||
f.UintVar(&globalOptions.PackSize, "pack-size", uint(targetPackSize), "set target pack size in MiB. (default: $RESTIC_PACK_SIZE)")
|
||||
f.StringSliceVarP(&globalOptions.Options, "option", "o", []string{}, "set extended option (`key=value`, can be specified multiple times)")
|
||||
// Use our "generate" command instead of the cobra provided "completion" command
|
||||
cmdRoot.CompletionOptions.DisableDefaultCmd = true
|
||||
|
|
@ -448,7 +448,7 @@ func OpenRepository(opts GlobalOptions) (*repository.Repository, error) {
|
|||
|
||||
s, err := repository.New(be, repository.Options{
|
||||
Compression: opts.Compression,
|
||||
PackSize: opts.MinPackSize * 1024 * 1024,
|
||||
PackSize: opts.PackSize * 1024 * 1024,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
|
|||
|
|
@ -554,6 +554,7 @@ environment variables. The following lists these environment variables:
|
|||
RESTIC_CACHE_DIR Location of the cache directory
|
||||
RESTIC_COMPRESSION Compression mode (only available for repository format version 2)
|
||||
RESTIC_PROGRESS_FPS Frames per second by which the progress bar is updated
|
||||
RESTIC_PACK_SIZE Target size for pack files
|
||||
|
||||
TMPDIR Location for temporary files
|
||||
|
||||
|
|
|
|||
|
|
@ -59,8 +59,8 @@ upload connections, it is desirable to use larger pack sizes to reduce the numbe
|
|||
files in the repository and improve upload performance. Notable examples are OpenStack
|
||||
Swift and some Google Drive Team accounts, where there are hard limits on the total
|
||||
number of files. Larger pack sizes can also improve the backup speed for a repository
|
||||
stored on a local HDD. This can be achieved by either using the ``--min-packsize`` option
|
||||
or defining the ``$RESTIC_MIN_PACKSIZE`` environment variable. Restic currently defaults
|
||||
stored on a local HDD. This can be achieved by either using the ``--pack-size`` option
|
||||
or defining the ``$RESTIC_PACK_SIZE`` environment variable. Restic currently defaults
|
||||
to a 16 MiB pack size.
|
||||
|
||||
The side effect of increasing the pack size is requiring more disk space for temporary pack
|
||||
|
|
|
|||
|
|
@ -56,7 +56,7 @@ Usage help is available:
|
|||
--key-hint key key ID of key to try decrypting first (default: $RESTIC_KEY_HINT)
|
||||
--limit-download int limits downloads to a maximum rate in KiB/s. (default: unlimited)
|
||||
--limit-upload int limits uploads to a maximum rate in KiB/s. (default: unlimited)
|
||||
--min-packsize uint set min pack size in MiB. (default: $RESTIC_MIN_PACKSIZE)
|
||||
--pack-size uint set target pack size in MiB. (default: $RESTIC_PACK_SIZE)
|
||||
--no-cache do not use a local cache
|
||||
--no-lock do not lock the repository, this allows some operations on read-only repositories
|
||||
-o, --option key=value set extended option (key=value, can be specified multiple times)
|
||||
|
|
@ -129,7 +129,7 @@ command:
|
|||
--key-hint key key ID of key to try decrypting first (default: $RESTIC_KEY_HINT)
|
||||
--limit-download int limits downloads to a maximum rate in KiB/s. (default: unlimited)
|
||||
--limit-upload int limits uploads to a maximum rate in KiB/s. (default: unlimited)
|
||||
--min-packsize uint set min pack size in MiB. (default: $RESTIC_MIN_PACKSIZE)
|
||||
--pack-size uint set target pack size in MiB. (default: $RESTIC_PACK_SIZE)
|
||||
--no-cache do not use a local cache
|
||||
--no-lock do not lock the repository, this allows some operations on read-only repositories
|
||||
-o, --option key=value set extended option (key=value, can be specified multiple times)
|
||||
|
|
|
|||
|
|
@ -143,8 +143,8 @@ func (r *Repository) Config() restic.Config {
|
|||
return r.cfg
|
||||
}
|
||||
|
||||
// MinPackSize return the minimum size of a pack file before uploading
|
||||
func (r *Repository) MinPackSize() uint {
|
||||
// PackSize return the target size of a pack file when uploading
|
||||
func (r *Repository) PackSize() uint {
|
||||
return r.opts.PackSize
|
||||
}
|
||||
|
||||
|
|
@ -516,8 +516,8 @@ func (r *Repository) StartPackUploader(ctx context.Context, wg *errgroup.Group)
|
|||
innerWg, ctx := errgroup.WithContext(ctx)
|
||||
r.packerWg = innerWg
|
||||
r.uploader = newPackerUploader(ctx, innerWg, r, r.be.Connections())
|
||||
r.treePM = newPackerManager(r.key, restic.TreeBlob, r.MinPackSize(), r.uploader.QueuePacker)
|
||||
r.dataPM = newPackerManager(r.key, restic.DataBlob, r.MinPackSize(), r.uploader.QueuePacker)
|
||||
r.treePM = newPackerManager(r.key, restic.TreeBlob, r.PackSize(), r.uploader.QueuePacker)
|
||||
r.dataPM = newPackerManager(r.key, restic.DataBlob, r.PackSize(), r.uploader.QueuePacker)
|
||||
|
||||
wg.Go(func() error {
|
||||
return innerWg.Wait()
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ type Repository interface {
|
|||
LookupBlobSize(ID, BlobType) (uint, bool)
|
||||
|
||||
Config() Config
|
||||
MinPackSize() uint
|
||||
PackSize() uint
|
||||
|
||||
// List calls the function fn for each file of type t in the repository.
|
||||
// When an error is returned by fn, processing stops and List() returns the
|
||||
|
|
|
|||
Loading…
Reference in New Issue