octoleo/restic
2
2
Fork 0
mirror of https://github.com/octoleo/restic.git synced 2024-11-26 23:06:32 +00:00
Code Issues Releases Wiki Activity

Apply changelog entry / documentation improvements from review

Browse Source
This commit is contained in:
Michael Eischer 2024-02-04 18:09:32 +01:00
parent 261b1455c7
commit a72c2b74f3
5 changed files with 29 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
changelog/unreleased/issue-4529
View File

@ -1,14 +1,18 @@
Enhancement: Verify data integrity before upload
Enhancement: Add extra verification of data integrity before upload
Hardware issues or a bug in restic could cause restic to create corrupted files
that were then uploaded to the repository. Detecting such corruption usually
required explicitly running the `check --read-data` command.
Hardware issues, or a bug in restic or its dependencies, could previously cause
corruption in the files restic created and stored in the repository. Detecting
such corruption previously required explicitly running the `check --read-data`
or `check --read-data-subset` commands.
To prevent the upload of corrupted data to the repository, restic now
additionally verifies that files can be decoded and contain the correct data
beforehand. This increases the CPU usage during backups. If absolutely
necessary, you can disable the verification using the option
`--no-extra-verify`.
To further ensure data integrity, even in the case of hardware issues or
software bugs, restic now performs additional verification of the files about
to be uploaded to the repository.
These extra checks will increase CPU usage during backups. They can therefore,
if absolutely necessary, be disabled using the `--no-extra-verify` global
option. Please note that this should be combined with more active checking
using the previously mentioned check commands.
https://github.com/restic/restic/issues/4529
https://github.com/restic/restic/pull/4681

2
cmd/restic/global.go
View File

@ -140,7 +140,7 @@ func init() {
f.BoolVar(&globalOptions.InsecureTLS, "insecure-tls", false, "skip TLS certificate verification when connecting to the repository (insecure)")
f.BoolVar(&globalOptions.CleanupCache, "cleanup-cache", false, "auto remove old cache directories")
f.Var(&globalOptions.Compression, "compression", "compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION)")
f.BoolVar(&globalOptions.NoExtraVerify, "no-extra-verify", false, "skip verification of data before upload")
f.BoolVar(&globalOptions.NoExtraVerify, "no-extra-verify", false, "skip additional verification of data before upload (see documentation)")
f.IntVar(&globalOptions.Limits.UploadKb, "limit-upload", 0, "limits uploads to a maximum `rate` in KiB/s. (default: unlimited)")
f.IntVar(&globalOptions.Limits.DownloadKb, "limit-download", 0, "limits downloads to a maximum `rate` in KiB/s. (default: unlimited)")
f.UintVar(&globalOptions.PackSize, "pack-size", 0, "set target pack `size` in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)")

15
doc/047_tuning_backup_parameters.rst
View File

@ -63,12 +63,15 @@ variable ``RESTIC_COMPRESSION``.
Data Verification
=================
To prevent the upload of corrupted data to the repository, restic verifies that files can
be decoded and contain the correct data beforehand. This increases the CPU usage during
backups. If necessary, you can disable this verification using the option ``--no-extra-verify``.
However, in this case you should verify the repository integrity more actively using
``restic check --read-data``. Otherwise, data corruption due to hardware issues or software
bugs might go unnoticed.
To prevent the upload of corrupted data to the repository, which can happen due
to hardware issues or software bugs, restic verifies that generated files can
be decoded and contain the correct data beforehand. This increases the CPU usage
during backups. If necessary, you can disable this verification using the
``--no-extra-verify`` option of the ``backup`` command. However, in this case
you should verify the repository integrity more actively using
``restic check --read-data`` (or the similar ``--read-data-subset`` option).
Otherwise, data corruption due to hardware issues or software bugs might go
unnoticed.
File Read Concurrency

3
internal/pack/pack.go
View File

@ -87,7 +87,8 @@ func (p *Packer) Finalize() error {
encryptedHeader = binary.LittleEndian.AppendUint32(encryptedHeader, uint32(len(encryptedHeader)))
if err := verifyHeader(p.k, encryptedHeader, p.blobs); err != nil {
return fmt.Errorf("detected data corruption while writing pack-file header: %w\nCorrupted data is either caused by hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting", err)
//nolint:revive // ignore linter warnings about error message spelling
return fmt.Errorf("Detected data corruption while writing pack-file header: %w\nCorrupted data is either caused by hardware issues or software bugs. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting.", err)
}
// append the header

6
internal/repository/repository.go
View File

@ -425,7 +425,8 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data
ciphertext = r.key.Seal(ciphertext, nonce, data, nil)
if err := r.verifyCiphertext(ciphertext, uncompressedLength, id); err != nil {
return 0, fmt.Errorf("detected data corruption while saving blob %v: %w\nCorrupted blobs are either caused by hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting", id, err)
//nolint:revive // ignore linter warnings about error message spelling
return 0, fmt.Errorf("Detected data corruption while saving blob %v: %w\nCorrupted blobs are either caused by hardware issues or software bugs. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting.", id, err)
}
// find suitable packer and add blob
@ -521,7 +522,8 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, buf []
ciphertext = r.key.Seal(ciphertext, nonce, p, nil)
if err := r.verifyUnpacked(ciphertext, t, buf); err != nil {
return restic.ID{}, fmt.Errorf("detected data corruption while saving file of type %v: %w\nCorrupted data is either caused by hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting", t, err)
//nolint:revive // ignore linter warnings about error message spelling
return restic.ID{}, fmt.Errorf("Detected data corruption while saving file of type %v: %w\nCorrupted data is either caused by hardware issues or software bugs. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting.", t, err)
}
if t == restic.ConfigFile {