From 0164f5310d627b2e920f7427c265483634b020da Mon Sep 17 00:00:00 2001
From: Michael Eischer <michael.eischer@fau.de>
Date: Fri, 2 Feb 2024 20:10:29 +0100
Subject: [PATCH 01/19] Downgrade klauspost/compress to fix data corruption at
 max. compression

---
 go.mod | 2 ++
 go.sum | 4 ++--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/go.mod b/go.mod
index f07fc10ee..970770a90 100644
--- a/go.mod
+++ b/go.mod
@@ -36,6 +36,8 @@ require (
 	google.golang.org/api v0.149.0
 )
 
+replace github.com/klauspost/compress => github.com/klauspost/compress v1.17.2
+
 require (
 	cloud.google.com/go v0.110.9 // indirect
 	cloud.google.com/go/compute v1.23.1 // indirect
diff --git a/go.sum b/go.sum
index 01e58cccb..129eafed2 100644
--- a/go.sum
+++ b/go.sum
@@ -108,8 +108,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
-github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4=
-github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
+github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
+github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
 github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc=
 github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=

From 29e1caf8250c1965467fb6da1f6155e953b4075d Mon Sep 17 00:00:00 2001
From: Michael Eischer <michael.eischer@fau.de>
Date: Sat, 3 Feb 2024 18:52:51 +0100
Subject: [PATCH 02/19] add changelog draft for data corruption on max
 compression

---
 changelog/unreleased/issue-4677 | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 changelog/unreleased/issue-4677

diff --git a/changelog/unreleased/issue-4677 b/changelog/unreleased/issue-4677
new file mode 100644
index 000000000..8fa6cf65b
--- /dev/null
+++ b/changelog/unreleased/issue-4677
@@ -0,0 +1,19 @@
+Bugfix: Downgrade zstd library to fix rare data corruption at max. compression
+
+In restic 0.16.3, backups where the compression level was set to `max` (using
+`--compression max`) could in rare and very specific circumstances result in
+data corruption due to a bug in the library used for compressing data. Restic
+0.16.1 and 0.16.2 were not affected.
+
+Restic now uses the previous version of the library used to compress data, the
+same version used by restic 0.16.2. Please note that the `auto` compression
+level (which restic uses by default) was never affected, and even if you used
+`max` compression, chances of being affected by this issue are small.
+
+To check a repository for any corruption, run `restic check --read-data`. This
+will download and verify the whole repository and can be used at any time to
+completely verify the integrity of a repository. If the `check` command detects
+anomalies, follow the suggested steps.
+
+https://github.com/restic/restic/issues/4677
+https://github.com/restic/restic/pull/4679

From 0ea62b5ac6804a425ebbb9dfbfe65c296836e36b Mon Sep 17 00:00:00 2001
From: Michael Eischer <michael.eischer@fau.de>
Date: Sat, 3 Feb 2024 17:47:36 +0100
Subject: [PATCH 03/19] repository: make repo.Options configurable for test
 repos

---
 internal/archiver/archiver_test.go          | 2 +-
 internal/migrations/upgrade_repo_v2_test.go | 2 +-
 internal/repository/fuzz_test.go            | 3 +--
 internal/repository/testing.go              | 9 +++++----
 internal/restic/lock_test.go                | 2 +-
 5 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/internal/archiver/archiver_test.go b/internal/archiver/archiver_test.go
index 3c87055d8..5a9896a48 100644
--- a/internal/archiver/archiver_test.go
+++ b/internal/archiver/archiver_test.go
@@ -1879,7 +1879,7 @@ func TestArchiverContextCanceled(t *testing.T) {
 	})
 
 	// Ensure that the archiver itself reports the canceled context and not just the backend
-	repo := repository.TestRepositoryWithBackend(t, &noCancelBackend{mem.New()}, 0)
+	repo := repository.TestRepositoryWithBackend(t, &noCancelBackend{mem.New()}, 0, repository.Options{})
 
 	back := restictest.Chdir(t, tempdir)
 	defer back()
diff --git a/internal/migrations/upgrade_repo_v2_test.go b/internal/migrations/upgrade_repo_v2_test.go
index 96fc7788e..7f251de93 100644
--- a/internal/migrations/upgrade_repo_v2_test.go
+++ b/internal/migrations/upgrade_repo_v2_test.go
@@ -69,7 +69,7 @@ func TestUpgradeRepoV2Failure(t *testing.T) {
 		Backend:                   be,
 	}
 
-	repo := repository.TestRepositoryWithBackend(t, be, 1)
+	repo := repository.TestRepositoryWithBackend(t, be, 1, repository.Options{})
 	if repo.Config().Version != 1 {
 		t.Fatal("test repo has wrong version")
 	}
diff --git a/internal/repository/fuzz_test.go b/internal/repository/fuzz_test.go
index b4036288c..80372f8e0 100644
--- a/internal/repository/fuzz_test.go
+++ b/internal/repository/fuzz_test.go
@@ -4,7 +4,6 @@ import (
 	"context"
 	"testing"
 
-	"github.com/restic/restic/internal/backend/mem"
 	"github.com/restic/restic/internal/restic"
 	"golang.org/x/sync/errgroup"
 )
@@ -19,7 +18,7 @@ func FuzzSaveLoadBlob(f *testing.F) {
 		}
 
 		id := restic.Hash(blob)
-		repo := TestRepositoryWithBackend(t, mem.New(), 2)
+		repo := TestRepositoryWithVersion(t, 2)
 
 		var wg errgroup.Group
 		repo.StartPackUploader(context.TODO(), &wg)
diff --git a/internal/repository/testing.go b/internal/repository/testing.go
index 4936cc368..9bdd65901 100644
--- a/internal/repository/testing.go
+++ b/internal/repository/testing.go
@@ -43,7 +43,7 @@ const TestChunkerPol = chunker.Pol(0x3DA3358B4DC173)
 // TestRepositoryWithBackend returns a repository initialized with a test
 // password. If be is nil, an in-memory backend is used. A constant polynomial
 // is used for the chunker and low-security test parameters.
-func TestRepositoryWithBackend(t testing.TB, be restic.Backend, version uint) restic.Repository {
+func TestRepositoryWithBackend(t testing.TB, be restic.Backend, version uint, opts Options) restic.Repository {
 	t.Helper()
 	TestUseLowSecurityKDFParameters(t)
 	restic.TestDisableCheckPolynomial(t)
@@ -52,7 +52,7 @@ func TestRepositoryWithBackend(t testing.TB, be restic.Backend, version uint) re
 		be = TestBackend(t)
 	}
 
-	repo, err := New(be, Options{})
+	repo, err := New(be, opts)
 	if err != nil {
 		t.Fatalf("TestRepository(): new repo failed: %v", err)
 	}
@@ -78,6 +78,7 @@ func TestRepository(t testing.TB) restic.Repository {
 func TestRepositoryWithVersion(t testing.TB, version uint) restic.Repository {
 	t.Helper()
 	dir := os.Getenv("RESTIC_TEST_REPO")
+	opts := Options{}
 	if dir != "" {
 		_, err := os.Stat(dir)
 		if err != nil {
@@ -85,7 +86,7 @@ func TestRepositoryWithVersion(t testing.TB, version uint) restic.Repository {
 			if err != nil {
 				t.Fatalf("error creating local backend at %v: %v", dir, err)
 			}
-			return TestRepositoryWithBackend(t, be, version)
+			return TestRepositoryWithBackend(t, be, version, opts)
 		}
 
 		if err == nil {
@@ -93,7 +94,7 @@ func TestRepositoryWithVersion(t testing.TB, version uint) restic.Repository {
 		}
 	}
 
-	return TestRepositoryWithBackend(t, nil, version)
+	return TestRepositoryWithBackend(t, nil, version, opts)
 }
 
 // TestOpenLocal opens a local repository.
diff --git a/internal/restic/lock_test.go b/internal/restic/lock_test.go
index f3c405c9c..2eb22be1b 100644
--- a/internal/restic/lock_test.go
+++ b/internal/restic/lock_test.go
@@ -65,7 +65,7 @@ func (be *failLockLoadingBackend) Load(ctx context.Context, h restic.Handle, len
 
 func TestMultipleLockFailure(t *testing.T) {
 	be := &failLockLoadingBackend{Backend: mem.New()}
-	repo := repository.TestRepositoryWithBackend(t, be, 0)
+	repo := repository.TestRepositoryWithBackend(t, be, 0, repository.Options{})
 	restic.TestSetLockTimeout(t, 5*time.Millisecond)
 
 	lock1, err := restic.NewLock(context.TODO(), repo)

From 2f30c940b2f6ca645f87a06a841ff137e3153afe Mon Sep 17 00:00:00 2001
From: Michael Eischer <michael.eischer@fau.de>
Date: Fri, 2 Feb 2024 21:15:39 +0100
Subject: [PATCH 04/19] backup: verify blobs before upload

This only covers the blobs themselves, the pack header is not verified
so far. Unpacked files are also not covered by the integrity check.
---
 internal/repository/repository.go | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/internal/repository/repository.go b/internal/repository/repository.go
index 0b50382b8..7855438c9 100644
--- a/internal/repository/repository.go
+++ b/internal/repository/repository.go
@@ -423,6 +423,11 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data
 	// encrypt blob
 	ciphertext = r.key.Seal(ciphertext, nonce, data, nil)
 
+	if err := r.verifyCiphertext(ciphertext, uncompressedLength, id); err != nil {
+		// FIXME call to action
+		return 0, fmt.Errorf("detected data corruption while saving blob %v: %w", id, err)
+	}
+
 	// find suitable packer and add blob
 	var pm *packerManager
 
@@ -438,6 +443,27 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data
 	return pm.SaveBlob(ctx, t, id, ciphertext, uncompressedLength)
 }
 
+func (r *Repository) verifyCiphertext(buf []byte, uncompressedLength int, id restic.ID) error {
+	nonce, ciphertext := buf[:r.key.NonceSize()], buf[r.key.NonceSize():]
+	plaintext, err := r.key.Open(nil, nonce, ciphertext, nil)
+	if err != nil {
+		return fmt.Errorf("decryption failed: %w", err)
+	}
+	if uncompressedLength != 0 {
+		// DecodeAll will allocate a slice if it is not large enough since it
+		// knows the decompressed size (because we're using EncodeAll)
+		plaintext, err = r.getZstdDecoder().DecodeAll(plaintext, nil)
+		if err != nil {
+			return fmt.Errorf("decompression failed: %w", err)
+		}
+	}
+	if !restic.Hash(plaintext).Equal(id) {
+		return errors.New("hash mismatch")
+	}
+
+	return nil
+}
+
 func (r *Repository) compressUnpacked(p []byte) ([]byte, error) {
 	// compression is only available starting from version 2
 	if r.cfg.Version < 2 {

From cb85fb46ddf15d8c6d149d23cadcaab5e7164786 Mon Sep 17 00:00:00 2001
From: Michael Eischer <michael.eischer@fau.de>
Date: Sat, 3 Feb 2024 17:30:58 +0100
Subject: [PATCH 05/19] backup: verify unpacked files before upload

---
 internal/repository/repository.go | 27 ++++++++++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/internal/repository/repository.go b/internal/repository/repository.go
index 7855438c9..4fa7e487c 100644
--- a/internal/repository/repository.go
+++ b/internal/repository/repository.go
@@ -500,7 +500,8 @@ func (r *Repository) decompressUnpacked(p []byte) ([]byte, error) {
 
 // SaveUnpacked encrypts data and stores it in the backend. Returned is the
 // storage hash.
-func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, p []byte) (id restic.ID, err error) {
+func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, buf []byte) (id restic.ID, err error) {
+	p := buf
 	if t != restic.ConfigFile {
 		p, err = r.compressUnpacked(p)
 		if err != nil {
@@ -515,6 +516,11 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, p []by
 
 	ciphertext = r.key.Seal(ciphertext, nonce, p, nil)
 
+	if err := r.verifyUnpacked(ciphertext, t, buf); err != nil {
+		// FIXME call to action
+		return restic.ID{}, fmt.Errorf("detected data corruption while saving file of type %v: %w", t, err)
+	}
+
 	if t == restic.ConfigFile {
 		id = restic.ID{}
 	} else {
@@ -532,6 +538,25 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, p []by
 	return id, nil
 }
 
+func (r *Repository) verifyUnpacked(buf []byte, t restic.FileType, expected []byte) error {
+	nonce, ciphertext := buf[:r.key.NonceSize()], buf[r.key.NonceSize():]
+	plaintext, err := r.key.Open(nil, nonce, ciphertext, nil)
+	if err != nil {
+		return fmt.Errorf("decryption failed: %w", err)
+	}
+	if t != restic.ConfigFile {
+		plaintext, err = r.decompressUnpacked(plaintext)
+		if err != nil {
+			return fmt.Errorf("decompression failed: %w", err)
+		}
+	}
+
+	if !bytes.Equal(plaintext, expected) {
+		return errors.New("data mismatch")
+	}
+	return nil
+}
+
 // Flush saves all remaining packs and the index
 func (r *Repository) Flush(ctx context.Context) error {
 	if err := r.flushPacks(ctx); err != nil {

From 885431ec2b64bae061246ef8d0fab19322e29fe3 Mon Sep 17 00:00:00 2001
From: Michael Eischer <michael.eischer@fau.de>
Date: Sat, 3 Feb 2024 17:47:48 +0100
Subject: [PATCH 06/19] repository: Allow skipping verification for tests

Some tests have to explicitly create pack files with blobs that don't
match their ID. For those blobs the builtin verification of the
repository must be disabled.
---
 internal/repository/repack_test.go |  6 ++++--
 internal/repository/repository.go  | 13 +++++++++++--
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/internal/repository/repack_test.go b/internal/repository/repack_test.go
index c8570a9d4..00567aad9 100644
--- a/internal/repository/repack_test.go
+++ b/internal/repository/repack_test.go
@@ -346,7 +346,8 @@ func TestRepackWrongBlob(t *testing.T) {
 }
 
 func testRepackWrongBlob(t *testing.T, version uint) {
-	repo := repository.TestRepositoryWithVersion(t, version)
+	// disable verification to allow adding corrupted blobs to the repository
+	repo := repository.TestRepositoryWithBackend(t, nil, version, repository.Options{NoVerifyPack: true})
 
 	seed := time.Now().UnixNano()
 	rand.Seed(seed)
@@ -371,7 +372,8 @@ func TestRepackBlobFallback(t *testing.T) {
 }
 
 func testRepackBlobFallback(t *testing.T, version uint) {
-	repo := repository.TestRepositoryWithVersion(t, version)
+	// disable verification to allow adding corrupted blobs to the repository
+	repo := repository.TestRepositoryWithBackend(t, nil, version, repository.Options{NoVerifyPack: true})
 
 	seed := time.Now().UnixNano()
 	rand.Seed(seed)
diff --git a/internal/repository/repository.go b/internal/repository/repository.go
index 4fa7e487c..c7c36a74a 100644
--- a/internal/repository/repository.go
+++ b/internal/repository/repository.go
@@ -59,8 +59,9 @@ type Repository struct {
 }
 
 type Options struct {
-	Compression CompressionMode
-	PackSize    uint
+	Compression  CompressionMode
+	PackSize     uint
+	NoVerifyPack bool
 }
 
 // CompressionMode configures if data should be compressed.
@@ -444,6 +445,10 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data
 }
 
 func (r *Repository) verifyCiphertext(buf []byte, uncompressedLength int, id restic.ID) error {
+	if r.opts.NoVerifyPack {
+		return nil
+	}
+
 	nonce, ciphertext := buf[:r.key.NonceSize()], buf[r.key.NonceSize():]
 	plaintext, err := r.key.Open(nil, nonce, ciphertext, nil)
 	if err != nil {
@@ -539,6 +544,10 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, buf []
 }
 
 func (r *Repository) verifyUnpacked(buf []byte, t restic.FileType, expected []byte) error {
+	if r.opts.NoVerifyPack {
+		return nil
+	}
+
 	nonce, ciphertext := buf[:r.key.NonceSize()], buf[r.key.NonceSize():]
 	plaintext, err := r.key.Open(nil, nonce, ciphertext, nil)
 	if err != nil {

From 8ef5425351944a1d52fa76eb510cbd619c10ea8c Mon Sep 17 00:00:00 2001
From: Michael Eischer <michael.eischer@fau.de>
Date: Sat, 3 Feb 2024 18:13:34 +0100
Subject: [PATCH 07/19] repository: test verification of blobs/unpacked data

---
 .../repository/repository_internal_test.go    | 100 ++++++++++++++++++
 1 file changed, 100 insertions(+)

diff --git a/internal/repository/repository_internal_test.go b/internal/repository/repository_internal_test.go
index e5ab6e5b7..2a9976ace 100644
--- a/internal/repository/repository_internal_test.go
+++ b/internal/repository/repository_internal_test.go
@@ -3,8 +3,10 @@ package repository
 import (
 	"math/rand"
 	"sort"
+	"strings"
 	"testing"
 
+	"github.com/restic/restic/internal/crypto"
 	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
 )
@@ -72,3 +74,101 @@ func BenchmarkSortCachedPacksFirst(b *testing.B) {
 		sortCachedPacksFirst(cache, cpy[:])
 	}
 }
+
+func TestBlobVerification(t *testing.T) {
+	repo := TestRepository(t).(*Repository)
+
+	type DamageType string
+	const (
+		damageData       DamageType = "data"
+		damageCompressed DamageType = "compressed"
+		damageCiphertext DamageType = "ciphertext"
+	)
+
+	for _, test := range []struct {
+		damage DamageType
+		msg    string
+	}{
+		{"", ""},
+		{damageData, "hash mismatch"},
+		{damageCompressed, "decompression failed"},
+		{damageCiphertext, "ciphertext verification failed"},
+	} {
+		plaintext := rtest.Random(800, 1234)
+		id := restic.Hash(plaintext)
+		if test.damage == damageData {
+			plaintext[42] ^= 0x42
+		}
+
+		uncompressedLength := uint(len(plaintext))
+		plaintext = repo.getZstdEncoder().EncodeAll(plaintext, nil)
+
+		if test.damage == damageCompressed {
+			plaintext = plaintext[:len(plaintext)-8]
+		}
+
+		nonce := crypto.NewRandomNonce()
+		ciphertext := append([]byte{}, nonce...)
+		ciphertext = repo.Key().Seal(ciphertext, nonce, plaintext, nil)
+
+		if test.damage == damageCiphertext {
+			ciphertext[42] ^= 0x42
+		}
+
+		err := repo.verifyCiphertext(ciphertext, int(uncompressedLength), id)
+		if test.msg == "" {
+			rtest.Assert(t, err == nil, "expected no error, got %v", err)
+		} else {
+			rtest.Assert(t, strings.Contains(err.Error(), test.msg), "expected error to contain %q, got %q", test.msg, err)
+		}
+	}
+}
+
+func TestUnpackedVerification(t *testing.T) {
+	repo := TestRepository(t).(*Repository)
+
+	type DamageType string
+	const (
+		damageData       DamageType = "data"
+		damageCompressed DamageType = "compressed"
+		damageCiphertext DamageType = "ciphertext"
+	)
+
+	for _, test := range []struct {
+		damage DamageType
+		msg    string
+	}{
+		{"", ""},
+		{damageData, "data mismatch"},
+		{damageCompressed, "decompression failed"},
+		{damageCiphertext, "ciphertext verification failed"},
+	} {
+		plaintext := rtest.Random(800, 1234)
+		orig := append([]byte{}, plaintext...)
+		if test.damage == damageData {
+			plaintext[42] ^= 0x42
+		}
+
+		compressed := []byte{2}
+		compressed = repo.getZstdEncoder().EncodeAll(plaintext, compressed)
+
+		if test.damage == damageCompressed {
+			compressed = compressed[:len(compressed)-8]
+		}
+
+		nonce := crypto.NewRandomNonce()
+		ciphertext := append([]byte{}, nonce...)
+		ciphertext = repo.Key().Seal(ciphertext, nonce, compressed, nil)
+
+		if test.damage == damageCiphertext {
+			ciphertext[42] ^= 0x42
+		}
+
+		err := repo.verifyUnpacked(ciphertext, restic.IndexFile, orig)
+		if test.msg == "" {
+			rtest.Assert(t, err == nil, "expected no error, got %v", err)
+		} else {
+			rtest.Assert(t, strings.Contains(err.Error(), test.msg), "expected error to contain %q, got %q", test.msg, err)
+		}
+	}
+}

From dc11d012bb5d09dd0ae246c2e5c911b6808fde00 Mon Sep 17 00:00:00 2001
From: Michael Eischer <michael.eischer@fau.de>
Date: Sat, 3 Feb 2024 18:17:18 +0100
Subject: [PATCH 08/19] Make --no-verify-pack globally available

Verifying all blobs before upload comes with a notable performance
impact. Allow users to skip it if necessary.
---
 cmd/restic/global.go | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/cmd/restic/global.go b/cmd/restic/global.go
index 9e3a3a7a9..5bdf03bdc 100644
--- a/cmd/restic/global.go
+++ b/cmd/restic/global.go
@@ -67,6 +67,7 @@ type GlobalOptions struct {
 	CleanupCache    bool
 	Compression     repository.CompressionMode
 	PackSize        uint
+	NoVerifyPack    bool
 
 	backend.TransportOptions
 	limiter.Limits
@@ -139,6 +140,7 @@ func init() {
 	f.BoolVar(&globalOptions.InsecureTLS, "insecure-tls", false, "skip TLS certificate verification when connecting to the repository (insecure)")
 	f.BoolVar(&globalOptions.CleanupCache, "cleanup-cache", false, "auto remove old cache directories")
 	f.Var(&globalOptions.Compression, "compression", "compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION)")
+	f.BoolVar(&globalOptions.NoVerifyPack, "no-verify-pack", false, "skip verification of data before upload")
 	f.IntVar(&globalOptions.Limits.UploadKb, "limit-upload", 0, "limits uploads to a maximum `rate` in KiB/s. (default: unlimited)")
 	f.IntVar(&globalOptions.Limits.DownloadKb, "limit-download", 0, "limits downloads to a maximum `rate` in KiB/s. (default: unlimited)")
 	f.UintVar(&globalOptions.PackSize, "pack-size", 0, "set target pack `size` in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)")
@@ -453,8 +455,9 @@ func OpenRepository(ctx context.Context, opts GlobalOptions) (*repository.Reposi
 	}
 
 	s, err := repository.New(be, repository.Options{
-		Compression: opts.Compression,
-		PackSize:    opts.PackSize * 1024 * 1024,
+		Compression:  opts.Compression,
+		PackSize:     opts.PackSize * 1024 * 1024,
+		NoVerifyPack: opts.NoVerifyPack,
 	})
 	if err != nil {
 		return nil, errors.Fatal(err.Error())

From d8916bc3d9987a376f9748da7a306bdad8e79ed4 Mon Sep 17 00:00:00 2001
From: Michael Eischer <michael.eischer@fau.de>
Date: Sat, 3 Feb 2024 18:42:02 +0100
Subject: [PATCH 09/19] repository: ask users to report corrupted data while
 saving blobs

---
 internal/repository/repository.go | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/internal/repository/repository.go b/internal/repository/repository.go
index c7c36a74a..1e253c24a 100644
--- a/internal/repository/repository.go
+++ b/internal/repository/repository.go
@@ -425,8 +425,7 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data
 	ciphertext = r.key.Seal(ciphertext, nonce, data, nil)
 
 	if err := r.verifyCiphertext(ciphertext, uncompressedLength, id); err != nil {
-		// FIXME call to action
-		return 0, fmt.Errorf("detected data corruption while saving blob %v: %w", id, err)
+		return 0, fmt.Errorf("detected data corruption while saving blob %v: %w\nCorrupted blobs are either caused by hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting", id, err)
 	}
 
 	// find suitable packer and add blob
@@ -522,8 +521,7 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, buf []
 	ciphertext = r.key.Seal(ciphertext, nonce, p, nil)
 
 	if err := r.verifyUnpacked(ciphertext, t, buf); err != nil {
-		// FIXME call to action
-		return restic.ID{}, fmt.Errorf("detected data corruption while saving file of type %v: %w", t, err)
+		return restic.ID{}, fmt.Errorf("detected data corruption while saving file of type %v: %w\nCorrupted data is either caused by hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting", t, err)
 	}
 
 	if t == restic.ConfigFile {

From 75e72d826ce9b117430f1d594a24d4954691f60c Mon Sep 17 00:00:00 2001
From: Michael Eischer <michael.eischer@fau.de>
Date: Sun, 4 Feb 2024 11:58:29 +0100
Subject: [PATCH 10/19] pack: verify integrity of pack file header

---
 internal/pack/pack.go               | 43 +++++++++++++++------
 internal/pack/pack_internal_test.go | 58 +++++++++++++++++++++++++++++
 2 files changed, 90 insertions(+), 11 deletions(-)

diff --git a/internal/pack/pack.go b/internal/pack/pack.go
index 34ad9d071..34e87f1f9 100644
--- a/internal/pack/pack.go
+++ b/internal/pack/pack.go
@@ -1,6 +1,7 @@
 package pack
 
 import (
+	"bytes"
 	"context"
 	"encoding/binary"
 	"fmt"
@@ -74,7 +75,7 @@ func (p *Packer) Finalize() error {
 	p.m.Lock()
 	defer p.m.Unlock()
 
-	header, err := p.makeHeader()
+	header, err := makeHeader(p.blobs)
 	if err != nil {
 		return err
 	}
@@ -83,6 +84,11 @@ func (p *Packer) Finalize() error {
 	nonce := crypto.NewRandomNonce()
 	encryptedHeader = append(encryptedHeader, nonce...)
 	encryptedHeader = p.k.Seal(encryptedHeader, nonce, header, nil)
+	encryptedHeader = binary.LittleEndian.AppendUint32(encryptedHeader, uint32(len(encryptedHeader)))
+
+	if err := verifyHeader(p.k, encryptedHeader, p.blobs); err != nil {
+		return fmt.Errorf("detected data corruption while writing pack-file header: %w\nCorrupted data is either caused by hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting", err)
+	}
 
 	// append the header
 	n, err := p.wr.Write(encryptedHeader)
@@ -90,18 +96,33 @@ func (p *Packer) Finalize() error {
 		return errors.Wrap(err, "Write")
 	}
 
-	hdrBytes := len(encryptedHeader)
-	if n != hdrBytes {
+	if n != len(encryptedHeader) {
 		return errors.New("wrong number of bytes written")
 	}
+	p.bytes += uint(len(encryptedHeader))
 
-	// write length
-	err = binary.Write(p.wr, binary.LittleEndian, uint32(hdrBytes))
+	return nil
+}
+
+func verifyHeader(k *crypto.Key, header []byte, expected []restic.Blob) error {
+	// do not offer a way to skip the pack header verification, as pack headers are usually small enough
+	// to not result in a significant performance impact
+
+	decoded, hdrSize, err := List(k, bytes.NewReader(header), int64(len(header)))
 	if err != nil {
-		return errors.Wrap(err, "binary.Write")
+		return fmt.Errorf("header decoding failed: %w", err)
+	}
+	if hdrSize != uint32(len(header)) {
+		return fmt.Errorf("unexpected header size %v instead of %v", hdrSize, len(header))
+	}
+	if len(decoded) != len(expected) {
+		return fmt.Errorf("pack header size mismatch")
+	}
+	for i := 0; i < len(decoded); i++ {
+		if decoded[i] != expected[i] {
+			return fmt.Errorf("pack header entry mismatch got %v instead of %v", decoded[i], expected[i])
+		}
 	}
-	p.bytes += uint(hdrBytes + binary.Size(uint32(0)))
-
 	return nil
 }
 
@@ -111,10 +132,10 @@ func (p *Packer) HeaderOverhead() int {
 }
 
 // makeHeader constructs the header for p.
-func (p *Packer) makeHeader() ([]byte, error) {
-	buf := make([]byte, 0, len(p.blobs)*int(entrySize))
+func makeHeader(blobs []restic.Blob) ([]byte, error) {
+	buf := make([]byte, 0, len(blobs)*int(entrySize))
 
-	for _, b := range p.blobs {
+	for _, b := range blobs {
 		switch {
 		case b.Type == restic.DataBlob && b.UncompressedLength == 0:
 			buf = append(buf, 0)
diff --git a/internal/pack/pack_internal_test.go b/internal/pack/pack_internal_test.go
index c1a4867ea..2e7400ad0 100644
--- a/internal/pack/pack_internal_test.go
+++ b/internal/pack/pack_internal_test.go
@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/binary"
 	"io"
+	"strings"
 	"testing"
 
 	"github.com/restic/restic/internal/crypto"
@@ -177,3 +178,60 @@ func TestReadRecords(t *testing.T) {
 		}
 	}
 }
+
+func TestUnpackedVerification(t *testing.T) {
+	// create random keys
+	k := crypto.NewRandomKey()
+	blobs := []restic.Blob{
+		{
+			BlobHandle:         restic.NewRandomBlobHandle(),
+			Length:             42,
+			Offset:             0,
+			UncompressedLength: 2 * 42,
+		},
+	}
+
+	type DamageType string
+	const (
+		damageData       DamageType = "data"
+		damageCiphertext DamageType = "ciphertext"
+		damageLength     DamageType = "length"
+	)
+
+	for _, test := range []struct {
+		damage DamageType
+		msg    string
+	}{
+		{"", ""},
+		{damageData, "pack header entry mismatch"},
+		{damageCiphertext, "ciphertext verification failed"},
+		{damageLength, "header decoding failed"},
+	} {
+		header, err := makeHeader(blobs)
+		rtest.OK(t, err)
+
+		if test.damage == damageData {
+			header[8] ^= 0x42
+		}
+
+		encryptedHeader := make([]byte, 0, crypto.CiphertextLength(len(header)))
+		nonce := crypto.NewRandomNonce()
+		encryptedHeader = append(encryptedHeader, nonce...)
+		encryptedHeader = k.Seal(encryptedHeader, nonce, header, nil)
+		encryptedHeader = binary.LittleEndian.AppendUint32(encryptedHeader, uint32(len(encryptedHeader)))
+
+		if test.damage == damageCiphertext {
+			encryptedHeader[8] ^= 0x42
+		}
+		if test.damage == damageLength {
+			encryptedHeader[len(encryptedHeader)-1] ^= 0x42
+		}
+
+		err = verifyHeader(k, encryptedHeader, blobs)
+		if test.msg == "" {
+			rtest.Assert(t, err == nil, "expected no error, got %v", err)
+		} else {
+			rtest.Assert(t, strings.Contains(err.Error(), test.msg), "expected error to contain %q, got %q", test.msg, err)
+		}
+	}
+}

From 4589da7eb9bc9037c9dc3b85ee0ced0569aeb2bf Mon Sep 17 00:00:00 2001
From: Michael Eischer <michael.eischer@fau.de>
Date: Sun, 4 Feb 2024 15:48:11 +0100
Subject: [PATCH 11/19] add data verification changelog entry

---
 changelog/unreleased/issue-4529 | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 changelog/unreleased/issue-4529

diff --git a/changelog/unreleased/issue-4529 b/changelog/unreleased/issue-4529
new file mode 100644
index 000000000..2e8bbbed7
--- /dev/null
+++ b/changelog/unreleased/issue-4529
@@ -0,0 +1,14 @@
+Enhancement: Verify data integrity before upload
+
+Hardware issues or a bug in restic could cause restic to create corrupted files
+that were then uploaded to the repository. Detecting such corruption usually
+required explicitly running the `check --read-data` command.
+
+To prevent the upload of corrupted data to the repository, restic now
+additionally verifies that files can be decoded and contain the correct data
+beforehand. This increases the CPU usage during backups. If absolutely
+necessary, you can disable the verification using the option
+`--no-verify-pack`.
+
+https://github.com/restic/restic/issues/4529
+https://github.com/restic/restic/pull/4681

From 2a0bd2b637b4b791f92f3d84c565aaaf39b9ddb2 Mon Sep 17 00:00:00 2001
From: Michael Eischer <michael.eischer@fau.de>
Date: Sun, 4 Feb 2024 16:50:50 +0100
Subject: [PATCH 12/19] rename `--no-verify-pack` to `--no-extra-verify`

---
 changelog/unreleased/issue-4529    |  2 +-
 cmd/restic/global.go               | 10 +++++-----
 internal/repository/repack_test.go |  4 ++--
 internal/repository/repository.go  | 10 +++++-----
 4 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/changelog/unreleased/issue-4529 b/changelog/unreleased/issue-4529
index 2e8bbbed7..c3ec69510 100644
--- a/changelog/unreleased/issue-4529
+++ b/changelog/unreleased/issue-4529
@@ -8,7 +8,7 @@ To prevent the upload of corrupted data to the repository, restic now
 additionally verifies that files can be decoded and contain the correct data
 beforehand. This increases the CPU usage during backups. If absolutely
 necessary, you can disable the verification using the option
-`--no-verify-pack`.
+`--no-extra-verify`.
 
 https://github.com/restic/restic/issues/4529
 https://github.com/restic/restic/pull/4681
diff --git a/cmd/restic/global.go b/cmd/restic/global.go
index 5bdf03bdc..8568e41c3 100644
--- a/cmd/restic/global.go
+++ b/cmd/restic/global.go
@@ -67,7 +67,7 @@ type GlobalOptions struct {
 	CleanupCache    bool
 	Compression     repository.CompressionMode
 	PackSize        uint
-	NoVerifyPack    bool
+	NoExtraVerify   bool
 
 	backend.TransportOptions
 	limiter.Limits
@@ -140,7 +140,7 @@ func init() {
 	f.BoolVar(&globalOptions.InsecureTLS, "insecure-tls", false, "skip TLS certificate verification when connecting to the repository (insecure)")
 	f.BoolVar(&globalOptions.CleanupCache, "cleanup-cache", false, "auto remove old cache directories")
 	f.Var(&globalOptions.Compression, "compression", "compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION)")
-	f.BoolVar(&globalOptions.NoVerifyPack, "no-verify-pack", false, "skip verification of data before upload")
+	f.BoolVar(&globalOptions.NoExtraVerify, "no-extra-verify", false, "skip verification of data before upload")
 	f.IntVar(&globalOptions.Limits.UploadKb, "limit-upload", 0, "limits uploads to a maximum `rate` in KiB/s. (default: unlimited)")
 	f.IntVar(&globalOptions.Limits.DownloadKb, "limit-download", 0, "limits downloads to a maximum `rate` in KiB/s. (default: unlimited)")
 	f.UintVar(&globalOptions.PackSize, "pack-size", 0, "set target pack `size` in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)")
@@ -455,9 +455,9 @@ func OpenRepository(ctx context.Context, opts GlobalOptions) (*repository.Reposi
 	}
 
 	s, err := repository.New(be, repository.Options{
-		Compression:  opts.Compression,
-		PackSize:     opts.PackSize * 1024 * 1024,
-		NoVerifyPack: opts.NoVerifyPack,
+		Compression:   opts.Compression,
+		PackSize:      opts.PackSize * 1024 * 1024,
+		NoExtraVerify: opts.NoExtraVerify,
 	})
 	if err != nil {
 		return nil, errors.Fatal(err.Error())
diff --git a/internal/repository/repack_test.go b/internal/repository/repack_test.go
index 00567aad9..1ecbf3e1c 100644
--- a/internal/repository/repack_test.go
+++ b/internal/repository/repack_test.go
@@ -347,7 +347,7 @@ func TestRepackWrongBlob(t *testing.T) {
 
 func testRepackWrongBlob(t *testing.T, version uint) {
 	// disable verification to allow adding corrupted blobs to the repository
-	repo := repository.TestRepositoryWithBackend(t, nil, version, repository.Options{NoVerifyPack: true})
+	repo := repository.TestRepositoryWithBackend(t, nil, version, repository.Options{NoExtraVerify: true})
 
 	seed := time.Now().UnixNano()
 	rand.Seed(seed)
@@ -373,7 +373,7 @@ func TestRepackBlobFallback(t *testing.T) {
 
 func testRepackBlobFallback(t *testing.T, version uint) {
 	// disable verification to allow adding corrupted blobs to the repository
-	repo := repository.TestRepositoryWithBackend(t, nil, version, repository.Options{NoVerifyPack: true})
+	repo := repository.TestRepositoryWithBackend(t, nil, version, repository.Options{NoExtraVerify: true})
 
 	seed := time.Now().UnixNano()
 	rand.Seed(seed)
diff --git a/internal/repository/repository.go b/internal/repository/repository.go
index 1e253c24a..cea43b0d3 100644
--- a/internal/repository/repository.go
+++ b/internal/repository/repository.go
@@ -59,9 +59,9 @@ type Repository struct {
 }
 
 type Options struct {
-	Compression  CompressionMode
-	PackSize     uint
-	NoVerifyPack bool
+	Compression   CompressionMode
+	PackSize      uint
+	NoExtraVerify bool
 }
 
 // CompressionMode configures if data should be compressed.
@@ -444,7 +444,7 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data
 }
 
 func (r *Repository) verifyCiphertext(buf []byte, uncompressedLength int, id restic.ID) error {
-	if r.opts.NoVerifyPack {
+	if r.opts.NoExtraVerify {
 		return nil
 	}
 
@@ -542,7 +542,7 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, buf []
 }
 
 func (r *Repository) verifyUnpacked(buf []byte, t restic.FileType, expected []byte) error {
-	if r.opts.NoVerifyPack {
+	if r.opts.NoExtraVerify {
 		return nil
 	}
 

From 261b1455c79335152edb74cbc3626a6248d568f7 Mon Sep 17 00:00:00 2001
From: Michael Eischer <michael.eischer@fau.de>
Date: Sun, 4 Feb 2024 17:11:49 +0100
Subject: [PATCH 13/19] add documentation for --no-extra-verify option

---
 doc/047_tuning_backup_parameters.rst | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/doc/047_tuning_backup_parameters.rst b/doc/047_tuning_backup_parameters.rst
index 6ea39dc75..5773ac161 100644
--- a/doc/047_tuning_backup_parameters.rst
+++ b/doc/047_tuning_backup_parameters.rst
@@ -60,6 +60,17 @@ only applied for the single run of restic. The option can also be set via the en
 variable ``RESTIC_COMPRESSION``.
 
 
+Data Verification
+=================
+
+To prevent the upload of corrupted data to the repository, restic verifies that files can
+be decoded and contain the correct data beforehand. This increases the CPU usage during
+backups. If necessary, you can disable this verification using the option ``--no-extra-verify``.
+However, in this case you should verify the repository integrity more actively using
+``restic check --read-data``. Otherwise, data corruption due to hardware issues or software
+bugs might go unnoticed.
+
+
 File Read Concurrency
 =====================
 

From a72c2b74f3a3ffa259e406462b4afda49726187d Mon Sep 17 00:00:00 2001
From: Michael Eischer <michael.eischer@fau.de>
Date: Sun, 4 Feb 2024 18:09:32 +0100
Subject: [PATCH 14/19] Apply changelog entry / documentation improvements from
 review

---
 changelog/unreleased/issue-4529      | 22 +++++++++++++---------
 cmd/restic/global.go                 |  2 +-
 doc/047_tuning_backup_parameters.rst | 15 +++++++++------
 internal/pack/pack.go                |  3 ++-
 internal/repository/repository.go    |  6 ++++--
 5 files changed, 29 insertions(+), 19 deletions(-)

diff --git a/changelog/unreleased/issue-4529 b/changelog/unreleased/issue-4529
index c3ec69510..fed726d2d 100644
--- a/changelog/unreleased/issue-4529
+++ b/changelog/unreleased/issue-4529
@@ -1,14 +1,18 @@
-Enhancement: Verify data integrity before upload
+Enhancement: Add extra verification of data integrity before upload
 
-Hardware issues or a bug in restic could cause restic to create corrupted files
-that were then uploaded to the repository. Detecting such corruption usually
-required explicitly running the `check --read-data` command.
+Hardware issues, or a bug in restic or its dependencies, could previously cause
+corruption in the files restic created and stored in the repository. Detecting
+such corruption previously required explicitly running the `check --read-data`
+or `check --read-data-subset` commands.
 
-To prevent the upload of corrupted data to the repository, restic now
-additionally verifies that files can be decoded and contain the correct data
-beforehand. This increases the CPU usage during backups. If absolutely
-necessary, you can disable the verification using the option
-`--no-extra-verify`.
+To further ensure data integrity, even in the case of hardware issues or
+software bugs, restic now performs additional verification of the files about
+to be uploaded to the repository.
+
+These extra checks will increase CPU usage during backups. They can therefore,
+if absolutely necessary, be disabled using the `--no-extra-verify` global
+option. Please note that this should be combined with more active checking
+using the previously mentioned check commands.
 
 https://github.com/restic/restic/issues/4529
 https://github.com/restic/restic/pull/4681
diff --git a/cmd/restic/global.go b/cmd/restic/global.go
index 8568e41c3..528c6e129 100644
--- a/cmd/restic/global.go
+++ b/cmd/restic/global.go
@@ -140,7 +140,7 @@ func init() {
 	f.BoolVar(&globalOptions.InsecureTLS, "insecure-tls", false, "skip TLS certificate verification when connecting to the repository (insecure)")
 	f.BoolVar(&globalOptions.CleanupCache, "cleanup-cache", false, "auto remove old cache directories")
 	f.Var(&globalOptions.Compression, "compression", "compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION)")
-	f.BoolVar(&globalOptions.NoExtraVerify, "no-extra-verify", false, "skip verification of data before upload")
+	f.BoolVar(&globalOptions.NoExtraVerify, "no-extra-verify", false, "skip additional verification of data before upload (see documentation)")
 	f.IntVar(&globalOptions.Limits.UploadKb, "limit-upload", 0, "limits uploads to a maximum `rate` in KiB/s. (default: unlimited)")
 	f.IntVar(&globalOptions.Limits.DownloadKb, "limit-download", 0, "limits downloads to a maximum `rate` in KiB/s. (default: unlimited)")
 	f.UintVar(&globalOptions.PackSize, "pack-size", 0, "set target pack `size` in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)")
diff --git a/doc/047_tuning_backup_parameters.rst b/doc/047_tuning_backup_parameters.rst
index 5773ac161..d8fb2c9b6 100644
--- a/doc/047_tuning_backup_parameters.rst
+++ b/doc/047_tuning_backup_parameters.rst
@@ -63,12 +63,15 @@ variable ``RESTIC_COMPRESSION``.
 Data Verification
 =================
 
-To prevent the upload of corrupted data to the repository, restic verifies that files can
-be decoded and contain the correct data beforehand. This increases the CPU usage during
-backups. If necessary, you can disable this verification using the option ``--no-extra-verify``.
-However, in this case you should verify the repository integrity more actively using
-``restic check --read-data``. Otherwise, data corruption due to hardware issues or software
-bugs might go unnoticed.
+To prevent the upload of corrupted data to the repository, which can happen due
+to hardware issues or software bugs, restic verifies that generated files can
+be decoded and contain the correct data beforehand. This increases the CPU usage
+during backups. If necessary, you can disable this verification using the
+``--no-extra-verify`` option of the ``backup`` command. However, in this case
+you should verify the repository integrity more actively using
+``restic check --read-data`` (or the similar ``--read-data-subset`` option).
+Otherwise, data corruption due to hardware issues or software bugs might go
+unnoticed.
 
 
 File Read Concurrency
diff --git a/internal/pack/pack.go b/internal/pack/pack.go
index 34e87f1f9..f9e7896e0 100644
--- a/internal/pack/pack.go
+++ b/internal/pack/pack.go
@@ -87,7 +87,8 @@ func (p *Packer) Finalize() error {
 	encryptedHeader = binary.LittleEndian.AppendUint32(encryptedHeader, uint32(len(encryptedHeader)))
 
 	if err := verifyHeader(p.k, encryptedHeader, p.blobs); err != nil {
-		return fmt.Errorf("detected data corruption while writing pack-file header: %w\nCorrupted data is either caused by hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting", err)
+		//nolint:revive // ignore linter warnings about error message spelling
+		return fmt.Errorf("Detected data corruption while writing pack-file header: %w\nCorrupted data is either caused by hardware issues or software bugs. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting.", err)
 	}
 
 	// append the header
diff --git a/internal/repository/repository.go b/internal/repository/repository.go
index cea43b0d3..40794508f 100644
--- a/internal/repository/repository.go
+++ b/internal/repository/repository.go
@@ -425,7 +425,8 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data
 	ciphertext = r.key.Seal(ciphertext, nonce, data, nil)
 
 	if err := r.verifyCiphertext(ciphertext, uncompressedLength, id); err != nil {
-		return 0, fmt.Errorf("detected data corruption while saving blob %v: %w\nCorrupted blobs are either caused by hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting", id, err)
+		//nolint:revive // ignore linter warnings about error message spelling
+		return 0, fmt.Errorf("Detected data corruption while saving blob %v: %w\nCorrupted blobs are either caused by hardware issues or software bugs. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting.", id, err)
 	}
 
 	// find suitable packer and add blob
@@ -521,7 +522,8 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, buf []
 	ciphertext = r.key.Seal(ciphertext, nonce, p, nil)
 
 	if err := r.verifyUnpacked(ciphertext, t, buf); err != nil {
-		return restic.ID{}, fmt.Errorf("detected data corruption while saving file of type %v: %w\nCorrupted data is either caused by hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting", t, err)
+		//nolint:revive // ignore linter warnings about error message spelling
+		return restic.ID{}, fmt.Errorf("Detected data corruption while saving file of type %v: %w\nCorrupted data is either caused by hardware issues or software bugs. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting.", t, err)
 	}
 
 	if t == restic.ConfigFile {

From 6cd2804bfff06957e835e76d7a67c444d202dcbf Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Sun, 4 Feb 2024 19:50:34 +0100
Subject: [PATCH 15/19] Prepare changelog for 0.16.4

---
 changelog/{unreleased => 0.16.4_2024-02-04}/issue-4529 | 0
 changelog/{unreleased => 0.16.4_2024-02-04}/issue-4677 | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 rename changelog/{unreleased => 0.16.4_2024-02-04}/issue-4529 (100%)
 rename changelog/{unreleased => 0.16.4_2024-02-04}/issue-4677 (100%)

diff --git a/changelog/unreleased/issue-4529 b/changelog/0.16.4_2024-02-04/issue-4529
similarity index 100%
rename from changelog/unreleased/issue-4529
rename to changelog/0.16.4_2024-02-04/issue-4529
diff --git a/changelog/unreleased/issue-4677 b/changelog/0.16.4_2024-02-04/issue-4677
similarity index 100%
rename from changelog/unreleased/issue-4677
rename to changelog/0.16.4_2024-02-04/issue-4677

From b0ead75de5d0f50b87bfe5146d5544dd3d92c28f Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Sun, 4 Feb 2024 19:50:34 +0100
Subject: [PATCH 16/19] Generate CHANGELOG.md for 0.16.4

---
 CHANGELOG.md | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 52 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index ba6fa7a11..b8969a443 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,6 @@
 # Table of Contents
 
+* [Changelog for 0.16.4](#changelog-for-restic-0164-2024-02-04)
 * [Changelog for 0.16.3](#changelog-for-restic-0163-2024-01-14)
 * [Changelog for 0.16.2](#changelog-for-restic-0162-2023-10-29)
 * [Changelog for 0.16.1](#changelog-for-restic-0161-2023-10-24)
@@ -32,6 +33,57 @@
 * [Changelog for 0.6.0](#changelog-for-restic-060-2017-05-29)
 
 
+# Changelog for restic 0.16.4 (2024-02-04)
+The following sections list the changes in restic 0.16.4 relevant to
+restic users. The changes are ordered by importance.
+
+## Summary
+
+ * Fix #4677: Downgrade zstd library to fix rare data corruption at max. compression
+ * Enh #4529: Add extra verification of data integrity before upload
+
+## Details
+
+ * Bugfix #4677: Downgrade zstd library to fix rare data corruption at max. compression
+
+   In restic 0.16.3, backups where the compression level was set to `max` (using
+   `--compression max`) could in rare and very specific circumstances result in
+   data corruption due to a bug in the library used for compressing data. Restic
+   0.16.1 and 0.16.2 were not affected.
+
+   Restic now uses the previous version of the library used to compress data, the
+   same version used by restic 0.16.2. Please note that the `auto` compression
+   level (which restic uses by default) was never affected, and even if you used
+   `max` compression, chances of being affected by this issue are small.
+
+   To check a repository for any corruption, run `restic check --read-data`. This
+   will download and verify the whole repository and can be used at any time to
+   completely verify the integrity of a repository. If the `check` command detects
+   anomalies, follow the suggested steps.
+
+   https://github.com/restic/restic/issues/4677
+   https://github.com/restic/restic/pull/4679
+
+ * Enhancement #4529: Add extra verification of data integrity before upload
+
+   Hardware issues, or a bug in restic or its dependencies, could previously cause
+   corruption in the files restic created and stored in the repository. Detecting
+   such corruption previously required explicitly running the `check --read-data`
+   or `check --read-data-subset` commands.
+
+   To further ensure data integrity, even in the case of hardware issues or
+   software bugs, restic now performs additional verification of the files about to
+   be uploaded to the repository.
+
+   These extra checks will increase CPU usage during backups. They can therefore,
+   if absolutely necessary, be disabled using the `--no-extra-verify` global
+   option. Please note that this should be combined with more active checking using
+   the previously mentioned check commands.
+
+   https://github.com/restic/restic/issues/4529
+   https://github.com/restic/restic/pull/4681
+
+
 # Changelog for restic 0.16.3 (2024-01-14)
 The following sections list the changes in restic 0.16.3 relevant to
 restic users. The changes are ordered by importance.

From 811be5984d5d42acf58714d13aeb93cebf4fe2e0 Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Sun, 4 Feb 2024 19:50:51 +0100
Subject: [PATCH 17/19] Update manpages and auto-completion

---
 doc/bash-completion.sh            | 33 +++++++++++++++++++++++++++++++
 doc/man/restic-backup.1           |  4 ++++
 doc/man/restic-cache.1            |  4 ++++
 doc/man/restic-cat.1              |  4 ++++
 doc/man/restic-check.1            |  4 ++++
 doc/man/restic-copy.1             |  4 ++++
 doc/man/restic-diff.1             |  4 ++++
 doc/man/restic-dump.1             |  4 ++++
 doc/man/restic-find.1             |  4 ++++
 doc/man/restic-forget.1           |  4 ++++
 doc/man/restic-generate.1         |  4 ++++
 doc/man/restic-init.1             |  4 ++++
 doc/man/restic-key.1              |  4 ++++
 doc/man/restic-list.1             |  4 ++++
 doc/man/restic-ls.1               |  4 ++++
 doc/man/restic-migrate.1          |  4 ++++
 doc/man/restic-mount.1            |  4 ++++
 doc/man/restic-prune.1            |  4 ++++
 doc/man/restic-recover.1          |  4 ++++
 doc/man/restic-repair-index.1     |  4 ++++
 doc/man/restic-repair-packs.1     |  4 ++++
 doc/man/restic-repair-snapshots.1 |  4 ++++
 doc/man/restic-repair.1           |  4 ++++
 doc/man/restic-restore.1          |  4 ++++
 doc/man/restic-rewrite.1          |  4 ++++
 doc/man/restic-self-update.1      |  4 ++++
 doc/man/restic-snapshots.1        |  4 ++++
 doc/man/restic-stats.1            |  4 ++++
 doc/man/restic-tag.1              |  4 ++++
 doc/man/restic-unlock.1           |  4 ++++
 doc/man/restic-version.1          |  4 ++++
 doc/man/restic.1                  |  4 ++++
 32 files changed, 157 insertions(+)

diff --git a/doc/bash-completion.sh b/doc/bash-completion.sh
index e691af363..cae37a6ca 100644
--- a/doc/bash-completion.sh
+++ b/doc/bash-completion.sh
@@ -488,6 +488,7 @@ _restic_backup()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -560,6 +561,7 @@ _restic_cache()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -624,6 +626,7 @@ _restic_cat()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -696,6 +699,7 @@ _restic_check()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -794,6 +798,7 @@ _restic_copy()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -860,6 +865,7 @@ _restic_diff()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -944,6 +950,7 @@ _restic_dump()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -1058,6 +1065,7 @@ _restic_find()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -1228,6 +1236,7 @@ _restic_forget()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -1312,6 +1321,7 @@ _restic_generate()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -1372,6 +1382,7 @@ _restic_help()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -1463,6 +1474,7 @@ _restic_init()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -1539,6 +1551,7 @@ _restic_key()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -1603,6 +1616,7 @@ _restic_list()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -1689,6 +1703,7 @@ _restic_ls()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -1757,6 +1772,7 @@ _restic_migrate()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -1849,6 +1865,7 @@ _restic_mount()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -1935,6 +1952,7 @@ _restic_prune()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -1999,6 +2017,7 @@ _restic_recover()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -2059,6 +2078,7 @@ _restic_repair_help()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -2126,6 +2146,7 @@ _restic_repair_index()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -2190,6 +2211,7 @@ _restic_repair_packs()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -2274,6 +2296,7 @@ _restic_repair_snapshots()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -2342,6 +2365,7 @@ _restic_repair()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -2450,6 +2474,7 @@ _restic_restore()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -2552,6 +2577,7 @@ _restic_rewrite()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -2620,6 +2646,7 @@ _restic_self-update()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -2712,6 +2739,7 @@ _restic_snapshots()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -2794,6 +2822,7 @@ _restic_stats()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -2884,6 +2913,7 @@ _restic_tag()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -2950,6 +2980,7 @@ _restic_unlock()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -3014,6 +3045,7 @@ _restic_version()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -3106,6 +3138,7 @@ _restic_root_command()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
diff --git a/doc/man/restic-backup.1 b/doc/man/restic-backup.1
index c3bccdfa5..730685271 100644
--- a/doc/man/restic-backup.1
+++ b/doc/man/restic-backup.1
@@ -171,6 +171,10 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories
diff --git a/doc/man/restic-cache.1 b/doc/man/restic-cache.1
index 3ae27ea57..c170c1624 100644
--- a/doc/man/restic-cache.1
+++ b/doc/man/restic-cache.1
@@ -80,6 +80,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories
diff --git a/doc/man/restic-cat.1 b/doc/man/restic-cat.1
index c1df138aa..b42a58e14 100644
--- a/doc/man/restic-cat.1
+++ b/doc/man/restic-cat.1
@@ -68,6 +68,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories
diff --git a/doc/man/restic-check.1 b/doc/man/restic-check.1
index 17eb972bc..9c1dc77e5 100644
--- a/doc/man/restic-check.1
+++ b/doc/man/restic-check.1
@@ -85,6 +85,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories
diff --git a/doc/man/restic-copy.1 b/doc/man/restic-copy.1
index be8f21e25..bd9795f44 100644
--- a/doc/man/restic-copy.1
+++ b/doc/man/restic-copy.1
@@ -109,6 +109,10 @@ new destination repository using the "init" command.
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories
diff --git a/doc/man/restic-diff.1 b/doc/man/restic-diff.1
index a01a2562b..28f3a4838 100644
--- a/doc/man/restic-diff.1
+++ b/doc/man/restic-diff.1
@@ -93,6 +93,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories
diff --git a/doc/man/restic-dump.1 b/doc/man/restic-dump.1
index 6fa1f8200..7fa3f777d 100644
--- a/doc/man/restic-dump.1
+++ b/doc/man/restic-dump.1
@@ -96,6 +96,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories
diff --git a/doc/man/restic-find.1 b/doc/man/restic-find.1
index 72bc3a0b6..c3297c43f 100644
--- a/doc/man/restic-find.1
+++ b/doc/man/restic-find.1
@@ -117,6 +117,10 @@ It can also be used to search for restic blobs or trees for troubleshooting.
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories
diff --git a/doc/man/restic-forget.1 b/doc/man/restic-forget.1
index 757022a21..d0c4cfc74 100644
--- a/doc/man/restic-forget.1
+++ b/doc/man/restic-forget.1
@@ -179,6 +179,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories
diff --git a/doc/man/restic-generate.1 b/doc/man/restic-generate.1
index aef3a5e55..84f659ef2 100644
--- a/doc/man/restic-generate.1
+++ b/doc/man/restic-generate.1
@@ -89,6 +89,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories
diff --git a/doc/man/restic-init.1 b/doc/man/restic-init.1
index 27d7f5874..5f19c8f8c 100644
--- a/doc/man/restic-init.1
+++ b/doc/man/restic-init.1
@@ -96,6 +96,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories
diff --git a/doc/man/restic-key.1 b/doc/man/restic-key.1
index 855ef5443..8d1813188 100644
--- a/doc/man/restic-key.1
+++ b/doc/man/restic-key.1
@@ -80,6 +80,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories
diff --git a/doc/man/restic-list.1 b/doc/man/restic-list.1
index 95eeac5f7..e399038a2 100644
--- a/doc/man/restic-list.1
+++ b/doc/man/restic-list.1
@@ -68,6 +68,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories
diff --git a/doc/man/restic-ls.1 b/doc/man/restic-ls.1
index 0cd0f5a88..10b0657a3 100644
--- a/doc/man/restic-ls.1
+++ b/doc/man/restic-ls.1
@@ -107,6 +107,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories
diff --git a/doc/man/restic-migrate.1 b/doc/man/restic-migrate.1
index eca0ef8e1..7e48f726c 100644
--- a/doc/man/restic-migrate.1
+++ b/doc/man/restic-migrate.1
@@ -74,6 +74,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories
diff --git a/doc/man/restic-mount.1 b/doc/man/restic-mount.1
index 33c016ffa..aab607fcf 100644
--- a/doc/man/restic-mount.1
+++ b/doc/man/restic-mount.1
@@ -144,6 +144,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories
diff --git a/doc/man/restic-prune.1 b/doc/man/restic-prune.1
index e4a32cac3..c54d5d7ff 100644
--- a/doc/man/restic-prune.1
+++ b/doc/man/restic-prune.1
@@ -97,6 +97,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories
diff --git a/doc/man/restic-recover.1 b/doc/man/restic-recover.1
index 26d2fc7bd..010fbafd7 100644
--- a/doc/man/restic-recover.1
+++ b/doc/man/restic-recover.1
@@ -70,6 +70,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories
diff --git a/doc/man/restic-repair-index.1 b/doc/man/restic-repair-index.1
index 35e2845b8..f06be64c0 100644
--- a/doc/man/restic-repair-index.1
+++ b/doc/man/restic-repair-index.1
@@ -73,6 +73,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories
diff --git a/doc/man/restic-repair-packs.1 b/doc/man/restic-repair-packs.1
index b21211925..f3671fe18 100644
--- a/doc/man/restic-repair-packs.1
+++ b/doc/man/restic-repair-packs.1
@@ -72,6 +72,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories
diff --git a/doc/man/restic-repair-snapshots.1 b/doc/man/restic-repair-snapshots.1
index f59067f05..9369f25f2 100644
--- a/doc/man/restic-repair-snapshots.1
+++ b/doc/man/restic-repair-snapshots.1
@@ -107,6 +107,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories
diff --git a/doc/man/restic-repair.1 b/doc/man/restic-repair.1
index dbe783df4..77aecc173 100644
--- a/doc/man/restic-repair.1
+++ b/doc/man/restic-repair.1
@@ -63,6 +63,10 @@ Repair the repository
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories
diff --git a/doc/man/restic-restore.1 b/doc/man/restic-restore.1
index d8c1b72e1..4635b1e43 100644
--- a/doc/man/restic-restore.1
+++ b/doc/man/restic-restore.1
@@ -117,6 +117,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories
diff --git a/doc/man/restic-rewrite.1 b/doc/man/restic-rewrite.1
index 8a06aef40..d63c653e6 100644
--- a/doc/man/restic-rewrite.1
+++ b/doc/man/restic-rewrite.1
@@ -121,6 +121,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories
diff --git a/doc/man/restic-self-update.1 b/doc/man/restic-self-update.1
index 28fd24a92..92ab5add3 100644
--- a/doc/man/restic-self-update.1
+++ b/doc/man/restic-self-update.1
@@ -75,6 +75,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories
diff --git a/doc/man/restic-snapshots.1 b/doc/man/restic-snapshots.1
index cb34d6c8e..6203bbf2b 100644
--- a/doc/man/restic-snapshots.1
+++ b/doc/man/restic-snapshots.1
@@ -92,6 +92,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories
diff --git a/doc/man/restic-stats.1 b/doc/man/restic-stats.1
index cf0374351..9d37163de 100644
--- a/doc/man/restic-stats.1
+++ b/doc/man/restic-stats.1
@@ -114,6 +114,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories
diff --git a/doc/man/restic-tag.1 b/doc/man/restic-tag.1
index 162d50d29..b1468c74d 100644
--- a/doc/man/restic-tag.1
+++ b/doc/man/restic-tag.1
@@ -99,6 +99,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories
diff --git a/doc/man/restic-unlock.1 b/doc/man/restic-unlock.1
index 0274c56e8..0b3b43f2a 100644
--- a/doc/man/restic-unlock.1
+++ b/doc/man/restic-unlock.1
@@ -72,6 +72,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories
diff --git a/doc/man/restic-version.1 b/doc/man/restic-version.1
index 774e19453..ccc23038f 100644
--- a/doc/man/restic-version.1
+++ b/doc/man/restic-version.1
@@ -69,6 +69,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories
diff --git a/doc/man/restic.1 b/doc/man/restic.1
index 427ce7c65..333eab76a 100644
--- a/doc/man/restic.1
+++ b/doc/man/restic.1
@@ -65,6 +65,10 @@ The full documentation can be found at https://restic.readthedocs.io/ .
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories

From 3786536dc18ef27aedcfa8e4c6953b48353eee79 Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Sun, 4 Feb 2024 19:50:52 +0100
Subject: [PATCH 18/19] Add version for 0.16.4

---
 VERSION              | 2 +-
 cmd/restic/global.go | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/VERSION b/VERSION
index 7eb3095a3..5f2491c5a 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-0.16.3
+0.16.4
diff --git a/cmd/restic/global.go b/cmd/restic/global.go
index 528c6e129..e979dcc2b 100644
--- a/cmd/restic/global.go
+++ b/cmd/restic/global.go
@@ -43,7 +43,7 @@ import (
 	"golang.org/x/term"
 )
 
-var version = "0.16.3-dev (compiled manually)"
+var version = "0.16.4"
 
 // TimeFormat is the format used for all timestamps printed by restic.
 const TimeFormat = "2006-01-02 15:04:05"

From 0f9fa44de5a27efd22a17c9e49669a29d5ecf33a Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Sun, 4 Feb 2024 19:50:56 +0100
Subject: [PATCH 19/19] Set development version for 0.16.4

---
 cmd/restic/global.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/restic/global.go b/cmd/restic/global.go
index e979dcc2b..c179e8d33 100644
--- a/cmd/restic/global.go
+++ b/cmd/restic/global.go
@@ -43,7 +43,7 @@ import (
 	"golang.org/x/term"
 )
 
-var version = "0.16.4"
+var version = "0.16.4-dev (compiled manually)"
 
 // TimeFormat is the format used for all timestamps printed by restic.
 const TimeFormat = "2006-01-02 15:04:05"