diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index 2a9115b19..ac4996b7c 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -107,7 +107,7 @@ func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer) return repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error { h := restic.Handle{Type: restic.PackFile, Name: id.String()} - blobs, _, err := pack.List(repo.Key(), restic.ReaderAt(ctx, repo.Backend(), h), size) + blobs, _, err := pack.List(repo.Key(), backend.ReaderAt(ctx, repo.Backend(), h), size) if err != nil { Warnf("error for pack %v: %v\n", id.Str(), err) return nil @@ -525,7 +525,7 @@ func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) erro Printf(" ========================================\n") Printf(" inspect the pack itself\n") - blobs, _, err := pack.List(repo.Key(), restic.ReaderAt(ctx, repo.Backend(), h), fi.Size) + blobs, _, err := pack.List(repo.Key(), backend.ReaderAt(ctx, repo.Backend(), h), fi.Size) if err != nil { return fmt.Errorf("pack %v: %v", id.Str(), err) } diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 7410b4097..91e6a7509 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -16,6 +16,7 @@ import ( "github.com/restic/restic/internal/backend/azure" "github.com/restic/restic/internal/backend/b2" "github.com/restic/restic/internal/backend/gs" + "github.com/restic/restic/internal/backend/limiter" "github.com/restic/restic/internal/backend/local" "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/backend/rclone" @@ -26,7 +27,6 @@ import ( "github.com/restic/restic/internal/cache" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/fs" - "github.com/restic/restic/internal/limiter" "github.com/restic/restic/internal/options" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" diff --git a/internal/archiver/testing.go b/internal/archiver/testing.go index d8ad0a9e7..0c2bd3ba6 100644 --- a/internal/archiver/testing.go +++ b/internal/archiver/testing.go @@ -11,6 +11,7 @@ import ( "testing" "time" + "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/restic" @@ -225,7 +226,7 @@ func TestEnsureFileContent(ctx context.Context, t testing.TB, repo restic.Reposi return } - content := make([]byte, restic.CiphertextLength(len(file.Content))) + content := make([]byte, crypto.CiphertextLength(len(file.Content))) pos := 0 for _, id := range node.Content { part, err := repo.LoadBlob(ctx, restic.DataBlob, id, content[pos:]) diff --git a/internal/backend/backend_retry_test.go b/internal/backend/backend_retry_test.go index 4013f4ea5..e8f4d7315 100644 --- a/internal/backend/backend_retry_test.go +++ b/internal/backend/backend_retry_test.go @@ -7,8 +7,8 @@ import ( "io/ioutil" "testing" + "github.com/restic/restic/internal/backend/mock" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/mock" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" ) diff --git a/internal/limiter/limiter.go b/internal/backend/limiter/limiter.go similarity index 100% rename from internal/limiter/limiter.go rename to internal/backend/limiter/limiter.go diff --git a/internal/limiter/limiter_backend.go b/internal/backend/limiter/limiter_backend.go similarity index 100% rename from internal/limiter/limiter_backend.go rename to internal/backend/limiter/limiter_backend.go diff --git a/internal/limiter/limiter_backend_test.go b/internal/backend/limiter/limiter_backend_test.go similarity index 98% rename from internal/limiter/limiter_backend_test.go rename to internal/backend/limiter/limiter_backend_test.go index 10e216be9..1014dbed1 100644 --- a/internal/limiter/limiter_backend_test.go +++ b/internal/backend/limiter/limiter_backend_test.go @@ -8,7 +8,7 @@ import ( "io" "testing" - "github.com/restic/restic/internal/mock" + "github.com/restic/restic/internal/backend/mock" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) diff --git a/internal/limiter/static_limiter.go b/internal/backend/limiter/static_limiter.go similarity index 100% rename from internal/limiter/static_limiter.go rename to internal/backend/limiter/static_limiter.go diff --git a/internal/limiter/static_limiter_test.go b/internal/backend/limiter/static_limiter_test.go similarity index 100% rename from internal/limiter/static_limiter_test.go rename to internal/backend/limiter/static_limiter_test.go diff --git a/internal/mock/backend.go b/internal/backend/mock/backend.go similarity index 100% rename from internal/mock/backend.go rename to internal/backend/mock/backend.go diff --git a/internal/backend/rclone/backend.go b/internal/backend/rclone/backend.go index 8c1305f7f..0f367fe01 100644 --- a/internal/backend/rclone/backend.go +++ b/internal/backend/rclone/backend.go @@ -16,10 +16,10 @@ import ( "time" "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/backend/limiter" "github.com/restic/restic/internal/backend/rest" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/limiter" "golang.org/x/net/context/ctxhttp" "golang.org/x/net/http2" ) diff --git a/internal/restic/readerat.go b/internal/backend/readerat.go similarity index 75% rename from internal/restic/readerat.go rename to internal/backend/readerat.go index 1a781c03f..ff2e40393 100644 --- a/internal/restic/readerat.go +++ b/internal/backend/readerat.go @@ -1,4 +1,4 @@ -package restic +package backend import ( "context" @@ -6,12 +6,13 @@ import ( "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/restic" ) type backendReaderAt struct { ctx context.Context - be Backend - h Handle + be restic.Backend + h restic.Handle } func (brd backendReaderAt) ReadAt(p []byte, offset int64) (n int, err error) { @@ -21,12 +22,12 @@ func (brd backendReaderAt) ReadAt(p []byte, offset int64) (n int, err error) { // ReaderAt returns an io.ReaderAt for a file in the backend. The returned reader // should not escape the caller function to avoid unexpected interactions with the // embedded context -func ReaderAt(ctx context.Context, be Backend, h Handle) io.ReaderAt { +func ReaderAt(ctx context.Context, be restic.Backend, h restic.Handle) io.ReaderAt { return backendReaderAt{ctx: ctx, be: be, h: h} } // ReadAt reads from the backend handle h at the given position. -func ReadAt(ctx context.Context, be Backend, h Handle, offset int64, p []byte) (n int, err error) { +func ReadAt(ctx context.Context, be restic.Backend, h restic.Handle, offset int64, p []byte) (n int, err error) { debug.Log("ReadAt(%v) at %v, len %v", h, offset, len(p)) err = be.Load(ctx, h, len(p), offset, func(rd io.Reader) (ierr error) { diff --git a/internal/backend/utils_test.go b/internal/backend/utils_test.go index 0841a57e4..2e77fa9bd 100644 --- a/internal/backend/utils_test.go +++ b/internal/backend/utils_test.go @@ -10,8 +10,8 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/mem" + "github.com/restic/restic/internal/backend/mock" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/mock" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) diff --git a/internal/cache/file.go b/internal/cache/file.go index 2ee66f520..8ed4be77e 100644 --- a/internal/cache/file.go +++ b/internal/cache/file.go @@ -8,6 +8,7 @@ import ( "runtime" "github.com/pkg/errors" + "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/restic" @@ -58,7 +59,7 @@ func (c *Cache) load(h restic.Handle, length int, offset int64) (io.ReadCloser, return nil, errors.WithStack(err) } - if fi.Size() <= int64(restic.CiphertextLength(0)) { + if fi.Size() <= int64(crypto.CiphertextLength(0)) { _ = f.Close() _ = c.remove(h) return nil, errors.Errorf("cached file %v is truncated, removing", h) @@ -116,7 +117,7 @@ func (c *Cache) Save(h restic.Handle, rd io.Reader) error { return errors.Wrap(err, "Copy") } - if n <= int64(restic.CiphertextLength(0)) { + if n <= int64(crypto.CiphertextLength(0)) { _ = f.Close() _ = fs.Remove(f.Name()) debug.Log("trying to cache truncated file %v, removing", h) diff --git a/internal/restic/buffer.go b/internal/crypto/buffer.go similarity index 69% rename from internal/restic/buffer.go rename to internal/crypto/buffer.go index 899f4ea6f..b098d5c72 100644 --- a/internal/restic/buffer.go +++ b/internal/crypto/buffer.go @@ -1,21 +1,19 @@ -package restic - -import "github.com/restic/restic/internal/crypto" +package crypto // NewBlobBuffer returns a buffer that is large enough to hold a blob of size // plaintext bytes, including the crypto overhead. func NewBlobBuffer(size int) []byte { - return make([]byte, size, size+crypto.Extension) + return make([]byte, size, size+Extension) } // PlaintextLength returns the plaintext length of a blob with ciphertextSize // bytes. func PlaintextLength(ciphertextSize int) int { - return ciphertextSize - crypto.Extension + return ciphertextSize - Extension } // CiphertextLength returns the encrypted length of a blob with plaintextSize // bytes. func CiphertextLength(plaintextSize int) int { - return plaintextSize + crypto.Extension + return plaintextSize + Extension } diff --git a/internal/pack/pack.go b/internal/pack/pack.go index 1d991ccb5..7ac06db72 100644 --- a/internal/pack/pack.go +++ b/internal/pack/pack.go @@ -79,7 +79,7 @@ func (p *Packer) Finalize() error { return err } - encryptedHeader := make([]byte, 0, restic.CiphertextLength(len(header))) + encryptedHeader := make([]byte, 0, crypto.CiphertextLength(len(header))) nonce := crypto.NewRandomNonce() encryptedHeader = append(encryptedHeader, nonce...) encryptedHeader = p.k.Seal(encryptedHeader, nonce, header, nil) @@ -107,7 +107,7 @@ func (p *Packer) Finalize() error { // HeaderOverhead returns an estimate of the number of bytes written by a call to Finalize. func (p *Packer) HeaderOverhead() int { - return restic.CiphertextLength(0) + binary.Size(uint32(0)) + return crypto.CiphertextLength(0) + binary.Size(uint32(0)) } // makeHeader constructs the header for p. @@ -275,7 +275,7 @@ func List(k *crypto.Key, rd io.ReaderAt, size int64) (entries []restic.Blob, hdr return nil, 0, err } - if len(buf) < restic.CiphertextLength(0) { + if len(buf) < crypto.CiphertextLength(0) { return nil, 0, errors.New("invalid header, too small") } diff --git a/internal/pack/pack_test.go b/internal/pack/pack_test.go index 7c8250613..3f7077390 100644 --- a/internal/pack/pack_test.go +++ b/internal/pack/pack_test.go @@ -9,6 +9,7 @@ import ( "io" "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/mem" "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/pack" @@ -128,7 +129,7 @@ func TestUnpackReadSeeker(t *testing.T) { handle := restic.Handle{Type: restic.PackFile, Name: id.String()} rtest.OK(t, b.Save(context.TODO(), handle, restic.NewByteReader(packData, b.Hasher()))) - verifyBlobs(t, bufs, k, restic.ReaderAt(context.TODO(), b, handle), packSize) + verifyBlobs(t, bufs, k, backend.ReaderAt(context.TODO(), b, handle), packSize) } func TestShortPack(t *testing.T) { @@ -141,5 +142,5 @@ func TestShortPack(t *testing.T) { handle := restic.Handle{Type: restic.PackFile, Name: id.String()} rtest.OK(t, b.Save(context.TODO(), handle, restic.NewByteReader(packData, b.Hasher()))) - verifyBlobs(t, bufs, k, restic.ReaderAt(context.TODO(), b, handle), packSize) + verifyBlobs(t, bufs, k, backend.ReaderAt(context.TODO(), b, handle), packSize) } diff --git a/internal/repository/index.go b/internal/repository/index.go index a35f8952e..8978b60d5 100644 --- a/internal/repository/index.go +++ b/internal/repository/index.go @@ -7,6 +7,7 @@ import ( "sync" "time" + "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" @@ -194,7 +195,7 @@ func (idx *Index) LookupSize(bh restic.BlobHandle) (plaintextLength uint, found if e.uncompressedLength != 0 { return uint(e.uncompressedLength), true } - return uint(restic.PlaintextLength(int(e.length))), true + return uint(crypto.PlaintextLength(int(e.length))), true } // Supersedes returns the list of indexes this index supersedes, if any. diff --git a/internal/repository/key.go b/internal/repository/key.go index 8ff4c192c..4ce59a1f5 100644 --- a/internal/repository/key.go +++ b/internal/repository/key.go @@ -263,7 +263,7 @@ func AddKey(ctx context.Context, s *Repository, password, username, hostname str } nonce := crypto.NewRandomNonce() - ciphertext := make([]byte, 0, restic.CiphertextLength(len(buf))) + ciphertext := make([]byte, 0, crypto.CiphertextLength(len(buf))) ciphertext = append(ciphertext, nonce...) ciphertext = newkey.user.Seal(ciphertext, nonce, buf, nil) newkey.Data = ciphertext diff --git a/internal/repository/master_index_test.go b/internal/repository/master_index_test.go index d7e4f1678..2430c83dc 100644 --- a/internal/repository/master_index_test.go +++ b/internal/repository/master_index_test.go @@ -8,6 +8,7 @@ import ( "time" "github.com/restic/restic/internal/checker" + "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" @@ -22,7 +23,7 @@ func TestMasterIndex(t *testing.T) { PackID: restic.NewRandomID(), Blob: restic.Blob{ BlobHandle: bhInIdx1, - Length: uint(restic.CiphertextLength(10)), + Length: uint(crypto.CiphertextLength(10)), Offset: 0, }, } @@ -31,7 +32,7 @@ func TestMasterIndex(t *testing.T) { PackID: restic.NewRandomID(), Blob: restic.Blob{ BlobHandle: bhInIdx2, - Length: uint(restic.CiphertextLength(100)), + Length: uint(crypto.CiphertextLength(100)), Offset: 10, UncompressedLength: 200, }, @@ -41,7 +42,7 @@ func TestMasterIndex(t *testing.T) { PackID: restic.NewRandomID(), Blob: restic.Blob{ BlobHandle: bhInIdx12, - Length: uint(restic.CiphertextLength(123)), + Length: uint(crypto.CiphertextLength(123)), Offset: 110, }, } @@ -50,7 +51,7 @@ func TestMasterIndex(t *testing.T) { PackID: restic.NewRandomID(), Blob: restic.Blob{ BlobHandle: bhInIdx12, - Length: uint(restic.CiphertextLength(123)), + Length: uint(crypto.CiphertextLength(123)), Offset: 50, UncompressedLength: 80, }, diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 84193a4f3..ac9537f67 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -14,6 +14,7 @@ import ( "github.com/cenkalti/backoff/v4" "github.com/klauspost/compress/zstd" "github.com/restic/chunker" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/dryrun" "github.com/restic/restic/internal/cache" "github.com/restic/restic/internal/crypto" @@ -263,7 +264,7 @@ func (r *Repository) LoadBlob(ctx context.Context, t restic.BlobType, id restic. buf = buf[:blob.Length] } - n, err := restic.ReadAt(ctx, r.be, h, int64(blob.Offset), buf) + n, err := backend.ReadAt(ctx, r.be, h, int64(blob.Offset), buf) if err != nil { debug.Log("error loading blob %v: %v", blob, err) lastError = err @@ -398,7 +399,7 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data nonce := crypto.NewRandomNonce() - ciphertext := make([]byte, 0, restic.CiphertextLength(len(data))) + ciphertext := make([]byte, 0, crypto.CiphertextLength(len(data))) ciphertext = append(ciphertext, nonce...) // encrypt blob @@ -475,7 +476,7 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, p []by } } - ciphertext := restic.NewBlobBuffer(len(p)) + ciphertext := crypto.NewBlobBuffer(len(p)) ciphertext = ciphertext[:0] nonce := crypto.NewRandomNonce() ciphertext = append(ciphertext, nonce...) @@ -793,7 +794,7 @@ func (r *Repository) List(ctx context.Context, t restic.FileType, fn func(restic func (r *Repository) ListPack(ctx context.Context, id restic.ID, size int64) ([]restic.Blob, uint32, error) { h := restic.Handle{Type: restic.PackFile, Name: id.String()} - return pack.List(r.Key(), restic.ReaderAt(ctx, r.Backend(), h), size) + return pack.List(r.Key(), backend.ReaderAt(ctx, r.Backend(), h), size) } // Delete calls backend.Delete() if implemented, and returns an error diff --git a/internal/repository/repository_test.go b/internal/repository/repository_test.go index 38d3117a5..26b9e6244 100644 --- a/internal/repository/repository_test.go +++ b/internal/repository/repository_test.go @@ -190,7 +190,7 @@ func testLoadBlob(t *testing.T, version uint) { defer cleanup() length := 1000000 - buf := restic.NewBlobBuffer(length) + buf := crypto.NewBlobBuffer(length) _, err := io.ReadFull(rnd, buf) rtest.OK(t, err) @@ -201,7 +201,7 @@ func testLoadBlob(t *testing.T, version uint) { rtest.OK(t, err) rtest.OK(t, repo.Flush(context.Background())) - base := restic.CiphertextLength(length) + base := crypto.CiphertextLength(length) for _, testlength := range []int{0, base - 20, base - 1, base, base + 7, base + 15, base + 1000} { buf = make([]byte, 0, testlength) buf, err := repo.LoadBlob(context.TODO(), restic.DataBlob, id, buf) @@ -226,7 +226,7 @@ func benchmarkLoadBlob(b *testing.B, version uint) { defer cleanup() length := 1000000 - buf := restic.NewBlobBuffer(length) + buf := crypto.NewBlobBuffer(length) _, err := io.ReadFull(rnd, buf) rtest.OK(b, err) @@ -269,7 +269,7 @@ func benchmarkLoadUnpacked(b *testing.B, version uint) { defer cleanup() length := 1000000 - buf := restic.NewBlobBuffer(length) + buf := crypto.NewBlobBuffer(length) _, err := io.ReadFull(rnd, buf) rtest.OK(b, err) diff --git a/internal/restic/blob.go b/internal/restic/blob.go index a4fcdb1ac..4ac149adb 100644 --- a/internal/restic/blob.go +++ b/internal/restic/blob.go @@ -3,6 +3,7 @@ package restic import ( "fmt" + "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/errors" ) @@ -23,7 +24,7 @@ func (b Blob) DataLength() uint { if b.UncompressedLength != 0 { return b.UncompressedLength } - return uint(PlaintextLength(int(b.Length))) + return uint(crypto.PlaintextLength(int(b.Length))) } func (b Blob) IsCompressed() bool { diff --git a/internal/restorer/filerestorer_test.go b/internal/restorer/filerestorer_test.go index b5b52778c..fa781f8c8 100644 --- a/internal/restorer/filerestorer_test.go +++ b/internal/restorer/filerestorer_test.go @@ -61,7 +61,7 @@ func newTestRepo(content []TestFile) *TestRepo { key := crypto.NewRandomKey() seal := func(data []byte) []byte { - ciphertext := restic.NewBlobBuffer(len(data)) + ciphertext := crypto.NewBlobBuffer(len(data)) ciphertext = ciphertext[:0] // truncate the slice nonce := crypto.NewRandomNonce() ciphertext = append(ciphertext, nonce...) diff --git a/internal/restic/hardlinks_index.go b/internal/restorer/hardlinks_index.go similarity index 98% rename from internal/restic/hardlinks_index.go rename to internal/restorer/hardlinks_index.go index 0874f32a4..9cf45975a 100644 --- a/internal/restic/hardlinks_index.go +++ b/internal/restorer/hardlinks_index.go @@ -1,4 +1,4 @@ -package restic +package restorer import ( "sync" diff --git a/internal/restic/hardlinks_index_test.go b/internal/restorer/hardlinks_index_test.go similarity index 85% rename from internal/restic/hardlinks_index_test.go rename to internal/restorer/hardlinks_index_test.go index 8040d657f..75a2b83ee 100644 --- a/internal/restic/hardlinks_index_test.go +++ b/internal/restorer/hardlinks_index_test.go @@ -1,16 +1,16 @@ -package restic_test +package restorer_test import ( "testing" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/restorer" rtest "github.com/restic/restic/internal/test" ) // TestHardLinks contains various tests for HardlinkIndex. func TestHardLinks(t *testing.T) { - idx := restic.NewHardlinkIndex() + idx := restorer.NewHardlinkIndex() idx.Add(1, 2, "inode1-file1-on-device2") idx.Add(2, 3, "inode2-file2-on-device3") diff --git a/internal/restorer/restorer.go b/internal/restorer/restorer.go index 5a3f91368..a15a7bb9e 100644 --- a/internal/restorer/restorer.go +++ b/internal/restorer/restorer.go @@ -218,7 +218,7 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error { } } - idx := restic.NewHardlinkIndex() + idx := NewHardlinkIndex() filerestorer := newFileRestorer(dst, res.repo.Backend().Load, res.repo.Key(), res.repo.Index().Lookup, res.repo.Connections()) filerestorer.Error = res.Error