Merge pull request #3831 from MichaelEischer/move-code

Move code out of the restic package and consolidate backend specific code

cmd/restic/cmd_debug.go

@@ -107,7 +107,7 @@ func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer)
 	return repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error {
 		h := restic.Handle{Type: restic.PackFile, Name: id.String()}
 
-		blobs, _, err := pack.List(repo.Key(), restic.ReaderAt(ctx, repo.Backend(), h), size)
+		blobs, _, err := pack.List(repo.Key(), backend.ReaderAt(ctx, repo.Backend(), h), size)
 		if err != nil {
 			Warnf("error for pack %v: %v\n", id.Str(), err)
 			return nil
@@ -525,7 +525,7 @@ func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) erro
 	Printf("  ========================================\n")
 	Printf("  inspect the pack itself\n")
 
-	blobs, _, err := pack.List(repo.Key(), restic.ReaderAt(ctx, repo.Backend(), h), fi.Size)
+	blobs, _, err := pack.List(repo.Key(), backend.ReaderAt(ctx, repo.Backend(), h), fi.Size)
 	if err != nil {
 		return fmt.Errorf("pack %v: %v", id.Str(), err)
 	}

cmd/restic/global.go

@@ -16,6 +16,7 @@ import (
 	"github.com/restic/restic/internal/backend/azure"
 	"github.com/restic/restic/internal/backend/b2"
 	"github.com/restic/restic/internal/backend/gs"
+	"github.com/restic/restic/internal/backend/limiter"
 	"github.com/restic/restic/internal/backend/local"
 	"github.com/restic/restic/internal/backend/location"
 	"github.com/restic/restic/internal/backend/rclone"
@@ -26,7 +27,6 @@ import (
 	"github.com/restic/restic/internal/cache"
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/fs"
-	"github.com/restic/restic/internal/limiter"
 	"github.com/restic/restic/internal/options"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"

internal/archiver/testing.go

@@ -11,6 +11,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/restic/restic/internal/crypto"
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/restic"
@@ -225,7 +226,7 @@ func TestEnsureFileContent(ctx context.Context, t testing.TB, repo restic.Reposi
 		return
 	}
 
-	content := make([]byte, restic.CiphertextLength(len(file.Content)))
+	content := make([]byte, crypto.CiphertextLength(len(file.Content)))
 	pos := 0
 	for _, id := range node.Content {
 		part, err := repo.LoadBlob(ctx, restic.DataBlob, id, content[pos:])

internal/backend/backend_retry_test.go

@@ -7,8 +7,8 @@ import (
 	"io/ioutil"
 	"testing"
 
+	"github.com/restic/restic/internal/backend/mock"
 	"github.com/restic/restic/internal/errors"
-	"github.com/restic/restic/internal/mock"
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/test"
 )

internal/backend/limiter/limiter_backend_test.go

@@ -8,7 +8,7 @@ import (
 	"io"
 	"testing"
 
-	"github.com/restic/restic/internal/mock"
+	"github.com/restic/restic/internal/backend/mock"
 	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
 )

internal/backend/rclone/backend.go

@@ -16,10 +16,10 @@ import (
 	"time"
 
 	"github.com/restic/restic/internal/backend"
+	"github.com/restic/restic/internal/backend/limiter"
 	"github.com/restic/restic/internal/backend/rest"
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
-	"github.com/restic/restic/internal/limiter"
 	"golang.org/x/net/context/ctxhttp"
 	"golang.org/x/net/http2"
 )

internal/backend/readerat.go

@@ -1,4 +1,4 @@
-package restic
+package backend
 
 import (
 	"context"
@@ -6,12 +6,13 @@ import (
 
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/restic"
 )
 
 type backendReaderAt struct {
 	ctx context.Context
-	be  Backend
-	h   Handle
+	be  restic.Backend
+	h   restic.Handle
 }
 
 func (brd backendReaderAt) ReadAt(p []byte, offset int64) (n int, err error) {
@@ -21,12 +22,12 @@ func (brd backendReaderAt) ReadAt(p []byte, offset int64) (n int, err error) {
 // ReaderAt returns an io.ReaderAt for a file in the backend. The returned reader
 // should not escape the caller function to avoid unexpected interactions with the
 // embedded context
-func ReaderAt(ctx context.Context, be Backend, h Handle) io.ReaderAt {
+func ReaderAt(ctx context.Context, be restic.Backend, h restic.Handle) io.ReaderAt {
 	return backendReaderAt{ctx: ctx, be: be, h: h}
 }
 
 // ReadAt reads from the backend handle h at the given position.
-func ReadAt(ctx context.Context, be Backend, h Handle, offset int64, p []byte) (n int, err error) {
+func ReadAt(ctx context.Context, be restic.Backend, h restic.Handle, offset int64, p []byte) (n int, err error) {
 	debug.Log("ReadAt(%v) at %v, len %v", h, offset, len(p))
 
 	err = be.Load(ctx, h, len(p), offset, func(rd io.Reader) (ierr error) {

internal/backend/utils_test.go

@@ -10,8 +10,8 @@ import (
 
 	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/backend/mem"
+	"github.com/restic/restic/internal/backend/mock"
 	"github.com/restic/restic/internal/errors"
-	"github.com/restic/restic/internal/mock"
 	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
 )

internal/cache/file.go

@@ -8,6 +8,7 @@ import (
 	"runtime"
 
 	"github.com/pkg/errors"
+	"github.com/restic/restic/internal/crypto"
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/restic"
@@ -58,7 +59,7 @@ func (c *Cache) load(h restic.Handle, length int, offset int64) (io.ReadCloser,
 		return nil, errors.WithStack(err)
 	}
 
-	if fi.Size() <= int64(restic.CiphertextLength(0)) {
+	if fi.Size() <= int64(crypto.CiphertextLength(0)) {
 		_ = f.Close()
 		_ = c.remove(h)
 		return nil, errors.Errorf("cached file %v is truncated, removing", h)
@@ -116,7 +117,7 @@ func (c *Cache) Save(h restic.Handle, rd io.Reader) error {
 		return errors.Wrap(err, "Copy")
 	}
 
-	if n <= int64(restic.CiphertextLength(0)) {
+	if n <= int64(crypto.CiphertextLength(0)) {
 		_ = f.Close()
 		_ = fs.Remove(f.Name())
 		debug.Log("trying to cache truncated file %v, removing", h)

internal/crypto/buffer.go

@@ -1,21 +1,19 @@
-package restic
-
-import "github.com/restic/restic/internal/crypto"
+package crypto
 
 // NewBlobBuffer returns a buffer that is large enough to hold a blob of size
 // plaintext bytes, including the crypto overhead.
 func NewBlobBuffer(size int) []byte {
-	return make([]byte, size, size+crypto.Extension)
+	return make([]byte, size, size+Extension)
 }
 
 // PlaintextLength returns the plaintext length of a blob with ciphertextSize
 // bytes.
 func PlaintextLength(ciphertextSize int) int {
-	return ciphertextSize - crypto.Extension
+	return ciphertextSize - Extension
 }
 
 // CiphertextLength returns the encrypted length of a blob with plaintextSize
 // bytes.
 func CiphertextLength(plaintextSize int) int {
-	return plaintextSize + crypto.Extension
+	return plaintextSize + Extension
 }

internal/pack/pack.go

@@ -79,7 +79,7 @@ func (p *Packer) Finalize() error {
 		return err
 	}
 
-	encryptedHeader := make([]byte, 0, restic.CiphertextLength(len(header)))
+	encryptedHeader := make([]byte, 0, crypto.CiphertextLength(len(header)))
 	nonce := crypto.NewRandomNonce()
 	encryptedHeader = append(encryptedHeader, nonce...)
 	encryptedHeader = p.k.Seal(encryptedHeader, nonce, header, nil)
@@ -107,7 +107,7 @@ func (p *Packer) Finalize() error {
 
 // HeaderOverhead returns an estimate of the number of bytes written by a call to Finalize.
 func (p *Packer) HeaderOverhead() int {
-	return restic.CiphertextLength(0) + binary.Size(uint32(0))
+	return crypto.CiphertextLength(0) + binary.Size(uint32(0))
 }
 
 // makeHeader constructs the header for p.
@@ -275,7 +275,7 @@ func List(k *crypto.Key, rd io.ReaderAt, size int64) (entries []restic.Blob, hdr
 		return nil, 0, err
 	}
 
-	if len(buf) < restic.CiphertextLength(0) {
+	if len(buf) < crypto.CiphertextLength(0) {
 		return nil, 0, errors.New("invalid header, too small")
 	}
 

internal/pack/pack_test.go

@@ -9,6 +9,7 @@ import (
 	"io"
 	"testing"
 
+	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/backend/mem"
 	"github.com/restic/restic/internal/crypto"
 	"github.com/restic/restic/internal/pack"
@@ -128,7 +129,7 @@ func TestUnpackReadSeeker(t *testing.T) {
 
 	handle := restic.Handle{Type: restic.PackFile, Name: id.String()}
 	rtest.OK(t, b.Save(context.TODO(), handle, restic.NewByteReader(packData, b.Hasher())))
-	verifyBlobs(t, bufs, k, restic.ReaderAt(context.TODO(), b, handle), packSize)
+	verifyBlobs(t, bufs, k, backend.ReaderAt(context.TODO(), b, handle), packSize)
 }
 
 func TestShortPack(t *testing.T) {
@@ -141,5 +142,5 @@ func TestShortPack(t *testing.T) {
 
 	handle := restic.Handle{Type: restic.PackFile, Name: id.String()}
 	rtest.OK(t, b.Save(context.TODO(), handle, restic.NewByteReader(packData, b.Hasher())))
-	verifyBlobs(t, bufs, k, restic.ReaderAt(context.TODO(), b, handle), packSize)
+	verifyBlobs(t, bufs, k, backend.ReaderAt(context.TODO(), b, handle), packSize)
 }

internal/repository/index.go

@@ -7,6 +7,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/restic/restic/internal/crypto"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/restic"
 
@@ -194,7 +195,7 @@ func (idx *Index) LookupSize(bh restic.BlobHandle) (plaintextLength uint, found
 	if e.uncompressedLength != 0 {
 		return uint(e.uncompressedLength), true
 	}
-	return uint(restic.PlaintextLength(int(e.length))), true
+	return uint(crypto.PlaintextLength(int(e.length))), true
 }
 
 // Supersedes returns the list of indexes this index supersedes, if any.

internal/repository/key.go

@@ -263,7 +263,7 @@ func AddKey(ctx context.Context, s *Repository, password, username, hostname str
 	}
 
 	nonce := crypto.NewRandomNonce()
-	ciphertext := make([]byte, 0, restic.CiphertextLength(len(buf)))
+	ciphertext := make([]byte, 0, crypto.CiphertextLength(len(buf)))
 	ciphertext = append(ciphertext, nonce...)
 	ciphertext = newkey.user.Seal(ciphertext, nonce, buf, nil)
 	newkey.Data = ciphertext

internal/repository/master_index_test.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/restic/restic/internal/checker"
+	"github.com/restic/restic/internal/crypto"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
@@ -22,7 +23,7 @@ func TestMasterIndex(t *testing.T) {
 		PackID: restic.NewRandomID(),
 		Blob: restic.Blob{
 			BlobHandle: bhInIdx1,
-			Length:     uint(restic.CiphertextLength(10)),
+			Length:     uint(crypto.CiphertextLength(10)),
 			Offset:     0,
 		},
 	}
@@ -31,7 +32,7 @@ func TestMasterIndex(t *testing.T) {
 		PackID: restic.NewRandomID(),
 		Blob: restic.Blob{
 			BlobHandle:         bhInIdx2,
-			Length:             uint(restic.CiphertextLength(100)),
+			Length:             uint(crypto.CiphertextLength(100)),
 			Offset:             10,
 			UncompressedLength: 200,
 		},
@@ -41,7 +42,7 @@ func TestMasterIndex(t *testing.T) {
 		PackID: restic.NewRandomID(),
 		Blob: restic.Blob{
 			BlobHandle: bhInIdx12,
-			Length:     uint(restic.CiphertextLength(123)),
+			Length:     uint(crypto.CiphertextLength(123)),
 			Offset:     110,
 		},
 	}
@@ -50,7 +51,7 @@ func TestMasterIndex(t *testing.T) {
 		PackID: restic.NewRandomID(),
 		Blob: restic.Blob{
 			BlobHandle:         bhInIdx12,
-			Length:             uint(restic.CiphertextLength(123)),
+			Length:             uint(crypto.CiphertextLength(123)),
 			Offset:             50,
 			UncompressedLength: 80,
 		},

internal/repository/repository.go

@@ -14,6 +14,7 @@ import (
 	"github.com/cenkalti/backoff/v4"
 	"github.com/klauspost/compress/zstd"
 	"github.com/restic/chunker"
+	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/backend/dryrun"
 	"github.com/restic/restic/internal/cache"
 	"github.com/restic/restic/internal/crypto"
@@ -263,7 +264,7 @@ func (r *Repository) LoadBlob(ctx context.Context, t restic.BlobType, id restic.
 			buf = buf[:blob.Length]
 		}
 
-		n, err := restic.ReadAt(ctx, r.be, h, int64(blob.Offset), buf)
+		n, err := backend.ReadAt(ctx, r.be, h, int64(blob.Offset), buf)
 		if err != nil {
 			debug.Log("error loading blob %v: %v", blob, err)
 			lastError = err
@@ -398,7 +399,7 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data
 
 	nonce := crypto.NewRandomNonce()
 
-	ciphertext := make([]byte, 0, restic.CiphertextLength(len(data)))
+	ciphertext := make([]byte, 0, crypto.CiphertextLength(len(data)))
 	ciphertext = append(ciphertext, nonce...)
 
 	// encrypt blob
@@ -475,7 +476,7 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, p []by
 		}
 	}
 
-	ciphertext := restic.NewBlobBuffer(len(p))
+	ciphertext := crypto.NewBlobBuffer(len(p))
 	ciphertext = ciphertext[:0]
 	nonce := crypto.NewRandomNonce()
 	ciphertext = append(ciphertext, nonce...)
@@ -793,7 +794,7 @@ func (r *Repository) List(ctx context.Context, t restic.FileType, fn func(restic
 func (r *Repository) ListPack(ctx context.Context, id restic.ID, size int64) ([]restic.Blob, uint32, error) {
 	h := restic.Handle{Type: restic.PackFile, Name: id.String()}
 
-	return pack.List(r.Key(), restic.ReaderAt(ctx, r.Backend(), h), size)
+	return pack.List(r.Key(), backend.ReaderAt(ctx, r.Backend(), h), size)
 }
 
 // Delete calls backend.Delete() if implemented, and returns an error

internal/repository/repository_test.go

@@ -190,7 +190,7 @@ func testLoadBlob(t *testing.T, version uint) {
 	defer cleanup()
 
 	length := 1000000
-	buf := restic.NewBlobBuffer(length)
+	buf := crypto.NewBlobBuffer(length)
 	_, err := io.ReadFull(rnd, buf)
 	rtest.OK(t, err)
 
@@ -201,7 +201,7 @@ func testLoadBlob(t *testing.T, version uint) {
 	rtest.OK(t, err)
 	rtest.OK(t, repo.Flush(context.Background()))
 
-	base := restic.CiphertextLength(length)
+	base := crypto.CiphertextLength(length)
 	for _, testlength := range []int{0, base - 20, base - 1, base, base + 7, base + 15, base + 1000} {
 		buf = make([]byte, 0, testlength)
 		buf, err := repo.LoadBlob(context.TODO(), restic.DataBlob, id, buf)
@@ -226,7 +226,7 @@ func benchmarkLoadBlob(b *testing.B, version uint) {
 	defer cleanup()
 
 	length := 1000000
-	buf := restic.NewBlobBuffer(length)
+	buf := crypto.NewBlobBuffer(length)
 	_, err := io.ReadFull(rnd, buf)
 	rtest.OK(b, err)
 
@@ -269,7 +269,7 @@ func benchmarkLoadUnpacked(b *testing.B, version uint) {
 	defer cleanup()
 
 	length := 1000000
-	buf := restic.NewBlobBuffer(length)
+	buf := crypto.NewBlobBuffer(length)
 	_, err := io.ReadFull(rnd, buf)
 	rtest.OK(b, err)
 

internal/restic/blob.go

@@ -3,6 +3,7 @@ package restic
 import (
 	"fmt"
 
+	"github.com/restic/restic/internal/crypto"
 	"github.com/restic/restic/internal/errors"
 )
 
@@ -23,7 +24,7 @@ func (b Blob) DataLength() uint {
 	if b.UncompressedLength != 0 {
 		return b.UncompressedLength
 	}
-	return uint(PlaintextLength(int(b.Length)))
+	return uint(crypto.PlaintextLength(int(b.Length)))
 }
 
 func (b Blob) IsCompressed() bool {

internal/restorer/filerestorer_test.go

@@ -61,7 +61,7 @@ func newTestRepo(content []TestFile) *TestRepo {
 
 	key := crypto.NewRandomKey()
 	seal := func(data []byte) []byte {
-		ciphertext := restic.NewBlobBuffer(len(data))
+		ciphertext := crypto.NewBlobBuffer(len(data))
 		ciphertext = ciphertext[:0] // truncate the slice
 		nonce := crypto.NewRandomNonce()
 		ciphertext = append(ciphertext, nonce...)

internal/restorer/hardlinks_index.go

@@ -1,4 +1,4 @@
-package restic
+package restorer
 
 import (
 	"sync"

internal/restorer/hardlinks_index_test.go

@@ -1,16 +1,16 @@
-package restic_test
+package restorer_test
 
 import (
 	"testing"
 
-	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/restorer"
 	rtest "github.com/restic/restic/internal/test"
 )
 
 // TestHardLinks contains various tests for HardlinkIndex.
 func TestHardLinks(t *testing.T) {
 
-	idx := restic.NewHardlinkIndex()
+	idx := restorer.NewHardlinkIndex()
 
 	idx.Add(1, 2, "inode1-file1-on-device2")
 	idx.Add(2, 3, "inode2-file2-on-device3")

internal/restorer/restorer.go

@@ -218,7 +218,7 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error {
 		}
 	}
 
-	idx := restic.NewHardlinkIndex()
+	idx := NewHardlinkIndex()
 	filerestorer := newFileRestorer(dst, res.repo.Backend().Load, res.repo.Key(), res.repo.Index().Lookup, res.repo.Connections())
 	filerestorer.Error = res.Error