Change backend Save() function signature

src/restic/archiver/archiver_duplication_test.go

@@ -47,7 +47,7 @@ func forgetfulBackend() restic.Backend {
 		return 0, errors.New("not found")
 	}
 
-	be.SaveFn = func(h restic.Handle, p []byte) error {
+	be.SaveFn = func(h restic.Handle, rd io.Reader) error {
 		return nil
 	}
 

src/restic/backend.go

@@ -1,5 +1,7 @@
 package restic
 
+import "io"
+
 // Backend is used to store and access data.
 type Backend interface {
 	// Location returns a string that describes the type and location of the
@@ -22,7 +24,7 @@ type Backend interface {
 	Load(h Handle, p []byte, off int64) (int, error)
 
 	// Save stores the data in the backend under the given handle.
-	Save(h Handle, p []byte) error
+	Save(h Handle, rd io.Reader) error
 
 	// Stat returns information about the File identified by h.
 	Stat(h Handle) (FileInfo, error)

src/restic/backend/local/local.go

@@ -137,22 +137,18 @@ func (b *Local) Load(h restic.Handle, p []byte, off int64) (n int, err error) {
 	return io.ReadFull(f, p)
 }
 
-// writeToTempfile saves p into a tempfile in tempdir.
+// copyToTempfile saves p into a tempfile in tempdir.
-func writeToTempfile(tempdir string, p []byte) (filename string, err error) {
+func copyToTempfile(tempdir string, rd io.Reader) (filename string, err error) {
 	tmpfile, err := ioutil.TempFile(tempdir, "temp-")
 	if err != nil {
 		return "", errors.Wrap(err, "TempFile")
 	}
 
-	n, err := tmpfile.Write(p)
+	_, err = io.Copy(tmpfile, rd)
 	if err != nil {
 		return "", errors.Wrap(err, "Write")
 	}
 
-	if n != len(p) {
-		return "", errors.New("not all bytes writen")
-	}
-
 	if err = tmpfile.Sync(); err != nil {
 		return "", errors.Wrap(err, "Syncn")
 	}
@@ -166,14 +162,14 @@ func writeToTempfile(tempdir string, p []byte) (filename string, err error) {
 }
 
 // Save stores data in the backend at the handle.
-func (b *Local) Save(h restic.Handle, p []byte) (err error) {
+func (b *Local) Save(h restic.Handle, rd io.Reader) (err error) {
-	debug.Log("Save %v, length %v", h, len(p))
+	debug.Log("Save %v", h)
 	if err := h.Valid(); err != nil {
 		return err
 	}
 
-	tmpfile, err := writeToTempfile(filepath.Join(b.p, backend.Paths.Temp), p)
+	tmpfile, err := copyToTempfile(filepath.Join(b.p, backend.Paths.Temp), rd)
-	debug.Log("saved %v (%d bytes) to %v", h, len(p), tmpfile)
+	debug.Log("saved %v to %v", h, tmpfile)
 	if err != nil {
 		return err
 	}

src/restic/backend/mem/mem_backend.go

@@ -2,6 +2,7 @@ package mem
 
 import (
 	"io"
+	"io/ioutil"
 	"restic"
 	"sync"
 
@@ -93,7 +94,7 @@ func (be *MemoryBackend) Load(h restic.Handle, p []byte, off int64) (int, error)
 }
 
 // Save adds new Data to the backend.
-func (be *MemoryBackend) Save(h restic.Handle, p []byte) error {
+func (be *MemoryBackend) Save(h restic.Handle, rd io.Reader) error {
 	if err := h.Valid(); err != nil {
 		return err
 	}
@@ -109,10 +110,13 @@ func (be *MemoryBackend) Save(h restic.Handle, p []byte) error {
 		return errors.New("file already exists")
 	}
 
-	debug.Log("save %v bytes at %v", len(p), h)
+	buf, err := ioutil.ReadAll(rd)
-	buf := make([]byte, len(p))
+	if err != nil {
-	copy(buf, p)
+		return err
+	}
+
 	be.data[entry{h.Type, h.Name}] = buf
+	debug.Log("saved %v bytes at %v", len(buf), h)
 
 	return nil
 }

src/restic/backend/rest/rest.go

@@ -1,7 +1,6 @@
 package rest
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -19,6 +18,9 @@ import (
 
 const connLimit = 10
 
+// make sure the rest backend implements restic.Backend
+var _ restic.Backend = &restBackend{}
+
 // restPath returns the path to the given resource.
 func restPath(url *url.URL, h restic.Handle) string {
 	u := *url
@@ -123,13 +125,13 @@ func (b *restBackend) Load(h restic.Handle, p []byte, off int64) (n int, err err
 }
 
 // Save stores data in the backend at the handle.
-func (b *restBackend) Save(h restic.Handle, p []byte) (err error) {
+func (b *restBackend) Save(h restic.Handle, rd io.Reader) (err error) {
 	if err := h.Valid(); err != nil {
 		return err
 	}
 
 	<-b.connChan
-	resp, err := b.client.Post(restPath(b.url, h), "binary/octet-stream", bytes.NewReader(p))
+	resp, err := b.client.Post(restPath(b.url, h), "binary/octet-stream", rd)
 	b.connChan <- struct{}{}
 
 	if resp != nil {

src/restic/backend/s3/s3.go

@@ -1,7 +1,6 @@
 package s3
 
 import (
-	"bytes"
 	"io"
 	"path"
 	"restic"
@@ -147,12 +146,12 @@ func (be s3) Load(h restic.Handle, p []byte, off int64) (n int, err error) {
 }
 
 // Save stores data in the backend at the handle.
-func (be s3) Save(h restic.Handle, p []byte) (err error) {
+func (be s3) Save(h restic.Handle, rd io.Reader) (err error) {
 	if err := h.Valid(); err != nil {
 		return err
 	}
 
-	debug.Log("%v with %d bytes", h, len(p))
+	debug.Log("Save %v", h)
 
 	objName := be.s3path(h.Type, h.Name)
 
@@ -168,9 +167,9 @@ func (be s3) Save(h restic.Handle, p []byte) (err error) {
 		be.connChan <- struct{}{}
 	}()
 
-	debug.Log("PutObject(%v, %v, %v, %v)",
+	debug.Log("PutObject(%v, %v)",
-		be.bucketname, objName, int64(len(p)), "binary/octet-stream")
+		be.bucketname, objName)
-	n, err := be.client.PutObject(be.bucketname, objName, bytes.NewReader(p), "binary/octet-stream")
+	n, err := be.client.PutObject(be.bucketname, objName, rd, "binary/octet-stream")
 	debug.Log("%v -> %v bytes, err %#v", objName, n, err)
 
 	return errors.Wrap(err, "client.PutObject")

src/restic/backend/sftp/sftp.go

@@ -365,8 +365,8 @@ func (r *SFTP) Load(h restic.Handle, p []byte, off int64) (n int, err error) {
 }
 
 // Save stores data in the backend at the handle.
-func (r *SFTP) Save(h restic.Handle, p []byte) (err error) {
+func (r *SFTP) Save(h restic.Handle, rd io.Reader) (err error) {
-	debug.Log("save %v bytes to %v", h, len(p))
+	debug.Log("save to %v", h)
 	if err := r.clientError(); err != nil {
 		return err
 	}
@@ -380,16 +380,12 @@ func (r *SFTP) Save(h restic.Handle, p []byte) (err error) {
 		return err
 	}
 
-	debug.Log("save %v (%d bytes) to %v", h, len(p), filename)
+	n, err := io.Copy(tmpfile, rd)
-
-	n, err := tmpfile.Write(p)
 	if err != nil {
 		return errors.Wrap(err, "Write")
 	}
 
-	if n != len(p) {
+	debug.Log("saved %v (%d bytes) to %v", h, n, filename)
-		return errors.New("not all bytes writen")
-	}
 
 	err = tmpfile.Close()
 	if err != nil {

src/restic/backend/test/tests.go

@@ -8,6 +8,7 @@ import (
 	"reflect"
 	"restic"
 	"sort"
+	"strings"
 	"testing"
 
 	"restic/errors"
@@ -157,7 +158,7 @@ func TestConfig(t testing.TB) {
 		t.Fatalf("did not get expected error for non-existing config")
 	}
 
-	err = b.Save(restic.Handle{Type: restic.ConfigFile}, []byte(testString))
+	err = b.Save(restic.Handle{Type: restic.ConfigFile}, strings.NewReader(testString))
 	if err != nil {
 		t.Fatalf("Save() error: %v", err)
 	}
@@ -198,7 +199,7 @@ func TestLoad(t testing.TB) {
 	id := restic.Hash(data)
 
 	handle := restic.Handle{Type: restic.DataFile, Name: id.String()}
-	err = b.Save(handle, data)
+	err = b.Save(handle, bytes.NewReader(data))
 	if err != nil {
 		t.Fatalf("Save() error: %v", err)
 	}
@@ -323,7 +324,7 @@ func TestLoadNegativeOffset(t testing.TB) {
 	id := restic.Hash(data)
 
 	handle := restic.Handle{Type: restic.DataFile, Name: id.String()}
-	err := b.Save(handle, data)
+	err := b.Save(handle, bytes.NewReader(data))
 	if err != nil {
 		t.Fatalf("Save() error: %v", err)
 	}
@@ -384,7 +385,7 @@ func TestSave(t testing.TB) {
 			Type: restic.DataFile,
 			Name: fmt.Sprintf("%s-%d", id, i),
 		}
-		err := b.Save(h, data)
+		err := b.Save(h, bytes.NewReader(data))
 		test.OK(t, err)
 
 		buf, err := backend.LoadAll(b, h, nil)
@@ -430,7 +431,7 @@ func TestSaveFilenames(t testing.TB) {
 
 	for i, test := range filenameTests {
 		h := restic.Handle{Name: test.name, Type: restic.DataFile}
-		err := b.Save(h, []byte(test.data))
+		err := b.Save(h, strings.NewReader(test.data))
 		if err != nil {
 			t.Errorf("test %d failed: Save() returned %v", i, err)
 			continue
@@ -466,7 +467,7 @@ var testStrings = []struct {
 
 func store(t testing.TB, b restic.Backend, tpe restic.FileType, data []byte) {
 	id := restic.Hash(data)
-	err := b.Save(restic.Handle{Name: id.String(), Type: tpe}, data)
+	err := b.Save(restic.Handle{Name: id.String(), Type: tpe}, bytes.NewReader(data))
 	test.OK(t, err)
 }
 
@@ -530,7 +531,7 @@ func TestBackend(t testing.TB) {
 		ts := testStrings[0]
 
 		// create blob
-		err := b.Save(restic.Handle{Type: tpe, Name: ts.id}, []byte(ts.data))
+		err := b.Save(restic.Handle{Type: tpe, Name: ts.id}, strings.NewReader(ts.data))
 		test.Assert(t, err != nil, "expected error, got %v", err)
 
 		// remove and recreate
@@ -543,7 +544,7 @@ func TestBackend(t testing.TB) {
 		test.Assert(t, ok == false, "removed blob still present")
 
 		// create blob
-		err = b.Save(restic.Handle{Type: tpe, Name: ts.id}, []byte(ts.data))
+		err = b.Save(restic.Handle{Type: tpe, Name: ts.id}, strings.NewReader(ts.data))
 		test.OK(t, err)
 
 		// list items

src/restic/backend/utils_test.go

@@ -21,7 +21,7 @@ func TestLoadAll(t *testing.T) {
 		data := Random(23+i, rand.Intn(MiB)+500*KiB)
 
 		id := restic.Hash(data)
-		err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, data)
+		err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, bytes.NewReader(data))
 		OK(t, err)
 
 		buf, err := backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()}, nil)
@@ -46,7 +46,7 @@ func TestLoadSmallBuffer(t *testing.T) {
 		data := Random(23+i, rand.Intn(MiB)+500*KiB)
 
 		id := restic.Hash(data)
-		err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, data)
+		err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, bytes.NewReader(data))
 		OK(t, err)
 
 		buf := make([]byte, len(data)-23)
@@ -72,7 +72,7 @@ func TestLoadLargeBuffer(t *testing.T) {
 		data := Random(23+i, rand.Intn(MiB)+500*KiB)
 
 		id := restic.Hash(data)
-		err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, data)
+		err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, bytes.NewReader(data))
 		OK(t, err)
 
 		buf := make([]byte, len(data)+100)

src/restic/mock/backend.go

@@ -1,6 +1,7 @@
 package mock
 
 import (
+	"io"
 	"restic"
 
 	"restic/errors"
@@ -10,7 +11,7 @@ import (
 type Backend struct {
 	CloseFn    func() error
 	LoadFn     func(h restic.Handle, p []byte, off int64) (int, error)
-	SaveFn     func(h restic.Handle, p []byte) error
+	SaveFn     func(h restic.Handle, rd io.Reader) error
 	StatFn     func(h restic.Handle) (restic.FileInfo, error)
 	ListFn     func(restic.FileType, <-chan struct{}) <-chan string
 	RemoveFn   func(restic.FileType, string) error
@@ -47,12 +48,12 @@ func (m *Backend) Load(h restic.Handle, p []byte, off int64) (int, error) {
 }
 
 // Save data in the backend.
-func (m *Backend) Save(h restic.Handle, p []byte) error {
+func (m *Backend) Save(h restic.Handle, rd io.Reader) error {
 	if m.SaveFn == nil {
 		return errors.New("not implemented")
 	}
 
-	return m.SaveFn(h, p)
+	return m.SaveFn(h, rd)
 }
 
 // Stat an object in the backend.

src/restic/pack/pack_test.go

@@ -126,7 +126,7 @@ func TestUnpackReadSeeker(t *testing.T) {
 	id := restic.Hash(packData)
 
 	handle := restic.Handle{Type: restic.DataFile, Name: id.String()}
-	OK(t, b.Save(handle, packData))
+	OK(t, b.Save(handle, bytes.NewReader(packData)))
 	verifyBlobs(t, bufs, k, restic.ReaderAt(b, handle), packSize)
 }
 
@@ -139,6 +139,6 @@ func TestShortPack(t *testing.T) {
 	id := restic.Hash(packData)
 
 	handle := restic.Handle{Type: restic.DataFile, Name: id.String()}
-	OK(t, b.Save(handle, packData))
+	OK(t, b.Save(handle, bytes.NewReader(packData)))
 	verifyBlobs(t, bufs, k, restic.ReaderAt(b, handle), packSize)
 }

src/restic/repository/key.go

@@ -1,6 +1,7 @@
 package repository
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"os"
@@ -232,7 +233,7 @@ func AddKey(s *Repository, password string, template *crypto.Key) (*Key, error)
 		Name: restic.Hash(buf).String(),
 	}
 
-	err = s.be.Save(h, buf)
+	err = s.be.Save(h, bytes.NewReader(buf))
 	if err != nil {
 		return nil, err
 	}

src/restic/repository/packer_manager.go

@@ -1,6 +1,7 @@
 package repository
 
 import (
+	"bytes"
 	"io"
 	"io/ioutil"
 	"os"
@@ -17,7 +18,7 @@ import (
 
 // Saver implements saving data in a backend.
 type Saver interface {
-	Save(h restic.Handle, jp []byte) error
+	Save(restic.Handle, io.Reader) error
 }
 
 // packerManager keeps a list of open packs and creates new on demand.
@@ -117,7 +118,7 @@ func (r *Repository) savePacker(p *pack.Packer) error {
 	id := restic.Hash(data)
 	h := restic.Handle{Type: restic.DataFile, Name: id.String()}
 
-	err = r.be.Save(h, data)
+	err = r.be.Save(h, bytes.NewReader(data))
 	if err != nil {
 		debug.Log("Save(%v) error: %v", h, err)
 		return err

src/restic/repository/packer_manager_test.go

@@ -1,12 +1,14 @@
 package repository
 
 import (
+	"bytes"
 	"io"
 	"math/rand"
 	"os"
 	"restic"
 	"restic/backend/mem"
 	"restic/crypto"
+	"restic/mock"
 	"testing"
 )
 
@@ -65,7 +67,7 @@ func saveFile(t testing.TB, be Saver, filename string, n int) {
 
 	h := restic.Handle{Type: restic.DataFile, Name: restic.Hash(data).String()}
 
-	err = be.Save(h, data)
+	err = be.Save(h, bytes.NewReader(data))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -134,12 +136,6 @@ func flushRemainingPacks(t testing.TB, rnd *randReader, be Saver, pm *packerMana
 	return bytes
 }
 
-type fakeBackend struct{}
-
-func (f *fakeBackend) Save(h restic.Handle, data []byte) error {
-	return nil
-}
-
 func TestPackerManager(t *testing.T) {
 	rnd := newRandReader(rand.NewSource(23))
 
@@ -157,7 +153,9 @@ func TestPackerManager(t *testing.T) {
 func BenchmarkPackerManager(t *testing.B) {
 	rnd := newRandReader(rand.NewSource(23))
 
-	be := &fakeBackend{}
+	be := &mock.Backend{
+		SaveFn: func(restic.Handle, io.Reader) error { return nil },
+	}
 	pm := newPackerManager(be, crypto.NewRandomKey())
 	blobBuf := make([]byte, maxBlobSize)
 

src/restic/repository/repository.go

@@ -220,7 +220,7 @@ func (r *Repository) SaveUnpacked(t restic.FileType, p []byte) (id restic.ID, er
 	id = restic.Hash(ciphertext)
 	h := restic.Handle{Type: t, Name: id.String()}
 
-	err = r.be.Save(h, ciphertext)
+	err = r.be.Save(h, bytes.NewReader(ciphertext))
 	if err != nil {
 		debug.Log("error saving blob %v: %v", h, err)
 		return restic.ID{}, err