octoleo
/
restic
mirror of https://github.com/octoleo/restic.git
2
2
Fork 0
Code Issues Releases Wiki Activity

Change backend Save() function signature

This commit is contained in:
Alexander Neumann 2017-01-22 12:32:20 +01:00
parent c93f79f0f3
commit 9b48da5b4e
15 changed files with 63 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/restic/archiver/archiver_duplication_test.go
View File

@ -47,7 +47,7 @@ func forgetfulBackend() restic.Backend {
return 0, errors.New("not found")
}
be.SaveFn = func(h restic.Handle, p []byte) error {
be.SaveFn = func(h restic.Handle, rd io.Reader) error {
return nil
}

4
src/restic/backend.go
View File

@ -1,5 +1,7 @@
package restic
import "io"
// Backend is used to store and access data.
type Backend interface {
// Location returns a string that describes the type and location of the
@ -22,7 +24,7 @@ type Backend interface {
Load(h Handle, p []byte, off int64) (int, error)
// Save stores the data in the backend under the given handle.
Save(h Handle, p []byte) error
Save(h Handle, rd io.Reader) error
// Stat returns information about the File identified by h.
Stat(h Handle) (FileInfo, error)

18
src/restic/backend/local/local.go
View File

@ -137,22 +137,18 @@ func (b *Local) Load(h restic.Handle, p []byte, off int64) (n int, err error) {
return io.ReadFull(f, p)
}
// writeToTempfile saves p into a tempfile in tempdir.
func writeToTempfile(tempdir string, p []byte) (filename string, err error) {
// copyToTempfile saves p into a tempfile in tempdir.
func copyToTempfile(tempdir string, rd io.Reader) (filename string, err error) {
tmpfile, err := ioutil.TempFile(tempdir, "temp-")
if err != nil {
return "", errors.Wrap(err, "TempFile")
}
n, err := tmpfile.Write(p)
_, err = io.Copy(tmpfile, rd)
if err != nil {
return "", errors.Wrap(err, "Write")
}
if n != len(p) {
return "", errors.New("not all bytes writen")
}
if err = tmpfile.Sync(); err != nil {
return "", errors.Wrap(err, "Syncn")
}
@ -166,14 +162,14 @@ func writeToTempfile(tempdir string, p []byte) (filename string, err error) {
}
// Save stores data in the backend at the handle.
func (b *Local) Save(h restic.Handle, p []byte) (err error) {
debug.Log("Save %v, length %v", h, len(p))
func (b *Local) Save(h restic.Handle, rd io.Reader) (err error) {
debug.Log("Save %v", h)
if err := h.Valid(); err != nil {
return err
}
tmpfile, err := writeToTempfile(filepath.Join(b.p, backend.Paths.Temp), p)
debug.Log("saved %v (%d bytes) to %v", h, len(p), tmpfile)
tmpfile, err := copyToTempfile(filepath.Join(b.p, backend.Paths.Temp), rd)
debug.Log("saved %v to %v", h, tmpfile)
if err != nil {
return err
}

12
src/restic/backend/mem/mem_backend.go
View File

@ -2,6 +2,7 @@ package mem
import (
"io"
"io/ioutil"
"restic"
"sync"
@ -93,7 +94,7 @@ func (be *MemoryBackend) Load(h restic.Handle, p []byte, off int64) (int, error)
}
// Save adds new Data to the backend.
func (be *MemoryBackend) Save(h restic.Handle, p []byte) error {
func (be *MemoryBackend) Save(h restic.Handle, rd io.Reader) error {
if err := h.Valid(); err != nil {
return err
}
@ -109,10 +110,13 @@ func (be *MemoryBackend) Save(h restic.Handle, p []byte) error {
return errors.New("file already exists")
}
debug.Log("save %v bytes at %v", len(p), h)
buf := make([]byte, len(p))
copy(buf, p)
buf, err := ioutil.ReadAll(rd)
if err != nil {
return err
}
be.data[entry{h.Type, h.Name}] = buf
debug.Log("saved %v bytes at %v", len(buf), h)
return nil
}

8
src/restic/backend/rest/rest.go
View File

@ -1,7 +1,6 @@
package rest
import (
"bytes"
"encoding/json"
"fmt"
"io"
@ -19,6 +18,9 @@ import (
const connLimit = 10
// make sure the rest backend implements restic.Backend
var _ restic.Backend = &restBackend{}
// restPath returns the path to the given resource.
func restPath(url *url.URL, h restic.Handle) string {
u := *url
@ -123,13 +125,13 @@ func (b *restBackend) Load(h restic.Handle, p []byte, off int64) (n int, err err
}
// Save stores data in the backend at the handle.
func (b *restBackend) Save(h restic.Handle, p []byte) (err error) {
func (b *restBackend) Save(h restic.Handle, rd io.Reader) (err error) {
if err := h.Valid(); err != nil {
return err
}
<-b.connChan
resp, err := b.client.Post(restPath(b.url, h), "binary/octet-stream", bytes.NewReader(p))
resp, err := b.client.Post(restPath(b.url, h), "binary/octet-stream", rd)
b.connChan <- struct{}{}
if resp != nil {

11
src/restic/backend/s3/s3.go
View File

@ -1,7 +1,6 @@
package s3
import (
"bytes"
"io"
"path"
"restic"
@ -147,12 +146,12 @@ func (be s3) Load(h restic.Handle, p []byte, off int64) (n int, err error) {
}
// Save stores data in the backend at the handle.
func (be s3) Save(h restic.Handle, p []byte) (err error) {
func (be s3) Save(h restic.Handle, rd io.Reader) (err error) {
if err := h.Valid(); err != nil {
return err
}
debug.Log("%v with %d bytes", h, len(p))
debug.Log("Save %v", h)
objName := be.s3path(h.Type, h.Name)
@ -168,9 +167,9 @@ func (be s3) Save(h restic.Handle, p []byte) (err error) {
be.connChan <- struct{}{}
}()
debug.Log("PutObject(%v, %v, %v, %v)",
be.bucketname, objName, int64(len(p)), "binary/octet-stream")
n, err := be.client.PutObject(be.bucketname, objName, bytes.NewReader(p), "binary/octet-stream")
debug.Log("PutObject(%v, %v)",
be.bucketname, objName)
n, err := be.client.PutObject(be.bucketname, objName, rd, "binary/octet-stream")
debug.Log("%v -> %v bytes, err %#v", objName, n, err)
return errors.Wrap(err, "client.PutObject")

12
src/restic/backend/sftp/sftp.go
View File

@ -365,8 +365,8 @@ func (r *SFTP) Load(h restic.Handle, p []byte, off int64) (n int, err error) {
}
// Save stores data in the backend at the handle.
func (r *SFTP) Save(h restic.Handle, p []byte) (err error) {
debug.Log("save %v bytes to %v", h, len(p))
func (r *SFTP) Save(h restic.Handle, rd io.Reader) (err error) {
debug.Log("save to %v", h)
if err := r.clientError(); err != nil {
return err
}
@ -380,16 +380,12 @@ func (r *SFTP) Save(h restic.Handle, p []byte) (err error) {
return err
}
debug.Log("save %v (%d bytes) to %v", h, len(p), filename)
n, err := tmpfile.Write(p)
n, err := io.Copy(tmpfile, rd)
if err != nil {
return errors.Wrap(err, "Write")
}
if n != len(p) {
return errors.New("not all bytes writen")
}
debug.Log("saved %v (%d bytes) to %v", h, n, filename)
err = tmpfile.Close()
if err != nil {

17
src/restic/backend/test/tests.go
View File

@ -8,6 +8,7 @@ import (
"reflect"
"restic"
"sort"
"strings"
"testing"
"restic/errors"
@ -157,7 +158,7 @@ func TestConfig(t testing.TB) {
t.Fatalf("did not get expected error for non-existing config")
}
err = b.Save(restic.Handle{Type: restic.ConfigFile}, []byte(testString))
err = b.Save(restic.Handle{Type: restic.ConfigFile}, strings.NewReader(testString))
if err != nil {
t.Fatalf("Save() error: %v", err)
}
@ -198,7 +199,7 @@ func TestLoad(t testing.TB) {
id := restic.Hash(data)
handle := restic.Handle{Type: restic.DataFile, Name: id.String()}
err = b.Save(handle, data)
err = b.Save(handle, bytes.NewReader(data))
if err != nil {
t.Fatalf("Save() error: %v", err)
}
@ -323,7 +324,7 @@ func TestLoadNegativeOffset(t testing.TB) {
id := restic.Hash(data)
handle := restic.Handle{Type: restic.DataFile, Name: id.String()}
err := b.Save(handle, data)
err := b.Save(handle, bytes.NewReader(data))
if err != nil {
t.Fatalf("Save() error: %v", err)
}
@ -384,7 +385,7 @@ func TestSave(t testing.TB) {
Type: restic.DataFile,
Name: fmt.Sprintf("%s-%d", id, i),
}
err := b.Save(h, data)
err := b.Save(h, bytes.NewReader(data))
test.OK(t, err)
buf, err := backend.LoadAll(b, h, nil)
@ -430,7 +431,7 @@ func TestSaveFilenames(t testing.TB) {
for i, test := range filenameTests {
h := restic.Handle{Name: test.name, Type: restic.DataFile}
err := b.Save(h, []byte(test.data))
err := b.Save(h, strings.NewReader(test.data))
if err != nil {
t.Errorf("test %d failed: Save() returned %v", i, err)
continue
@ -466,7 +467,7 @@ var testStrings = []struct {
func store(t testing.TB, b restic.Backend, tpe restic.FileType, data []byte) {
id := restic.Hash(data)
err := b.Save(restic.Handle{Name: id.String(), Type: tpe}, data)
err := b.Save(restic.Handle{Name: id.String(), Type: tpe}, bytes.NewReader(data))
test.OK(t, err)
}
@ -530,7 +531,7 @@ func TestBackend(t testing.TB) {
ts := testStrings[0]
// create blob
err := b.Save(restic.Handle{Type: tpe, Name: ts.id}, []byte(ts.data))
err := b.Save(restic.Handle{Type: tpe, Name: ts.id}, strings.NewReader(ts.data))
test.Assert(t, err != nil, "expected error, got %v", err)
// remove and recreate
@ -543,7 +544,7 @@ func TestBackend(t testing.TB) {
test.Assert(t, ok == false, "removed blob still present")
// create blob
err = b.Save(restic.Handle{Type: tpe, Name: ts.id}, []byte(ts.data))
err = b.Save(restic.Handle{Type: tpe, Name: ts.id}, strings.NewReader(ts.data))
test.OK(t, err)
// list items

6
src/restic/backend/utils_test.go
View File

@ -21,7 +21,7 @@ func TestLoadAll(t *testing.T) {
data := Random(23+i, rand.Intn(MiB)+500*KiB)
id := restic.Hash(data)
err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, data)
err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, bytes.NewReader(data))
OK(t, err)
buf, err := backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()}, nil)
@ -46,7 +46,7 @@ func TestLoadSmallBuffer(t *testing.T) {
data := Random(23+i, rand.Intn(MiB)+500*KiB)
id := restic.Hash(data)
err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, data)
err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, bytes.NewReader(data))
OK(t, err)
buf := make([]byte, len(data)-23)
@ -72,7 +72,7 @@ func TestLoadLargeBuffer(t *testing.T) {
data := Random(23+i, rand.Intn(MiB)+500*KiB)
id := restic.Hash(data)
err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, data)
err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, bytes.NewReader(data))
OK(t, err)
buf := make([]byte, len(data)+100)

7
src/restic/mock/backend.go
View File

@ -1,6 +1,7 @@
package mock
import (
"io"
"restic"
"restic/errors"
@ -10,7 +11,7 @@ import (
type Backend struct {
CloseFn func() error
LoadFn func(h restic.Handle, p []byte, off int64) (int, error)
SaveFn func(h restic.Handle, p []byte) error
SaveFn func(h restic.Handle, rd io.Reader) error
StatFn func(h restic.Handle) (restic.FileInfo, error)
ListFn func(restic.FileType, <-chan struct{}) <-chan string
RemoveFn func(restic.FileType, string) error
@ -47,12 +48,12 @@ func (m *Backend) Load(h restic.Handle, p []byte, off int64) (int, error) {
}
// Save data in the backend.
func (m *Backend) Save(h restic.Handle, p []byte) error {
func (m *Backend) Save(h restic.Handle, rd io.Reader) error {
if m.SaveFn == nil {
return errors.New("not implemented")
}
return m.SaveFn(h, p)
return m.SaveFn(h, rd)
}
// Stat an object in the backend.

4
src/restic/pack/pack_test.go
View File

@ -126,7 +126,7 @@ func TestUnpackReadSeeker(t *testing.T) {
id := restic.Hash(packData)
handle := restic.Handle{Type: restic.DataFile, Name: id.String()}
OK(t, b.Save(handle, packData))
OK(t, b.Save(handle, bytes.NewReader(packData)))
verifyBlobs(t, bufs, k, restic.ReaderAt(b, handle), packSize)
}
@ -139,6 +139,6 @@ func TestShortPack(t *testing.T) {
id := restic.Hash(packData)
handle := restic.Handle{Type: restic.DataFile, Name: id.String()}
OK(t, b.Save(handle, packData))
OK(t, b.Save(handle, bytes.NewReader(packData)))
verifyBlobs(t, bufs, k, restic.ReaderAt(b, handle), packSize)
}

3
src/restic/repository/key.go
View File

@ -1,6 +1,7 @@
package repository
import (
"bytes"
"encoding/json"
"fmt"
"os"
@ -232,7 +233,7 @@ func AddKey(s *Repository, password string, template *crypto.Key) (*Key, error)
Name: restic.Hash(buf).String(),
}
err = s.be.Save(h, buf)
err = s.be.Save(h, bytes.NewReader(buf))
if err != nil {
return nil, err
}

5
src/restic/repository/packer_manager.go
View File

@ -1,6 +1,7 @@
package repository
import (
"bytes"
"io"
"io/ioutil"
"os"
@ -17,7 +18,7 @@ import (
// Saver implements saving data in a backend.
type Saver interface {
Save(h restic.Handle, jp []byte) error
Save(restic.Handle, io.Reader) error
}
// packerManager keeps a list of open packs and creates new on demand.
@ -117,7 +118,7 @@ func (r *Repository) savePacker(p *pack.Packer) error {
id := restic.Hash(data)
h := restic.Handle{Type: restic.DataFile, Name: id.String()}
err = r.be.Save(h, data)
err = r.be.Save(h, bytes.NewReader(data))
if err != nil {
debug.Log("Save(%v) error: %v", h, err)
return err

14
src/restic/repository/packer_manager_test.go
View File

@ -1,12 +1,14 @@
package repository
import (
"bytes"
"io"
"math/rand"
"os"
"restic"
"restic/backend/mem"
"restic/crypto"
"restic/mock"
"testing"
)
@ -65,7 +67,7 @@ func saveFile(t testing.TB, be Saver, filename string, n int) {
h := restic.Handle{Type: restic.DataFile, Name: restic.Hash(data).String()}
err = be.Save(h, data)
err = be.Save(h, bytes.NewReader(data))
if err != nil {
t.Fatal(err)
}
@ -134,12 +136,6 @@ func flushRemainingPacks(t testing.TB, rnd *randReader, be Saver, pm *packerMana
return bytes
}
type fakeBackend struct{}
func (f *fakeBackend) Save(h restic.Handle, data []byte) error {
return nil
}
func TestPackerManager(t *testing.T) {
rnd := newRandReader(rand.NewSource(23))
@ -157,7 +153,9 @@ func TestPackerManager(t *testing.T) {
func BenchmarkPackerManager(t *testing.B) {
rnd := newRandReader(rand.NewSource(23))
be := &fakeBackend{}
be := &mock.Backend{
SaveFn: func(restic.Handle, io.Reader) error { return nil },
}
pm := newPackerManager(be, crypto.NewRandomKey())
blobBuf := make([]byte, maxBlobSize)

2
src/restic/repository/repository.go
View File

@ -220,7 +220,7 @@ func (r *Repository) SaveUnpacked(t restic.FileType, p []byte) (id restic.ID, er
id = restic.Hash(ciphertext)
h := restic.Handle{Type: t, Name: id.String()}
err = r.be.Save(h, ciphertext)
err = r.be.Save(h, bytes.NewReader(ciphertext))
if err != nil {
debug.Log("error saving blob %v: %v", h, err)
return restic.ID{}, err