mirror of
https://github.com/octoleo/restic.git
synced 2024-12-23 03:18:55 +00:00
Remove Create() everywhere
This commit is contained in:
parent
ea29ad6f96
commit
1547d3b656
@ -1,7 +1,5 @@
|
||||
package backend
|
||||
|
||||
import "io"
|
||||
|
||||
// Type is the type of a Blob.
|
||||
type Type string
|
||||
|
||||
@ -21,10 +19,6 @@ type Backend interface {
|
||||
// repository.
|
||||
Location() string
|
||||
|
||||
// Create creates a new Blob. The data is available only after Finalize()
|
||||
// has been called on the returned Blob.
|
||||
Create() (Blob, error)
|
||||
|
||||
// Test a boolean value whether a Blob with the name and type exists.
|
||||
Test(t Type, name string) (bool, error)
|
||||
|
||||
@ -65,14 +59,3 @@ type Deleter interface {
|
||||
type BlobInfo struct {
|
||||
Size int64
|
||||
}
|
||||
|
||||
// Blob is old.
|
||||
type Blob interface {
|
||||
io.Writer
|
||||
|
||||
// Finalize moves the data blob to the final location for type and name.
|
||||
Finalize(t Type, name string) error
|
||||
|
||||
// Size returns the number of bytes written to the backend so far.
|
||||
Size() uint
|
||||
}
|
||||
|
@ -51,13 +51,6 @@ func TestLocalBackendLoad(t *testing.T) {
|
||||
test.TestLoad(t)
|
||||
}
|
||||
|
||||
func TestLocalBackendWrite(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestWrite(t)
|
||||
}
|
||||
|
||||
func TestLocalBackendSave(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
|
@ -14,8 +14,7 @@ import (
|
||||
"github.com/restic/restic/debug"
|
||||
)
|
||||
|
||||
var ErrWrongData = errors.New("wrong data returned by backend, checksum does not match")
|
||||
|
||||
// Local is a backend in a local directory.
|
||||
type Local struct {
|
||||
p string
|
||||
mu sync.Mutex
|
||||
@ -80,93 +79,6 @@ func (b *Local) Location() string {
|
||||
return b.p
|
||||
}
|
||||
|
||||
// Return temp directory in correct directory for this backend.
|
||||
func (b *Local) tempFile() (*os.File, error) {
|
||||
return ioutil.TempFile(filepath.Join(b.p, backend.Paths.Temp), "temp-")
|
||||
}
|
||||
|
||||
type localBlob struct {
|
||||
f *os.File
|
||||
size uint
|
||||
final bool
|
||||
basedir string
|
||||
}
|
||||
|
||||
func (lb *localBlob) Write(p []byte) (int, error) {
|
||||
if lb.final {
|
||||
return 0, errors.New("blob already closed")
|
||||
}
|
||||
|
||||
n, err := lb.f.Write(p)
|
||||
lb.size += uint(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (lb *localBlob) Size() uint {
|
||||
return lb.size
|
||||
}
|
||||
|
||||
func (lb *localBlob) Finalize(t backend.Type, name string) error {
|
||||
if lb.final {
|
||||
return errors.New("Already finalized")
|
||||
}
|
||||
|
||||
lb.final = true
|
||||
|
||||
err := lb.f.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("local: file.Close: %v", err)
|
||||
}
|
||||
|
||||
f := filename(lb.basedir, t, name)
|
||||
|
||||
// create directories if necessary, ignore errors
|
||||
if t == backend.Data {
|
||||
os.MkdirAll(filepath.Dir(f), backend.Modes.Dir)
|
||||
}
|
||||
|
||||
// test if new path already exists
|
||||
if _, err := os.Stat(f); err == nil {
|
||||
return fmt.Errorf("Close(): file %v already exists", f)
|
||||
}
|
||||
|
||||
if err := os.Rename(lb.f.Name(), f); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// set mode to read-only
|
||||
fi, err := os.Stat(f)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return setNewFileMode(f, fi)
|
||||
}
|
||||
|
||||
// Create creates a new Blob. The data is available only after Finalize()
|
||||
// has been called on the returned Blob.
|
||||
func (b *Local) Create() (backend.Blob, error) {
|
||||
// TODO: make sure that tempfile is removed upon error
|
||||
|
||||
// create tempfile in backend
|
||||
file, err := b.tempFile()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
blob := localBlob{
|
||||
f: file,
|
||||
basedir: b.p,
|
||||
}
|
||||
|
||||
b.mu.Lock()
|
||||
open, _ := b.open["blobs"]
|
||||
b.open["blobs"] = append(open, file)
|
||||
b.mu.Unlock()
|
||||
|
||||
return &blob, nil
|
||||
}
|
||||
|
||||
// Construct path for given Type and name.
|
||||
func filename(base string, t backend.Type, name string) string {
|
||||
if t == backend.Config {
|
||||
|
@ -51,13 +51,6 @@ func TestMemBackendLoad(t *testing.T) {
|
||||
test.TestLoad(t)
|
||||
}
|
||||
|
||||
func TestMemBackendWrite(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestWrite(t)
|
||||
}
|
||||
|
||||
func TestMemBackendSave(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
|
@ -37,10 +37,6 @@ func New() *MemoryBackend {
|
||||
return memTest(be, t, name)
|
||||
}
|
||||
|
||||
be.MockBackend.CreateFn = func() (backend.Blob, error) {
|
||||
return memCreate(be)
|
||||
}
|
||||
|
||||
be.MockBackend.LoadFn = func(h backend.Handle, p []byte, off int64) (int, error) {
|
||||
return memLoad(be, h, p, off)
|
||||
}
|
||||
@ -127,12 +123,6 @@ func (e *tempMemEntry) Finalize(t backend.Type, name string) error {
|
||||
return e.be.insert(t, name, e.data.Bytes())
|
||||
}
|
||||
|
||||
func memCreate(be *MemoryBackend) (backend.Blob, error) {
|
||||
blob := &tempMemEntry{be: be}
|
||||
debug.Log("MemoryBackend.Create", "create new blob %p", blob)
|
||||
return blob, nil
|
||||
}
|
||||
|
||||
func memLoad(be *MemoryBackend, h backend.Handle, p []byte, off int64) (int, error) {
|
||||
if err := h.Valid(); err != nil {
|
||||
return 0, err
|
||||
@ -179,6 +169,10 @@ func memSave(be *MemoryBackend, h backend.Handle, p []byte) error {
|
||||
h.Name = ""
|
||||
}
|
||||
|
||||
if _, ok := be.data[entry{h.Type, h.Name}]; ok {
|
||||
return errors.New("file already exists")
|
||||
}
|
||||
|
||||
debug.Log("MemoryBackend.Save", "save %v bytes at %v", len(p), h)
|
||||
buf := make([]byte, len(p))
|
||||
copy(buf, p)
|
||||
|
@ -6,7 +6,6 @@ import "errors"
|
||||
// should only be used for tests.
|
||||
type MockBackend struct {
|
||||
CloseFn func() error
|
||||
CreateFn func() (Blob, error)
|
||||
LoadFn func(h Handle, p []byte, off int64) (int, error)
|
||||
SaveFn func(h Handle, p []byte) error
|
||||
StatFn func(h Handle) (BlobInfo, error)
|
||||
@ -33,14 +32,6 @@ func (m *MockBackend) Location() string {
|
||||
return m.LocationFn()
|
||||
}
|
||||
|
||||
func (m *MockBackend) Create() (Blob, error) {
|
||||
if m.CreateFn == nil {
|
||||
return nil, errors.New("not implemented")
|
||||
}
|
||||
|
||||
return m.CreateFn()
|
||||
}
|
||||
|
||||
func (m *MockBackend) Load(h Handle, p []byte, off int64) (int, error) {
|
||||
if m.LoadFn == nil {
|
||||
return 0, errors.New("not implemented")
|
||||
|
@ -51,13 +51,6 @@ func TestS3BackendLoad(t *testing.T) {
|
||||
test.TestLoad(t)
|
||||
}
|
||||
|
||||
func TestS3BackendWrite(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestWrite(t)
|
||||
}
|
||||
|
||||
func TestS3BackendSave(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
|
@ -2,7 +2,6 @@ package s3
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
@ -23,6 +22,7 @@ func s3path(t backend.Type, name string) string {
|
||||
return backendPrefix + "/" + string(t) + "/" + name
|
||||
}
|
||||
|
||||
// S3Backend is a backend which stores the data on an S3 endpoint.
|
||||
type S3Backend struct {
|
||||
client minio.CloudStorageClient
|
||||
connChan chan struct{}
|
||||
@ -68,83 +68,6 @@ func (be *S3Backend) Location() string {
|
||||
return be.bucketname
|
||||
}
|
||||
|
||||
type s3Blob struct {
|
||||
b *S3Backend
|
||||
buf *bytes.Buffer
|
||||
final bool
|
||||
}
|
||||
|
||||
func (bb *s3Blob) Write(p []byte) (int, error) {
|
||||
if bb.final {
|
||||
return 0, errors.New("blob already closed")
|
||||
}
|
||||
|
||||
n, err := bb.buf.Write(p)
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (bb *s3Blob) Read(p []byte) (int, error) {
|
||||
return bb.buf.Read(p)
|
||||
}
|
||||
|
||||
func (bb *s3Blob) Close() error {
|
||||
bb.final = true
|
||||
bb.buf.Reset()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (bb *s3Blob) Size() uint {
|
||||
return uint(bb.buf.Len())
|
||||
}
|
||||
|
||||
func (bb *s3Blob) Finalize(t backend.Type, name string) error {
|
||||
debug.Log("s3.blob.Finalize()", "bucket %v, finalize %v, %d bytes", bb.b.bucketname, name, bb.buf.Len())
|
||||
if bb.final {
|
||||
return errors.New("Already finalized")
|
||||
}
|
||||
|
||||
bb.final = true
|
||||
|
||||
path := s3path(t, name)
|
||||
|
||||
// Check key does not already exist
|
||||
_, err := bb.b.client.StatObject(bb.b.bucketname, path)
|
||||
if err == nil {
|
||||
debug.Log("s3.blob.Finalize()", "%v already exists", name)
|
||||
return errors.New("key already exists")
|
||||
}
|
||||
|
||||
expectedBytes := bb.buf.Len()
|
||||
|
||||
<-bb.b.connChan
|
||||
debug.Log("s3.Finalize", "PutObject(%v, %v, %v, %v)",
|
||||
bb.b.bucketname, path, int64(bb.buf.Len()), "binary/octet-stream")
|
||||
n, err := bb.b.client.PutObject(bb.b.bucketname, path, bb.buf, "binary/octet-stream")
|
||||
debug.Log("s3.Finalize", "finalized %v -> n %v, err %#v", path, n, err)
|
||||
bb.b.connChan <- struct{}{}
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if n != int64(expectedBytes) {
|
||||
return errors.New("could not store all bytes")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Create creates a new Blob. The data is available only after Finalize()
|
||||
// has been called on the returned Blob.
|
||||
func (be *S3Backend) Create() (backend.Blob, error) {
|
||||
blob := s3Blob{
|
||||
b: be,
|
||||
buf: &bytes.Buffer{},
|
||||
}
|
||||
|
||||
return &blob, nil
|
||||
}
|
||||
|
||||
// Load returns the data stored in the backend for h at the given offset
|
||||
// and saves it in p. Load has the same semantics as io.ReaderAt.
|
||||
func (be S3Backend) Load(h backend.Handle, p []byte, off int64) (int, error) {
|
||||
|
@ -51,13 +51,6 @@ func TestSftpBackendLoad(t *testing.T) {
|
||||
test.TestLoad(t)
|
||||
}
|
||||
|
||||
func TestSftpBackendWrite(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestWrite(t)
|
||||
}
|
||||
|
||||
func TestSftpBackendSave(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
|
@ -22,6 +22,7 @@ const (
|
||||
tempfileRandomSuffixLength = 10
|
||||
)
|
||||
|
||||
// SFTP is a backend in a directory accessed via SFTP.
|
||||
type SFTP struct {
|
||||
c *sftp.Client
|
||||
p string
|
||||
@ -253,64 +254,7 @@ func (r *SFTP) renameFile(oldname string, t backend.Type, name string) error {
|
||||
return r.c.Chmod(filename, fi.Mode()&os.FileMode(^uint32(0222)))
|
||||
}
|
||||
|
||||
type sftpBlob struct {
|
||||
f *sftp.File
|
||||
tempname string
|
||||
size uint
|
||||
closed bool
|
||||
backend *SFTP
|
||||
}
|
||||
|
||||
func (sb *sftpBlob) Finalize(t backend.Type, name string) error {
|
||||
if sb.closed {
|
||||
return errors.New("Close() called on closed file")
|
||||
}
|
||||
sb.closed = true
|
||||
|
||||
err := sb.f.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("sftp: file.Close: %v", err)
|
||||
}
|
||||
|
||||
// rename file
|
||||
err = sb.backend.renameFile(sb.tempname, t, name)
|
||||
if err != nil {
|
||||
return fmt.Errorf("sftp: renameFile: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sb *sftpBlob) Write(p []byte) (int, error) {
|
||||
n, err := sb.f.Write(p)
|
||||
sb.size += uint(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (sb *sftpBlob) Size() uint {
|
||||
return sb.size
|
||||
}
|
||||
|
||||
// Create creates a new Blob. The data is available only after Finalize()
|
||||
// has been called on the returned Blob.
|
||||
func (r *SFTP) Create() (backend.Blob, error) {
|
||||
// TODO: make sure that tempfile is removed upon error
|
||||
|
||||
// create tempfile in backend
|
||||
filename, file, err := r.tempFile()
|
||||
if err != nil {
|
||||
return nil, errors.Annotate(err, "create tempfile")
|
||||
}
|
||||
|
||||
blob := sftpBlob{
|
||||
f: file,
|
||||
tempname: filename,
|
||||
backend: r,
|
||||
}
|
||||
|
||||
return &blob, nil
|
||||
}
|
||||
|
||||
// Join joins the given paths and cleans them afterwards.
|
||||
func Join(parts ...string) string {
|
||||
return filepath.Clean(strings.Join(parts, "/"))
|
||||
}
|
||||
@ -515,16 +459,16 @@ func (r *SFTP) List(t backend.Type, done <-chan struct{}) <-chan string {
|
||||
}
|
||||
|
||||
// Close closes the sftp connection and terminates the underlying command.
|
||||
func (s *SFTP) Close() error {
|
||||
if s == nil {
|
||||
func (r *SFTP) Close() error {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
s.c.Close()
|
||||
r.c.Close()
|
||||
|
||||
if err := s.cmd.Process.Kill(); err != nil {
|
||||
if err := r.cmd.Process.Kill(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return s.cmd.Wait()
|
||||
return r.cmd.Wait()
|
||||
}
|
||||
|
@ -51,13 +51,6 @@ func TestTestBackendLoad(t *testing.T) {
|
||||
test.TestLoad(t)
|
||||
}
|
||||
|
||||
func TestTestBackendWrite(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
}
|
||||
test.TestWrite(t)
|
||||
}
|
||||
|
||||
func TestTestBackendSave(t *testing.T) {
|
||||
if SkipMessage != "" {
|
||||
t.Skip(SkipMessage)
|
||||
|
@ -156,19 +156,9 @@ func TestConfig(t testing.TB) {
|
||||
t.Fatalf("did not get expected error for non-existing config")
|
||||
}
|
||||
|
||||
blob, err := b.Create()
|
||||
err = b.Save(backend.Handle{Type: backend.Config}, []byte(testString))
|
||||
if err != nil {
|
||||
t.Fatalf("Create() error: %v", err)
|
||||
}
|
||||
|
||||
_, err = blob.Write([]byte(testString))
|
||||
if err != nil {
|
||||
t.Fatalf("Write() error: %v", err)
|
||||
}
|
||||
|
||||
err = blob.Finalize(backend.Config, "")
|
||||
if err != nil {
|
||||
t.Fatalf("Finalize() error: %v", err)
|
||||
t.Fatalf("Save() error: %v", err)
|
||||
}
|
||||
|
||||
// try accessing the config with different names, should all return the
|
||||
@ -205,12 +195,11 @@ func TestLoad(t testing.TB) {
|
||||
data := Random(23, length)
|
||||
id := backend.Hash(data)
|
||||
|
||||
blob, err := b.Create()
|
||||
OK(t, err)
|
||||
|
||||
_, err = blob.Write([]byte(data))
|
||||
OK(t, err)
|
||||
OK(t, blob.Finalize(backend.Data, id.String()))
|
||||
handle := backend.Handle{Type: backend.Data, Name: id.String()}
|
||||
err = b.Save(handle, data)
|
||||
if err != nil {
|
||||
t.Fatalf("Save() error: %v", err)
|
||||
}
|
||||
|
||||
for i := 0; i < 500; i++ {
|
||||
l := rand.Intn(length + 2000)
|
||||
@ -229,8 +218,7 @@ func TestLoad(t testing.TB) {
|
||||
}
|
||||
|
||||
buf := make([]byte, l)
|
||||
h := backend.Handle{Type: backend.Data, Name: id.String()}
|
||||
n, err := b.Load(h, buf, int64(o))
|
||||
n, err := b.Load(handle, buf, int64(o))
|
||||
|
||||
// if we requested data beyond the end of the file, ignore
|
||||
// ErrUnexpectedEOF error
|
||||
@ -260,63 +248,6 @@ func TestLoad(t testing.TB) {
|
||||
OK(t, b.Remove(backend.Data, id.String()))
|
||||
}
|
||||
|
||||
// TestWrite tests writing data to the backend.
|
||||
func TestWrite(t testing.TB) {
|
||||
b := open(t)
|
||||
defer close(t)
|
||||
|
||||
length := rand.Intn(1<<23) + 2000
|
||||
|
||||
data := Random(23, length)
|
||||
id := backend.Hash(data)
|
||||
|
||||
for i := 0; i < 10; i++ {
|
||||
blob, err := b.Create()
|
||||
OK(t, err)
|
||||
|
||||
o := 0
|
||||
for o < len(data) {
|
||||
l := rand.Intn(len(data) - o)
|
||||
if len(data)-o < 20 {
|
||||
l = len(data) - o
|
||||
}
|
||||
|
||||
n, err := blob.Write(data[o : o+l])
|
||||
OK(t, err)
|
||||
if n != l {
|
||||
t.Fatalf("wrong number of bytes written, want %v, got %v", l, n)
|
||||
}
|
||||
|
||||
o += l
|
||||
}
|
||||
|
||||
name := fmt.Sprintf("%s-%d", id, i)
|
||||
OK(t, blob.Finalize(backend.Data, name))
|
||||
|
||||
buf, err := backend.LoadAll(b, backend.Handle{Type: backend.Data, Name: name}, nil)
|
||||
OK(t, err)
|
||||
if len(buf) != len(data) {
|
||||
t.Fatalf("number of bytes does not match, want %v, got %v", len(data), len(buf))
|
||||
}
|
||||
|
||||
if !bytes.Equal(buf, data) {
|
||||
t.Fatalf("data not equal")
|
||||
}
|
||||
|
||||
fi, err := b.Stat(backend.Handle{Type: backend.Data, Name: name})
|
||||
OK(t, err)
|
||||
|
||||
if fi.Size != int64(len(data)) {
|
||||
t.Fatalf("Stat() returned different size, want %q, got %d", len(data), fi.Size)
|
||||
}
|
||||
|
||||
err = b.Remove(backend.Data, name)
|
||||
if err != nil {
|
||||
t.Fatalf("error removing item: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestSave tests saving data in the backend.
|
||||
func TestSave(t testing.TB) {
|
||||
b := open(t)
|
||||
@ -415,13 +346,8 @@ var testStrings = []struct {
|
||||
|
||||
func store(t testing.TB, b backend.Backend, tpe backend.Type, data []byte) {
|
||||
id := backend.Hash(data)
|
||||
|
||||
blob, err := b.Create()
|
||||
err := b.Save(backend.Handle{Name: id.String(), Type: tpe}, data)
|
||||
OK(t, err)
|
||||
|
||||
_, err = blob.Write([]byte(data))
|
||||
OK(t, err)
|
||||
OK(t, blob.Finalize(tpe, id.String()))
|
||||
}
|
||||
|
||||
func read(t testing.TB, rd io.Reader, expectedData []byte) {
|
||||
@ -492,12 +418,7 @@ func TestBackend(t testing.TB) {
|
||||
test := testStrings[0]
|
||||
|
||||
// create blob
|
||||
blob, err := b.Create()
|
||||
OK(t, err)
|
||||
|
||||
_, err = blob.Write([]byte(test.data))
|
||||
OK(t, err)
|
||||
err = blob.Finalize(tpe, test.id)
|
||||
err := b.Save(backend.Handle{Type: tpe, Name: test.id}, []byte(test.data))
|
||||
Assert(t, err != nil, "expected error, got %v", err)
|
||||
|
||||
// remove and recreate
|
||||
@ -510,13 +431,9 @@ func TestBackend(t testing.TB) {
|
||||
Assert(t, ok == false, "removed blob still present")
|
||||
|
||||
// create blob
|
||||
blob, err = b.Create()
|
||||
err = b.Save(backend.Handle{Type: tpe, Name: test.id}, []byte(test.data))
|
||||
OK(t, err)
|
||||
|
||||
_, err = io.Copy(blob, bytes.NewReader([]byte(test.data)))
|
||||
OK(t, err)
|
||||
OK(t, blob.Finalize(tpe, test.id))
|
||||
|
||||
// list items
|
||||
IDs := backend.IDs{}
|
||||
|
||||
|
@ -20,14 +20,9 @@ func TestLoadAll(t *testing.T) {
|
||||
data := Random(23+i, rand.Intn(MiB)+500*KiB)
|
||||
|
||||
id := backend.Hash(data)
|
||||
|
||||
blob, err := b.Create()
|
||||
err := b.Save(backend.Handle{Name: id.String(), Type: backend.Data}, data)
|
||||
OK(t, err)
|
||||
|
||||
_, err = blob.Write([]byte(data))
|
||||
OK(t, err)
|
||||
OK(t, blob.Finalize(backend.Data, id.String()))
|
||||
|
||||
buf, err := backend.LoadAll(b, backend.Handle{Type: backend.Data, Name: id.String()}, nil)
|
||||
OK(t, err)
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user