octoleo
/
restic
mirror of https://github.com/octoleo/restic.git
2
2
Fork 0
Code Issues Releases Wiki Activity

wip

This commit is contained in:
Alexander Neumann 2016-08-31 22:51:35 +02:00
parent cc6a8b6e15
commit 4c95d2cfdc
33 changed files with 121 additions and 719 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/restic/archiver/archiver_int_test.go
View File

@ -82,12 +82,12 @@ func (j testPipeJob) Error() error { return j.err }
func (j testPipeJob) Info() os.FileInfo { return j.fi }
func (j testPipeJob) Result() chan<- pipe.Result { return j.res }
func testTreeWalker(done <-chan struct{}, out chan<- WalkTreeJob) {
func testTreeWalker(done <-chan struct{}, out chan<- restic.WalkTreeJob) {
for _, e := range treeJobs {
select {
case <-done:
return
case out <- WalkTreeJob{Path: e}:
case out <- restic.WalkTreeJob{Path: e}:
}
}
@ -109,7 +109,7 @@ func testPipeWalker(done <-chan struct{}, out chan<- pipe.Job) {
func TestArchivePipe(t *testing.T) {
done := make(chan struct{})
treeCh := make(chan WalkTreeJob)
treeCh := make(chan restic.WalkTreeJob)
pipeCh := make(chan pipe.Job)
go testTreeWalker(done, treeCh)

3
src/restic/backend.go
View File

@ -31,9 +31,6 @@ type Backend interface {
// arbitrary order. A goroutine is started for this. If the channel done is
// closed, sending stops.
List(t FileType, done <-chan struct{}) <-chan string
// Delete the complete repository.
Delete() error
}
// FileInfo is returned by Stat() and contains information about a file in the

14
src/restic/backend/generic.go
View File

@ -1,6 +1,10 @@
package backend
import "github.com/pkg/errors"
import (
"restic"
"github.com/pkg/errors"
)
// ErrNoIDPrefixFound is returned by Find() when no ID for the given prefix
// could be found.
@ -13,7 +17,7 @@ var ErrMultipleIDMatches = errors.New("multiple IDs with prefix found")
// Find loads the list of all blobs of type t and searches for names which
// start with prefix. If none is found, nil and ErrNoIDPrefixFound is returned.
// If more than one is found, nil and ErrMultipleIDMatches is returned.
func Find(be Lister, t Type, prefix string) (string, error) {
func Find(be restic.Lister, t restic.FileType, prefix string) (string, error) {
done := make(chan struct{})
defer close(done)
@ -41,7 +45,7 @@ const minPrefixLength = 8
// PrefixLength returns the number of bytes required so that all prefixes of
// all names of type t are unique.
func PrefixLength(be Lister, t Type) (int, error) {
func PrefixLength(be restic.Lister, t restic.FileType) (int, error) {
done := make(chan struct{})
defer close(done)
@ -53,7 +57,7 @@ func PrefixLength(be Lister, t Type) (int, error) {
// select prefixes of length l, test if the last one is the same as the current one
outer:
for l := minPrefixLength; l < IDSize; l++ {
for l := minPrefixLength; l < restic.IDSize; l++ {
var last string
for _, name := range list {
@ -66,5 +70,5 @@ outer:
return l, nil
}
return IDSize, nil
return restic.IDSize, nil
}

2
src/restic/backend/generic_test.go
View File

@ -16,7 +16,7 @@ func (m mockBackend) List(t restic.FileType, done <-chan struct{}) <-chan string
return m.list(t, done)
}
var samples = backend.IDs{
var samples = restic.IDs{
ParseID("20bdc1402a6fc9b633aaffffffffffffffffffffffffffffffffffffffffffff"),
ParseID("20bdc1402a6fc9b633ccd578c4a92d0f4ef1a457fa2e16c596bc73fb409d6cc0"),
ParseID("20bdc1402a6fc9b633ffffffffffffffffffffffffffffffffffffffffffffff"),

49
src/restic/backend/handle.go
View File

@ -1,49 +0,0 @@
package backend
import (
"fmt"
"github.com/pkg/errors"
)
// Handle is used to store and access data in a backend.
type Handle struct {
Type Type
Name string
}
func (h Handle) String() string {
name := h.Name
if len(name) > 10 {
name = name[:10]
}
return fmt.Sprintf("<%s/%s>", h.Type, name)
}
// Valid returns an error if h is not valid.
func (h Handle) Valid() error {
if h.Type == "" {
return errors.New("type is empty")
}
switch h.Type {
case Data:
case Key:
case Lock:
case Snapshot:
case Index:
case Config:
default:
return errors.Errorf("invalid Type %q", h.Type)
}
if h.Type == Config {
return nil
}
if h.Name == "" {
return errors.New("invalid Name")
}
return nil
}

28
src/restic/backend/handle_test.go
View File

@ -1,28 +0,0 @@
package backend
import "testing"
var handleTests = []struct {
h Handle
valid bool
}{
{Handle{Name: "foo"}, false},
{Handle{Type: "foobar"}, false},
{Handle{Type: Config, Name: ""}, true},
{Handle{Type: Data, Name: ""}, false},
{Handle{Type: "", Name: "x"}, false},
{Handle{Type: Lock, Name: "010203040506"}, true},
}
func TestHandleValid(t *testing.T) {
for i, test := range handleTests {
err := test.h.Valid()
if err != nil && test.valid {
t.Errorf("test %v failed: error returned for valid handle: %v", i, err)
}
if !test.valid && err == nil {
t.Errorf("test %v failed: expected error for invalid handle not found", i)
}
}
}

109
src/restic/backend/id.go
View File

@ -1,109 +0,0 @@
package backend
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"github.com/pkg/errors"
)
// Hash returns the ID for data.
func Hash(data []byte) ID {
return sha256.Sum256(data)
}
// IDSize contains the size of an ID, in bytes.
const IDSize = sha256.Size
// ID references content within a repository.
type ID [IDSize]byte
// ParseID converts the given string to an ID.
func ParseID(s string) (ID, error) {
b, err := hex.DecodeString(s)
if err != nil {
return ID{}, errors.Wrap(err, "hex.DecodeString")
}
if len(b) != IDSize {
return ID{}, errors.New("invalid length for hash")
}
id := ID{}
copy(id[:], b)
return id, nil
}
func (id ID) String() string {
return hex.EncodeToString(id[:])
}
const shortStr = 4
// Str returns the shortened string version of id.
func (id *ID) Str() string {
if id == nil {
return "[nil]"
}
if id.IsNull() {
return "[null]"
}
return hex.EncodeToString(id[:shortStr])
}
// IsNull returns true iff id only consists of null bytes.
func (id ID) IsNull() bool {
var nullID ID
return id == nullID
}
// Equal compares an ID to another other.
func (id ID) Equal(other ID) bool {
return id == other
}
// EqualString compares this ID to another one, given as a string.
func (id ID) EqualString(other string) (bool, error) {
s, err := hex.DecodeString(other)
if err != nil {
return false, errors.Wrap(err, "hex.DecodeString")
}
id2 := ID{}
copy(id2[:], s)
return id == id2, nil
}
// Compare compares this ID to another one, returning -1, 0, or 1.
func (id ID) Compare(other ID) int {
return bytes.Compare(other[:], id[:])
}
// MarshalJSON returns the JSON encoding of id.
func (id ID) MarshalJSON() ([]byte, error) {
return json.Marshal(id.String())
}
// UnmarshalJSON parses the JSON-encoded data and stores the result in id.
func (id *ID) UnmarshalJSON(b []byte) error {
var s string
err := json.Unmarshal(b, &s)
if err != nil {
return errors.Wrap(err, "Unmarshal")
}
_, err = hex.Decode(id[:], []byte(s))
if err != nil {
return errors.Wrap(err, "hex.Decode")
}
return nil
}

16
src/restic/backend/id_int_test.go
View File

@ -1,16 +0,0 @@
package backend
import "testing"
func TestIDMethods(t *testing.T) {
var id ID
if id.Str() != "[null]" {
t.Errorf("ID.Str() returned wrong value, want %v, got %v", "[null]", id.Str())
}
var pid *ID
if pid.Str() != "[nil]" {
t.Errorf("ID.Str() returned wrong value, want %v, got %v", "[nil]", pid.Str())
}
}

43
src/restic/backend/id_test.go
View File

@ -1,43 +0,0 @@
package backend_test
import (
"testing"
"restic/backend"
. "restic/test"
)
var TestStrings = []struct {
id string
data string
}{
{"c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2", "foobar"},
{"248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"},
{"cc5d46bdb4991c6eae3eb739c9c8a7a46fe9654fab79c47b4fe48383b5b25e1c", "foo/bar"},
{"4e54d2c721cbdb730f01b10b62dec622962b36966ec685880effa63d71c808f2", "foo/../../baz"},
}
func TestID(t *testing.T) {
for _, test := range TestStrings {
id, err := backend.ParseID(test.id)
OK(t, err)
id2, err := backend.ParseID(test.id)
OK(t, err)
Assert(t, id.Equal(id2), "ID.Equal() does not work as expected")
ret, err := id.EqualString(test.id)
OK(t, err)
Assert(t, ret, "ID.EqualString() returned wrong value")
// test json marshalling
buf, err := id.MarshalJSON()
OK(t, err)
Equals(t, "\""+test.id+"\"", string(buf))
var id3 backend.ID
err = id3.UnmarshalJSON(buf)
OK(t, err)
Equals(t, id, id3)
}
}

69
src/restic/backend/ids.go
View File

@ -1,69 +0,0 @@
package backend
import (
"encoding/hex"
"fmt"
)
// IDs is an ordered list of IDs that implements sort.Interface.
type IDs []ID
func (ids IDs) Len() int {
return len(ids)
}
func (ids IDs) Less(i, j int) bool {
if len(ids[i]) < len(ids[j]) {
return true
}
for k, b := range ids[i] {
if b == ids[j][k] {
continue
}
if b < ids[j][k] {
return true
}
return false
}
return false
}
func (ids IDs) Swap(i, j int) {
ids[i], ids[j] = ids[j], ids[i]
}
// Uniq returns list without duplicate IDs. The returned list retains the order
// of the original list so that the order of the first occurrence of each ID
// stays the same.
func (ids IDs) Uniq() (list IDs) {
seen := NewIDSet()
for _, id := range ids {
if seen.Has(id) {
continue
}
list = append(list, id)
seen.Insert(id)
}
return list
}
type shortID ID
func (id shortID) String() string {
return hex.EncodeToString(id[:shortStr])
}
func (ids IDs) String() string {
elements := make([]shortID, 0, len(ids))
for _, id := range ids {
elements = append(elements, shortID(id))
}
return fmt.Sprintf("%v", elements)
}

58
src/restic/backend/ids_test.go
View File

@ -1,58 +0,0 @@
package backend_test
import (
"reflect"
"testing"
"restic/backend"
. "restic/test"
)
var uniqTests = []struct {
before, after backend.IDs
}{
{
backend.IDs{
ParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"),
ParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"),
ParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"),
},
backend.IDs{
ParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"),
ParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"),
},
},
{
backend.IDs{
ParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"),
ParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"),
ParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"),
},
backend.IDs{
ParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"),
ParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"),
},
},
{
backend.IDs{
ParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"),
ParseID("f658198b405d7e80db5ace1980d125c8da62f636b586c46bf81dfb856a49d0c8"),
ParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"),
ParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"),
},
backend.IDs{
ParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"),
ParseID("f658198b405d7e80db5ace1980d125c8da62f636b586c46bf81dfb856a49d0c8"),
ParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"),
},
},
}
func TestUniqIDs(t *testing.T) {
for i, test := range uniqTests {
uniq := test.before.Uniq()
if !reflect.DeepEqual(uniq, test.after) {
t.Errorf("uniqIDs() test %v failed\n wanted: %v\n got: %v", i, test.after, uniq)
}
}
}

111
src/restic/backend/idset.go
View File

@ -1,111 +0,0 @@
package backend
import "sort"
// IDSet is a set of IDs.
type IDSet map[ID]struct{}
// NewIDSet returns a new IDSet, populated with ids.
func NewIDSet(ids ...ID) IDSet {
m := make(IDSet)
for _, id := range ids {
m[id] = struct{}{}
}
return m
}
// Has returns true iff id is contained in the set.
func (s IDSet) Has(id ID) bool {
_, ok := s[id]
return ok
}
// Insert adds id to the set.
func (s IDSet) Insert(id ID) {
s[id] = struct{}{}
}
// Delete removes id from the set.
func (s IDSet) Delete(id ID) {
delete(s, id)
}
// List returns a slice of all IDs in the set.
func (s IDSet) List() IDs {
list := make(IDs, 0, len(s))
for id := range s {
list = append(list, id)
}
sort.Sort(list)
return list
}
// Equals returns true iff s equals other.
func (s IDSet) Equals(other IDSet) bool {
if len(s) != len(other) {
return false
}
for id := range s {
if _, ok := other[id]; !ok {
return false
}
}
// length + one-way comparison is sufficient implication of equality
return true
}
// Merge adds the blobs in other to the current set.
func (s IDSet) Merge(other IDSet) {
for id := range other {
s.Insert(id)
}
}
// Intersect returns a new set containing the IDs that are present in both sets.
func (s IDSet) Intersect(other IDSet) (result IDSet) {
result = NewIDSet()
set1 := s
set2 := other
// iterate over the smaller set
if len(set2) < len(set1) {
set1, set2 = set2, set1
}
for id := range set1 {
if set2.Has(id) {
result.Insert(id)
}
}
return result
}
// Sub returns a new set containing all IDs that are present in s but not in
// other.
func (s IDSet) Sub(other IDSet) (result IDSet) {
result = NewIDSet()
for id := range s {
if !other.Has(id) {
result.Insert(id)
}
}
return result
}
func (s IDSet) String() string {
str := s.List().String()
if len(str) < 2 {
return "{}"
}
return "{" + str[1:len(str)-1] + "}"
}

35
src/restic/backend/idset_test.go
View File

@ -1,35 +0,0 @@
package backend_test
import (
"testing"
"restic/backend"
. "restic/test"
)
var idsetTests = []struct {
id backend.ID
seen bool
}{
{ParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), false},
{ParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"), false},
{ParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), true},
{ParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), true},
{ParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"), true},
{ParseID("f658198b405d7e80db5ace1980d125c8da62f636b586c46bf81dfb856a49d0c8"), false},
{ParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), true},
{ParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"), true},
{ParseID("f658198b405d7e80db5ace1980d125c8da62f636b586c46bf81dfb856a49d0c8"), true},
{ParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), true},
}
func TestIDSet(t *testing.T) {
set := backend.NewIDSet()
for i, test := range idsetTests {
seen := set.Has(test.id)
if seen != test.seen {
t.Errorf("IDSet test %v failed: wanted %v, got %v", i, test.seen, seen)
}
set.Insert(test.id)
}
}

63
src/restic/backend/interface.go
View File

@ -1,63 +0,0 @@
package backend
// Type is the type of a Blob.
type Type string
// These are the different data types a backend can store.
const (
Data Type = "data"
Key = "key"
Lock = "lock"
Snapshot = "snapshot"
Index = "index"
Config = "config"
)
// Backend is used to store and access data.
type Backend interface {
// Location returns a string that describes the type and location of the
// repository.
Location() string
// Test a boolean value whether a Blob with the name and type exists.
Test(t Type, name string) (bool, error)
// Remove removes a Blob with type t and name.
Remove(t Type, name string) error
// Close the backend
Close() error
Lister
// Load returns the data stored in the backend for h at the given offset
// and saves it in p. Load has the same semantics as io.ReaderAt, except
// that a negative offset is also allowed. In this case it references a
// position relative to the end of the file (similar to Seek()).
Load(h Handle, p []byte, off int64) (int, error)
// Save stores the data in the backend under the given handle.
Save(h Handle, p []byte) error
// Stat returns information about the blob identified by h.
Stat(h Handle) (BlobInfo, error)
}
// Lister implements listing data items stored in a backend.
type Lister interface {
// List returns a channel that yields all names of blobs of type t in an
// arbitrary order. A goroutine is started for this. If the channel done is
// closed, sending stops.
List(t Type, done <-chan struct{}) <-chan string
}
// Deleter are backends that allow to self-delete all content stored in them.
type Deleter interface {
// Delete the complete repository.
Delete() error
}
// BlobInfo is returned by Stat() and contains information about a stored blob.
type BlobInfo struct {
Size int64
}

6
src/restic/backend/local/local_test.go
View File

@ -4,8 +4,8 @@ import (
"fmt"
"io/ioutil"
"os"
"restic"
"restic/backend"
"restic/backend/local"
"restic/backend/test"
)
@ -30,7 +30,7 @@ func createTempdir() error {
}
func init() {
test.CreateFn = func() (backend.Backend, error) {
test.CreateFn = func() (restic.Backend, error) {
err := createTempdir()
if err != nil {
return nil, err
@ -38,7 +38,7 @@ func init() {
return local.Create(tempBackendDir)
}
test.OpenFn = func() (backend.Backend, error) {
test.OpenFn = func() (restic.Backend, error) {
err := createTempdir()
if err != nil {
return nil, err

9
src/restic/backend/mem/mem_backend_test.go
View File

@ -1,19 +1,20 @@
package mem_test
import (
"restic"
"github.com/pkg/errors"
"restic/backend"
"restic/backend/mem"
"restic/backend/test"
)
var be backend.Backend
var be restic.Backend
//go:generate go run ../test/generate_backend_tests.go
func init() {
test.CreateFn = func() (backend.Backend, error) {
test.CreateFn = func() (restic.Backend, error) {
if be != nil {
return nil, errors.New("temporary memory backend dir already exists")
}
@ -23,7 +24,7 @@ func init() {
return be, nil
}
test.OpenFn = func() (backend.Backend, error) {
test.OpenFn = func() (restic.Backend, error) {
if be == nil {
return nil, errors.New("repository not initialized")
}

16
src/restic/backend/rest/rest.go
View File

@ -54,7 +54,7 @@ type restBackend struct {
}
// Open opens the REST backend with the given config.
func Open(cfg Config) (backend.Backend, error) {
func Open(cfg Config) (restic.Backend, error) {
connChan := make(chan struct{}, connLimit)
for i := 0; i < connLimit; i++ {
connChan <- struct{}{}
@ -152,31 +152,31 @@ func (b *restBackend) Save(h restic.Handle, p []byte) (err error) {
}
// Stat returns information about a blob.
func (b *restBackend) Stat(h restic.Handle) (backend.BlobInfo, error) {
func (b *restBackend) Stat(h restic.Handle) (restic.FileInfo, error) {
if err := h.Valid(); err != nil {
return backend.BlobInfo{}, err
return restic.FileInfo{}, err
}
<-b.connChan
resp, err := b.client.Head(restPath(b.url, h))
b.connChan <- struct{}{}
if err != nil {
return backend.BlobInfo{}, errors.Wrap(err, "client.Head")
return restic.FileInfo{}, errors.Wrap(err, "client.Head")
}
if err = resp.Body.Close(); err != nil {
return backend.BlobInfo{}, errors.Wrap(err, "Close")
return restic.FileInfo{}, errors.Wrap(err, "Close")
}
if resp.StatusCode != 200 {
return backend.BlobInfo{}, errors.Errorf("unexpected HTTP response code %v", resp.StatusCode)
return restic.FileInfo{}, errors.Errorf("unexpected HTTP response code %v", resp.StatusCode)
}
if resp.ContentLength < 0 {
return backend.BlobInfo{}, errors.New("negative content length")
return restic.FileInfo{}, errors.New("negative content length")
}
bi := backend.BlobInfo{
bi := restic.FileInfo{
Size: resp.ContentLength,
}

5
src/restic/backend/rest/rest_test.go
View File

@ -8,7 +8,6 @@ import (
"github.com/pkg/errors"
"restic/backend"
"restic/backend/rest"
"restic/backend/test"
. "restic/test"
@ -32,7 +31,7 @@ func init() {
URL: url,
}
test.CreateFn = func() (backend.Backend, error) {
test.CreateFn = func() (restic.Backend, error) {
be, err := rest.Open(cfg)
if err != nil {
return nil, err
@ -50,7 +49,7 @@ func init() {
return be, nil
}
test.OpenFn = func() (backend.Backend, error) {
test.OpenFn = func() (restic.Backend, error) {
return rest.Open(cfg)
}
}

6
src/restic/backend/s3/s3_test.go
View File

@ -4,10 +4,10 @@ import (
"fmt"
"net/url"
"os"
"restic"
"github.com/pkg/errors"
"restic/backend"
"restic/backend/s3"
"restic/backend/test"
. "restic/test"
@ -38,7 +38,7 @@ func init() {
cfg.UseHTTP = true
}
test.CreateFn = func() (backend.Backend, error) {
test.CreateFn = func() (restic.Backend, error) {
be, err := s3.Open(cfg)
if err != nil {
return nil, err
@ -56,7 +56,7 @@ func init() {
return be, nil
}
test.OpenFn = func() (backend.Backend, error) {
test.OpenFn = func() (restic.Backend, error) {
return s3.Open(cfg)
}

12
src/restic/backend/sftp/sftp.go
View File

@ -34,6 +34,8 @@ type SFTP struct {
result <-chan error
}
var _ restic.Backend = &SFTP{}
func startClient(program string, args ...string) (*SFTP, error) {
// Connect to a remote host and request the sftp subsystem via the 'ssh'
// command. This assumes that passwordless login is correctly configured.
@ -401,22 +403,22 @@ func (r *SFTP) Save(h restic.Handle, p []byte) (err error) {
}
// Stat returns information about a blob.
func (r *SFTP) Stat(h restic.Handle) (backend.BlobInfo, error) {
func (r *SFTP) Stat(h restic.Handle) (restic.FileInfo, error) {
debug.Log("sftp.Stat", "stat %v", h)
if err := r.clientError(); err != nil {
return backend.BlobInfo{}, err
return restic.FileInfo{}, err
}
if err := h.Valid(); err != nil {
return backend.BlobInfo{}, err
return restic.FileInfo{}, err
}
fi, err := r.c.Lstat(r.filename(h.FileType, h.Name))
if err != nil {
return backend.BlobInfo{}, errors.Wrap(err, "Lstat")
return restic.FileInfo{}, errors.Wrap(err, "Lstat")
}
return backend.BlobInfo{Size: fi.Size()}, nil
return restic.FileInfo{Size: fi.Size()}, nil
}
// Test returns true if a blob of the given type and name exists in the backend.

6
src/restic/backend/sftp/sftp_backend_test.go
View File

@ -4,11 +4,11 @@ import (
"io/ioutil"
"os"
"path/filepath"
"restic"
"strings"
"github.com/pkg/errors"
"restic/backend"
"restic/backend/sftp"
"restic/backend/test"
@ -52,7 +52,7 @@ func init() {
args := []string{"-e"}
test.CreateFn = func() (backend.Backend, error) {
test.CreateFn = func() (restic.Backend, error) {
err := createTempdir()
if err != nil {
return nil, err
@ -61,7 +61,7 @@ func init() {
return sftp.Create(tempBackendDir, sftpserver, args...)
}
test.OpenFn = func() (backend.Backend, error) {
test.OpenFn = func() (restic.Backend, error) {
err := createTempdir()
if err != nil {
return nil, err

62
src/restic/backend/test/tests.go
View File

@ -18,18 +18,18 @@ import (
)
// CreateFn is a function that creates a temporary repository for the tests.
var CreateFn func() (backend.Backend, error)
var CreateFn func() (restic.Backend, error)
// OpenFn is a function that opens a previously created temporary repository.
var OpenFn func() (backend.Backend, error)
var OpenFn func() (restic.Backend, error)
// CleanupFn removes temporary files and directories created during the tests.
var CleanupFn func() error
var but backend.Backend // backendUnderTest
var but restic.Backend // backendUnderTest
var butInitialized bool
func open(t testing.TB) backend.Backend {
func open(t testing.TB) restic.Backend {
if OpenFn == nil {
t.Fatal("OpenFn not set")
}
@ -153,12 +153,12 @@ func TestConfig(t testing.TB) {
var testString = "Config"
// create config and read it back
_, err := backend.LoadAll(b, restic.Handle{Type: restic.ConfigFile}, nil)
_, err := backend.LoadAll(b, restic.Handle{FileType: restic.ConfigFile}, nil)
if err == nil {
t.Fatalf("did not get expected error for non-existing config")
}
err = b.Save(restic.Handle{Type: restic.ConfigFile}, []byte(testString))
err = b.Save(restic.Handle{FileType: restic.ConfigFile}, []byte(testString))
if err != nil {
t.Fatalf("Save() error: %v", err)
}
@ -166,7 +166,7 @@ func TestConfig(t testing.TB) {
// try accessing the config with different names, should all return the
// same config
for _, name := range []string{"", "foo", "bar", "0000000000000000000000000000000000000000000000000000000000000000"} {
h := restic.Handle{Type: restic.ConfigFile, Name: name}
h := restic.Handle{FileType: restic.ConfigFile, Name: name}
buf, err := backend.LoadAll(b, h, nil)
if err != nil {
t.Fatalf("unable to read config with name %q: %v", name, err)
@ -188,7 +188,7 @@ func TestLoad(t testing.TB) {
t.Fatalf("Load() did not return an error for invalid handle")
}
_, err = b.Load(restic.Handle{Type: restic.DataFile, Name: "foobar"}, nil, 0)
_, err = b.Load(restic.Handle{FileType: restic.DataFile, Name: "foobar"}, nil, 0)
if err == nil {
t.Fatalf("Load() did not return an error for non-existing blob")
}
@ -196,9 +196,9 @@ func TestLoad(t testing.TB) {
length := rand.Intn(1<<24) + 2000
data := Random(23, length)
id := backend.Hash(data)
id := restic.Hash(data)
handle := restic.Handle{Type: restic.DataFile, Name: id.String()}
handle := restic.Handle{FileType: restic.DataFile, Name: id.String()}
err = b.Save(handle, data)
if err != nil {
t.Fatalf("Save() error: %v", err)
@ -321,9 +321,9 @@ func TestLoadNegativeOffset(t testing.TB) {
length := rand.Intn(1<<24) + 2000
data := Random(23, length)
id := backend.Hash(data)
id := restic.Hash(data)
handle := restic.Handle{Type: restic.DataFile, Name: id.String()}
handle := restic.Handle{FileType: restic.DataFile, Name: id.String()}
err := b.Save(handle, data)
if err != nil {
t.Fatalf("Save() error: %v", err)
@ -373,7 +373,7 @@ func TestLoadNegativeOffset(t testing.TB) {
func TestSave(t testing.TB) {
b := open(t)
defer close(t)
var id backend.ID
var id restic.ID
for i := 0; i < 10; i++ {
length := rand.Intn(1<<23) + 200000
@ -382,8 +382,8 @@ func TestSave(t testing.TB) {
copy(id[:], data)
h := restic.Handle{
Type: restic.DataFile,
Name: fmt.Sprintf("%s-%d", id, i),
FileType: restic.DataFile,
Name: fmt.Sprintf("%s-%d", id, i),
}
err := b.Save(h, data)
OK(t, err)
@ -405,7 +405,7 @@ func TestSave(t testing.TB) {
t.Fatalf("Stat() returned different size, want %q, got %d", len(data), fi.Size)
}
err = b.Remove(h.Type, h.Name)
err = b.Remove(h.FileType, h.Name)
if err != nil {
t.Fatalf("error removing item: %v", err)
}
@ -430,7 +430,7 @@ func TestSaveFilenames(t testing.TB) {
defer close(t)
for i, test := range filenameTests {
h := restic.Handle{Name: test.name, Type: restic.DataFile}
h := restic.Handle{Name: test.name, FileType: restic.DataFile}
err := b.Save(h, []byte(test.data))
if err != nil {
t.Errorf("test %d failed: Save() returned %v", i, err)
@ -447,7 +447,7 @@ func TestSaveFilenames(t testing.TB) {
t.Errorf("test %d: returned wrong bytes", i)
}
err = b.Remove(h.Type, h.Name)
err = b.Remove(h.FileType, h.Name)
if err != nil {
t.Errorf("test %d failed: Remove() returned %v", i, err)
continue
@ -465,9 +465,9 @@ var testStrings = []struct {
{"4e54d2c721cbdb730f01b10b62dec622962b36966ec685880effa63d71c808f2", "foo/../../baz"},
}
func store(t testing.TB, b backend.Backend, tpe restic.FileType, data []byte) {
id := backend.Hash(data)
err := b.Save(restic.Handle{Name: id.String(), Type: tpe}, data)
func store(t testing.TB, b restic.Backend, tpe restic.FileType, data []byte) {
id := restic.Hash(data)
err := b.Save(restic.Handle{Name: id.String(), FileType: tpe}, data)
OK(t, err)
}
@ -490,7 +490,7 @@ func TestBackend(t testing.TB) {
} {
// detect non-existing files
for _, test := range testStrings {
id, err := backend.ParseID(test.id)
id, err := restic.ParseID(test.id)
OK(t, err)
// test if blob is already in repository
@ -499,7 +499,7 @@ func TestBackend(t testing.TB) {
Assert(t, !ret, "blob was found to exist before creating")
// try to stat a not existing blob
h := restic.Handle{Type: tpe, Name: id.String()}
h := restic.Handle{FileType: tpe, Name: id.String()}
_, err = b.Stat(h)
Assert(t, err != nil, "blob data could be extracted before creation")
@ -518,7 +518,7 @@ func TestBackend(t testing.TB) {
store(t, b, tpe, []byte(test.data))
// test Load()
h := restic.Handle{Type: tpe, Name: test.id}
h := restic.Handle{FileType: tpe, Name: test.id}
buf, err := backend.LoadAll(b, h, nil)
OK(t, err)
Equals(t, test.data, string(buf))
@ -539,7 +539,7 @@ func TestBackend(t testing.TB) {
test := testStrings[0]
// create blob
err := b.Save(restic.Handle{Type: tpe, Name: test.id}, []byte(test.data))
err := b.Save(restic.Handle{FileType: tpe, Name: test.id}, []byte(test.data))
Assert(t, err != nil, "expected error, got %v", err)
// remove and recreate
@ -552,19 +552,19 @@ func TestBackend(t testing.TB) {
Assert(t, ok == false, "removed blob still present")
// create blob
err = b.Save(restic.Handle{Type: tpe, Name: test.id}, []byte(test.data))
err = b.Save(restic.Handle{FileType: tpe, Name: test.id}, []byte(test.data))
OK(t, err)
// list items
IDs := backend.IDs{}
IDs := restic.IDs{}
for _, test := range testStrings {
id, err := backend.ParseID(test.id)
id, err := restic.ParseID(test.id)
OK(t, err)
IDs = append(IDs, id)
}
list := backend.IDs{}
list := restic.IDs{}
for s := range b.List(tpe, nil) {
list = append(list, ParseID(s))
@ -584,7 +584,7 @@ func TestBackend(t testing.TB) {
// remove content if requested
if TestCleanupTempDirs {
for _, test := range testStrings {
id, err := backend.ParseID(test.id)
id, err := restic.ParseID(test.id)
OK(t, err)
found, err := b.Test(tpe, id.String())
@ -605,7 +605,7 @@ func TestDelete(t testing.TB) {
b := open(t)
defer close(t)
be, ok := b.(backend.Deleter)
be, ok := b.(restic.Deleter)
if !ok {
return
}

9
src/restic/backend/test/tests_test.go
View File

@ -1,19 +1,20 @@
package test_test
import (
"restic"
"github.com/pkg/errors"
"restic/backend"
"restic/backend/mem"
"restic/backend/test"
)
var be backend.Backend
var be restic.Backend
//go:generate go run ../test/generate_backend_tests.go
func init() {
test.CreateFn = func() (backend.Backend, error) {
test.CreateFn = func() (restic.Backend, error) {
if be != nil {
return nil, errors.New("temporary memory backend dir already exists")
}
@ -23,7 +24,7 @@ func init() {
return be, nil
}
test.OpenFn = func() (backend.Backend, error) {
test.OpenFn = func() (restic.Backend, error) {
if be == nil {
return nil, errors.New("repository not initialized")
}

19
src/restic/backend/utils_test.go
View File

@ -3,6 +3,7 @@ package backend_test
import (
"bytes"
"math/rand"
"restic"
"testing"
"restic/backend"
@ -19,11 +20,11 @@ func TestLoadAll(t *testing.T) {
for i := 0; i < 20; i++ {
data := Random(23+i, rand.Intn(MiB)+500*KiB)
id := backend.Hash(data)
err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, data)
id := restic.Hash(data)
err := b.Save(restic.Handle{Name: id.String(), FileType: restic.DataFile}, data)
OK(t, err)
buf, err := backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()}, nil)
buf, err := backend.LoadAll(b, restic.Handle{FileType: restic.DataFile, Name: id.String()}, nil)
OK(t, err)
if len(buf) != len(data) {
@ -44,12 +45,12 @@ func TestLoadSmallBuffer(t *testing.T) {
for i := 0; i < 20; i++ {
data := Random(23+i, rand.Intn(MiB)+500*KiB)
id := backend.Hash(data)
err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, data)
id := restic.Hash(data)
err := b.Save(restic.Handle{Name: id.String(), FileType: restic.DataFile}, data)
OK(t, err)
buf := make([]byte, len(data)-23)
buf, err = backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()}, buf)
buf, err = backend.LoadAll(b, restic.Handle{FileType: restic.DataFile, Name: id.String()}, buf)
OK(t, err)
if len(buf) != len(data) {
@ -70,12 +71,12 @@ func TestLoadLargeBuffer(t *testing.T) {
for i := 0; i < 20; i++ {
data := Random(23+i, rand.Intn(MiB)+500*KiB)
id := backend.Hash(data)
err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, data)
id := restic.Hash(data)
err := b.Save(restic.Handle{Name: id.String(), FileType: restic.DataFile}, data)
OK(t, err)
buf := make([]byte, len(data)+100)
buf, err = backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()}, buf)
buf, err = backend.LoadAll(b, restic.Handle{FileType: restic.DataFile, Name: id.String()}, buf)
OK(t, err)
if len(buf) != len(data) {

10
src/restic/fuse/file.go
View File

@ -9,9 +9,7 @@ import (
"github.com/pkg/errors"
"restic"
"restic/backend"
"restic/debug"
"restic/pack"
"bazil.org/fuse"
"bazil.org/fuse/fs"
@ -28,8 +26,8 @@ var _ = fs.HandleReleaser(&file{})
// BlobLoader is an abstracted repository with a reduced set of methods used
// for fuse operations.
type BlobLoader interface {
LookupBlobSize(backend.ID, pack.BlobType) (uint, error)
LoadBlob(backend.ID, pack.BlobType, []byte) ([]byte, error)
LookupBlobSize(restic.ID, restic.BlobType) (uint, error)
LoadBlob(restic.ID, restic.BlobType, []byte) ([]byte, error)
}
type file struct {
@ -54,7 +52,7 @@ func newFile(repo BlobLoader, node *restic.Node, ownerIsRoot bool) (*file, error
var bytes uint64
sizes := make([]uint, len(node.Content))
for i, id := range node.Content {
size, err := repo.LookupBlobSize(id, pack.Data)
size, err := repo.LookupBlobSize(id, restic.DataBlob)
if err != nil {
return nil, err
}
@ -111,7 +109,7 @@ func (f *file) getBlobAt(i int) (blob []byte, err error) {
buf = make([]byte, f.sizes[i])
}
blob, err = f.repo.LoadBlob(f.node.Content[i], pack.Data, buf)
blob, err = f.repo.LoadBlob(f.node.Content[i], restic.DataBlob, buf)
if err != nil {
debug.Log("file.getBlobAt", "LoadBlob(%v, %v) failed: %v", f.node.Name, f.node.Content[i], err)
return nil, err

18
src/restic/fuse/file_test.go
View File

@ -14,20 +14,18 @@ import (
"bazil.org/fuse"
"restic"
"restic/backend"
"restic/pack"
. "restic/test"
)
type MockRepo struct {
blobs map[backend.ID][]byte
blobs map[restic.ID][]byte
}
func NewMockRepo(content map[backend.ID][]byte) *MockRepo {
func NewMockRepo(content map[restic.ID][]byte) *MockRepo {
return &MockRepo{blobs: content}
}
func (m *MockRepo) LookupBlobSize(id backend.ID, t pack.BlobType) (uint, error) {
func (m *MockRepo) LookupBlobSize(id restic.ID, t restic.BlobType) (uint, error) {
buf, ok := m.blobs[id]
if !ok {
return 0, errors.New("blob not found")
@ -36,7 +34,7 @@ func (m *MockRepo) LookupBlobSize(id backend.ID, t pack.BlobType) (uint, error)
return uint(len(buf)), nil
}
func (m *MockRepo) LoadBlob(id backend.ID, t pack.BlobType, buf []byte) ([]byte, error) {
func (m *MockRepo) LoadBlob(id restic.ID, t restic.BlobType, buf []byte) ([]byte, error) {
size, err := m.LookupBlobSize(id, t)
if err != nil {
return nil, err
@ -68,12 +66,12 @@ var testContentLengths = []uint{
}
var testMaxFileSize uint
func genTestContent() map[backend.ID][]byte {
m := make(map[backend.ID][]byte)
func genTestContent() map[restic.ID][]byte {
m := make(map[restic.ID][]byte)
for _, length := range testContentLengths {
buf := Random(int(length), int(length))
id := backend.Hash(buf)
id := restic.Hash(buf)
m[id] = buf
testMaxFileSize += length
}
@ -111,7 +109,7 @@ func TestFuseFile(t *testing.T) {
memfile := make([]byte, 0, maxBufSize)
var ids backend.IDs
var ids restic.IDs
for id, buf := range repo.blobs {
ids = append(ids, id)
memfile = append(memfile, buf...)

5
src/restic/fuse/fuse.go
View File

@ -5,13 +5,12 @@ package fuse
import (
"encoding/binary"
"restic/backend"
"restic"
)
// inodeFromBackendId returns a unique uint64 from a backend id.
// Endianness has no specific meaning, it is just the simplest way to
// transform a []byte to an uint64
func inodeFromBackendId(id backend.ID) uint64 {
func inodeFromBackendId(id restic.ID) uint64 {
return binary.BigEndian.Uint64(id[:8])
}

3
src/restic/fuse/snapshot.go
View File

@ -12,7 +12,6 @@ import (
"bazil.org/fuse/fs"
"restic"
"restic/backend"
"restic/debug"
"restic/repository"
@ -21,7 +20,7 @@ import (
type SnapshotWithId struct {
*restic.Snapshot
backend.ID
restic.ID
}
// These lines statically ensure that a *SnapshotsDir implement the given

11
src/restic/index/index.go
View File

@ -9,7 +9,6 @@ import (
"restic/debug"
"restic/list"
"restic/pack"
"restic/types"
"restic/worker"
"github.com/pkg/errors"
@ -43,7 +42,7 @@ func newIndex() *Index {
}
// New creates a new index for repo from scratch.
func New(repo types.Repository, p *restic.Progress) (*Index, error) {
func New(repo restic.Repository, p *restic.Progress) (*Index, error) {
done := make(chan struct{})
defer close(done)
@ -99,7 +98,7 @@ type indexJSON struct {
Packs []*packJSON `json:"packs"`
}
func loadIndexJSON(repo types.Repository, id backend.ID) (*indexJSON, error) {
func loadIndexJSON(repo restic.Repository, id backend.ID) (*indexJSON, error) {
debug.Log("index.loadIndexJSON", "process index %v\n", id.Str())
var idx indexJSON
@ -112,7 +111,7 @@ func loadIndexJSON(repo types.Repository, id backend.ID) (*indexJSON, error) {
}
// Load creates an index by loading all index files from the repo.
func Load(repo types.Repository, p *restic.Progress) (*Index, error) {
func Load(repo restic.Repository, p *restic.Progress) (*Index, error) {
debug.Log("index.Load", "loading indexes")
p.Start()
@ -300,7 +299,7 @@ func (idx *Index) FindBlob(h pack.Handle) ([]Location, error) {
}
// Save writes the complete index to the repo.
func (idx *Index) Save(repo types.Repository, supersedes backend.IDs) (backend.ID, error) {
func (idx *Index) Save(repo restic.Repository, supersedes backend.IDs) (backend.ID, error) {
packs := make(map[backend.ID][]pack.Blob, len(idx.Packs))
for id, p := range idx.Packs {
packs[id] = p.Entries
@ -310,7 +309,7 @@ func (idx *Index) Save(repo types.Repository, supersedes backend.IDs) (backend.I
}
// Save writes a new index containing the given packs.
func Save(repo types.Repository, packs map[backend.ID][]pack.Blob, supersedes backend.IDs) (backend.ID, error) {
func Save(repo restic.Repository, packs map[backend.ID][]pack.Blob, supersedes backend.IDs) (backend.ID, error) {
idx := &indexJSON{
Supersedes: supersedes,
Packs: make([]*packJSON, 0, len(packs)),

5
src/restic/repository.go
View File

@ -33,6 +33,11 @@ type Repository interface {
Flush() error
}
// Deleter removes all data stored in a backend/repo.
type Deleter interface {
Delete() error
}
// Lister allows listing files in a backend.
type Lister interface {
List(FileType, <-chan struct{}) <-chan string

3
src/restic/repository/repository.go
View File

@ -38,6 +38,7 @@ func New(be restic.Backend) *Repository {
return repo
}
// Config returns the repository configuration.
func (r *Repository) Config() restic.Config {
return r.cfg
}
@ -577,7 +578,7 @@ func (r *Repository) ListPack(id restic.ID) ([]restic.Blob, int64, error) {
// Delete calls backend.Delete() if implemented, and returns an error
// otherwise.
func (r *Repository) Delete() error {
if b, ok := r.be.(backend.Deleter); ok {
if b, ok := r.be.(restic.Deleter); ok {
return b.Delete()
}

8
src/restic/test/helpers.go
View File

@ -11,12 +11,12 @@ import (
"os/exec"
"path/filepath"
"reflect"
"restic"
"runtime"
"testing"
mrand "math/rand"
"restic/backend"
"restic/backend/local"
"restic/repository"
)
@ -63,9 +63,9 @@ func Equals(tb testing.TB, exp, act interface{}) {
}
}
// ParseID parses s as a backend.ID and panics if that fails.
func ParseID(s string) backend.ID {
id, err := backend.ParseID(s)
// ParseID parses s as a restic.ID and panics if that fails.
func ParseID(s string) restic.ID {
id, err := restic.ParseID(s)
if err != nil {
panic(err)
}

21
src/restic/types/repository.go
View File

@ -1,21 +0,0 @@
package types
import (
"restic"
"restic/backend"
"restic/pack"
)
// Repository manages encrypted and packed data stored in a backend.
type Repository interface {
LoadJSONUnpacked(restic.FileType, backend.ID, interface{}) error
SaveJSONUnpacked(restic.FileType, interface{}) (backend.ID, error)
Lister
}
// Lister combines lists packs in a repo and blobs in a pack.
type Lister interface {
List(restic.FileType, <-chan struct{}) <-chan backend.ID
ListPack(backend.ID) ([]pack.Blob, int64, error)
}