octoleo/restic
2
2
Fork 0
mirror of https://github.com/octoleo/restic.git synced 2024-11-26 14:56:29 +00:00
Code Issues Releases Wiki Activity

Add repository.SaveTree

Browse Source
This commit is contained in:
Alexander Neumann 2016-09-03 20:55:22 +02:00
parent 1fb80bf0e2
commit b5b3c0eaf8
7 changed files with 61 additions and 129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
src/restic/archiver/archive_reader.go
View File

@ -1,7 +1,6 @@
package archiver
import (
"encoding/json"
"io"
"restic"
"restic/debug"
@ -12,23 +11,6 @@ import (
"github.com/restic/chunker"
)
// saveTreeJSON stores a tree in the repository.
func saveTreeJSON(repo restic.Repository, item interface{}) (restic.ID, error) {
data, err := json.Marshal(item)
if err != nil {
return restic.ID{}, errors.Wrap(err, "")
}
data = append(data, '\n')
// check if tree has been saved before
id := restic.Hash(data)
if repo.Index().Has(id, restic.TreeBlob) {
return id, nil
}
return repo.SaveJSON(restic.TreeBlob, item)
}
// ArchiveReader reads from the reader and archives the data. Returned is the
// resulting snapshot and its ID.
func ArchiveReader(repo restic.Repository, p *restic.Progress, rd io.Reader, name string) (*restic.Snapshot, restic.ID, error) {
@ -93,7 +75,7 @@ func ArchiveReader(repo restic.Repository, p *restic.Progress, rd io.Reader, nam
},
}
treeID, err := saveTreeJSON(repo, tree)
treeID, err := repo.SaveTree(tree)
if err != nil {
return nil, restic.ID{}, err
}

2
src/restic/archiver/archiver.go
View File

@ -122,7 +122,7 @@ func (arch *Archiver) SaveTreeJSON(item interface{}) (restic.ID, error) {
return id, nil
}
return arch.repo.SaveJSON(restic.TreeBlob, item)
return arch.repo.SaveBlob(restic.TreeBlob, data, id)
}
func (arch *Archiver) reloadFileIfChanged(node *restic.Node, file fs.File) (*restic.Node, error) {

6
src/restic/repository.go
View File

@ -27,17 +27,17 @@ type Repository interface {
Flush() error
SaveJSON(BlobType, interface{}) (ID, error)
SaveUnpacked(FileType, []byte) (ID, error)
SaveJSONUnpacked(FileType, interface{}) (ID, error)
LoadJSONUnpacked(FileType, ID, interface{}) error
LoadAndDecrypt(FileType, ID) ([]byte, error)
LoadTree(ID) (*Tree, error)
LoadBlob(BlobType, ID, []byte) (int, error)
SaveBlob(BlobType, []byte, ID) (ID, error)
LoadTree(ID) (*Tree, error)
SaveTree(t *Tree) (ID, error)
}
// Deleter removes all data stored in a backend/repo.

95
src/restic/repository/repository.go
View File

@ -227,25 +227,6 @@ func (r *Repository) SaveAndEncrypt(t restic.BlobType, data []byte, id *restic.I
return *id, r.savePacker(packer)
}
// SaveJSON serialises item as JSON and encrypts and saves it in a pack in the
// backend as type t.
func (r *Repository) SaveJSON(t restic.BlobType, item interface{}) (restic.ID, error) {
debug.Log("Repo.SaveJSON", "save %v blob", t)
buf := getBuf()[:0]
defer freeBuf(buf)
wr := bytes.NewBuffer(buf)
enc := json.NewEncoder(wr)
err := enc.Encode(item)
if err != nil {
return restic.ID{}, errors.Errorf("json.Encode: %v", err)
}
buf = wr.Bytes()
return r.SaveAndEncrypt(t, buf, nil)
}
// SaveJSONUnpacked serialises item as JSON and encrypts and saves it in the
// backend as type t, without a pack. It returns the storage hash.
func (r *Repository) SaveJSONUnpacked(t restic.FileType, item interface{}) (restic.ID, error) {
@ -565,33 +546,6 @@ func (r *Repository) Close() error {
return r.be.Close()
}
// LoadTree loads a tree from the repository.
func (r *Repository) LoadTree(id restic.ID) (*restic.Tree, error) {
debug.Log("repo.LoadTree", "load tree %v", id.Str())
size, err := r.idx.LookupSize(id, restic.TreeBlob)
if err != nil {
return nil, err
}
debug.Log("repo.LoadTree", "size is %d, create buffer", size)
buf := make([]byte, size)
n, err := r.loadBlob(id, restic.TreeBlob, buf)
if err != nil {
return nil, err
}
buf = buf[:n]
t := &restic.Tree{}
err = json.Unmarshal(buf, t)
if err != nil {
return nil, err
}
return t, nil
}
// LoadBlob loads a blob of type t from the repository to the buffer.
func (r *Repository) LoadBlob(t restic.BlobType, id restic.ID, buf []byte) (int, error) {
debug.Log("repo.LoadBlob", "load blob %v into buf %p", id.Str(), buf)
@ -624,3 +578,52 @@ func (r *Repository) SaveBlob(t restic.BlobType, buf []byte, id restic.ID) (rest
}
return r.SaveAndEncrypt(t, buf, i)
}
// LoadTree loads a tree from the repository.
func (r *Repository) LoadTree(id restic.ID) (*restic.Tree, error) {
debug.Log("repo.LoadTree", "load tree %v", id.Str())
size, err := r.idx.LookupSize(id, restic.TreeBlob)
if err != nil {
return nil, err
}
debug.Log("repo.LoadTree", "size is %d, create buffer", size)
buf := make([]byte, size)
n, err := r.loadBlob(id, restic.TreeBlob, buf)
if err != nil {
return nil, err
}
buf = buf[:n]
t := &restic.Tree{}
err = json.Unmarshal(buf, t)
if err != nil {
return nil, err
}
return t, nil
}
// SaveTree stores a tree into the repository and returns the ID. The ID is
// checked against the index. The tree is only stored when the index does not
// contain the ID.
func (r *Repository) SaveTree(t *restic.Tree) (restic.ID, error) {
buf, err := json.Marshal(t)
if err != nil {
return restic.ID{}, errors.Wrap(err, "MarshalJSON")
}
// append a newline so that the data is always consistent (json.Encoder
// adds a newline after each object)
buf = append(buf, '\n')
id := restic.Hash(buf)
if r.idx.Has(id, restic.TreeBlob) {
return id, nil
}
_, err = r.SaveBlob(restic.TreeBlob, buf, id)
return id, err
}

53
src/restic/repository/repository_test.go
View File

@ -4,7 +4,6 @@ import (
"bytes"
"crypto/rand"
"crypto/sha256"
"encoding/json"
"io"
mrand "math/rand"
"path/filepath"
@ -15,58 +14,6 @@ import (
. "restic/test"
)
type testJSONStruct struct {
Foo uint32
Bar string
Baz []byte
}
var repoTests = []testJSONStruct{
testJSONStruct{Foo: 23, Bar: "Teststring", Baz: []byte("xx")},
}
func TestSaveJSON(t *testing.T) {
repo := SetupRepo()
defer TeardownRepo(repo)
for _, obj := range repoTests {
data, err := json.Marshal(obj)
OK(t, err)
data = append(data, '\n')
h := sha256.Sum256(data)
id, err := repo.SaveJSON(restic.TreeBlob, obj)
OK(t, err)
Assert(t, h == id,
"TestSaveJSON: wrong plaintext ID: expected %02x, got %02x",
h, id)
}
}
func BenchmarkSaveJSON(t *testing.B) {
repo := SetupRepo()
defer TeardownRepo(repo)
obj := repoTests[0]
data, err := json.Marshal(obj)
OK(t, err)
data = append(data, '\n')
h := sha256.Sum256(data)
t.ResetTimer()
for i := 0; i < t.N; i++ {
id, err := repo.SaveJSON(restic.TreeBlob, obj)
OK(t, err)
Assert(t, h == id,
"TestSaveJSON: wrong plaintext ID: expected %02x, got %02x",
h, id)
}
}
var testSizes = []int{5, 23, 2<<18 + 23, 1 << 20}
func TestSave(t *testing.T) {

12
src/restic/testing.go
View File

@ -64,17 +64,16 @@ const (
maxNodes = 32
)
func (fs fakeFileSystem) treeIsKnown(tree *Tree) (bool, ID) {
func (fs fakeFileSystem) treeIsKnown(tree *Tree) (bool, []byte, ID) {
data, err := json.Marshal(tree)
if err != nil {
fs.t.Fatalf("json.Marshal(tree) returned error: %v", err)
return false, ID{}
return false, nil, ID{}
}
data = append(data, '\n')
id := Hash(data)
return fs.blobIsKnown(id, TreeBlob), id
return fs.blobIsKnown(id, TreeBlob), data, id
}
func (fs fakeFileSystem) blobIsKnown(id ID, t BlobType) bool {
@ -132,11 +131,12 @@ func (fs fakeFileSystem) saveTree(seed int64, depth int) ID {
tree.Nodes = append(tree.Nodes, node)
}
if known, id := fs.treeIsKnown(&tree); known {
known, buf, id := fs.treeIsKnown(&tree)
if known {
return id
}
id, err := fs.repo.SaveJSON(TreeBlob, tree)
_, err := fs.repo.SaveBlob(TreeBlob, buf, id)
if err != nil {
fs.t.Fatal(err)
}

2
src/restic/tree_test.go
View File

@ -97,7 +97,7 @@ func TestLoadTree(t *testing.T) {
// save tree
tree := restic.NewTree()
id, err := repo.SaveJSON(restic.TreeBlob, tree)
id, err := repo.SaveTree(tree)
OK(t, err)
// save packs