mirror of
https://github.com/octoleo/restic.git
synced 2024-12-23 03:18:55 +00:00
Add repository.SaveTree
This commit is contained in:
parent
1fb80bf0e2
commit
b5b3c0eaf8
@ -1,7 +1,6 @@
|
||||
package archiver
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"restic"
|
||||
"restic/debug"
|
||||
@ -12,23 +11,6 @@ import (
|
||||
"github.com/restic/chunker"
|
||||
)
|
||||
|
||||
// saveTreeJSON stores a tree in the repository.
|
||||
func saveTreeJSON(repo restic.Repository, item interface{}) (restic.ID, error) {
|
||||
data, err := json.Marshal(item)
|
||||
if err != nil {
|
||||
return restic.ID{}, errors.Wrap(err, "")
|
||||
}
|
||||
data = append(data, '\n')
|
||||
|
||||
// check if tree has been saved before
|
||||
id := restic.Hash(data)
|
||||
if repo.Index().Has(id, restic.TreeBlob) {
|
||||
return id, nil
|
||||
}
|
||||
|
||||
return repo.SaveJSON(restic.TreeBlob, item)
|
||||
}
|
||||
|
||||
// ArchiveReader reads from the reader and archives the data. Returned is the
|
||||
// resulting snapshot and its ID.
|
||||
func ArchiveReader(repo restic.Repository, p *restic.Progress, rd io.Reader, name string) (*restic.Snapshot, restic.ID, error) {
|
||||
@ -93,7 +75,7 @@ func ArchiveReader(repo restic.Repository, p *restic.Progress, rd io.Reader, nam
|
||||
},
|
||||
}
|
||||
|
||||
treeID, err := saveTreeJSON(repo, tree)
|
||||
treeID, err := repo.SaveTree(tree)
|
||||
if err != nil {
|
||||
return nil, restic.ID{}, err
|
||||
}
|
||||
|
@ -122,7 +122,7 @@ func (arch *Archiver) SaveTreeJSON(item interface{}) (restic.ID, error) {
|
||||
return id, nil
|
||||
}
|
||||
|
||||
return arch.repo.SaveJSON(restic.TreeBlob, item)
|
||||
return arch.repo.SaveBlob(restic.TreeBlob, data, id)
|
||||
}
|
||||
|
||||
func (arch *Archiver) reloadFileIfChanged(node *restic.Node, file fs.File) (*restic.Node, error) {
|
||||
|
@ -27,17 +27,17 @@ type Repository interface {
|
||||
|
||||
Flush() error
|
||||
|
||||
SaveJSON(BlobType, interface{}) (ID, error)
|
||||
SaveUnpacked(FileType, []byte) (ID, error)
|
||||
SaveJSONUnpacked(FileType, interface{}) (ID, error)
|
||||
|
||||
LoadJSONUnpacked(FileType, ID, interface{}) error
|
||||
LoadAndDecrypt(FileType, ID) ([]byte, error)
|
||||
|
||||
LoadTree(ID) (*Tree, error)
|
||||
LoadBlob(BlobType, ID, []byte) (int, error)
|
||||
|
||||
SaveBlob(BlobType, []byte, ID) (ID, error)
|
||||
|
||||
LoadTree(ID) (*Tree, error)
|
||||
SaveTree(t *Tree) (ID, error)
|
||||
}
|
||||
|
||||
// Deleter removes all data stored in a backend/repo.
|
||||
|
@ -227,25 +227,6 @@ func (r *Repository) SaveAndEncrypt(t restic.BlobType, data []byte, id *restic.I
|
||||
return *id, r.savePacker(packer)
|
||||
}
|
||||
|
||||
// SaveJSON serialises item as JSON and encrypts and saves it in a pack in the
|
||||
// backend as type t.
|
||||
func (r *Repository) SaveJSON(t restic.BlobType, item interface{}) (restic.ID, error) {
|
||||
debug.Log("Repo.SaveJSON", "save %v blob", t)
|
||||
buf := getBuf()[:0]
|
||||
defer freeBuf(buf)
|
||||
|
||||
wr := bytes.NewBuffer(buf)
|
||||
|
||||
enc := json.NewEncoder(wr)
|
||||
err := enc.Encode(item)
|
||||
if err != nil {
|
||||
return restic.ID{}, errors.Errorf("json.Encode: %v", err)
|
||||
}
|
||||
|
||||
buf = wr.Bytes()
|
||||
return r.SaveAndEncrypt(t, buf, nil)
|
||||
}
|
||||
|
||||
// SaveJSONUnpacked serialises item as JSON and encrypts and saves it in the
|
||||
// backend as type t, without a pack. It returns the storage hash.
|
||||
func (r *Repository) SaveJSONUnpacked(t restic.FileType, item interface{}) (restic.ID, error) {
|
||||
@ -565,33 +546,6 @@ func (r *Repository) Close() error {
|
||||
return r.be.Close()
|
||||
}
|
||||
|
||||
// LoadTree loads a tree from the repository.
|
||||
func (r *Repository) LoadTree(id restic.ID) (*restic.Tree, error) {
|
||||
debug.Log("repo.LoadTree", "load tree %v", id.Str())
|
||||
|
||||
size, err := r.idx.LookupSize(id, restic.TreeBlob)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
debug.Log("repo.LoadTree", "size is %d, create buffer", size)
|
||||
buf := make([]byte, size)
|
||||
|
||||
n, err := r.loadBlob(id, restic.TreeBlob, buf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
buf = buf[:n]
|
||||
|
||||
t := &restic.Tree{}
|
||||
err = json.Unmarshal(buf, t)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// LoadBlob loads a blob of type t from the repository to the buffer.
|
||||
func (r *Repository) LoadBlob(t restic.BlobType, id restic.ID, buf []byte) (int, error) {
|
||||
debug.Log("repo.LoadBlob", "load blob %v into buf %p", id.Str(), buf)
|
||||
@ -624,3 +578,52 @@ func (r *Repository) SaveBlob(t restic.BlobType, buf []byte, id restic.ID) (rest
|
||||
}
|
||||
return r.SaveAndEncrypt(t, buf, i)
|
||||
}
|
||||
|
||||
// LoadTree loads a tree from the repository.
|
||||
func (r *Repository) LoadTree(id restic.ID) (*restic.Tree, error) {
|
||||
debug.Log("repo.LoadTree", "load tree %v", id.Str())
|
||||
|
||||
size, err := r.idx.LookupSize(id, restic.TreeBlob)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
debug.Log("repo.LoadTree", "size is %d, create buffer", size)
|
||||
buf := make([]byte, size)
|
||||
|
||||
n, err := r.loadBlob(id, restic.TreeBlob, buf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
buf = buf[:n]
|
||||
|
||||
t := &restic.Tree{}
|
||||
err = json.Unmarshal(buf, t)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// SaveTree stores a tree into the repository and returns the ID. The ID is
|
||||
// checked against the index. The tree is only stored when the index does not
|
||||
// contain the ID.
|
||||
func (r *Repository) SaveTree(t *restic.Tree) (restic.ID, error) {
|
||||
buf, err := json.Marshal(t)
|
||||
if err != nil {
|
||||
return restic.ID{}, errors.Wrap(err, "MarshalJSON")
|
||||
}
|
||||
|
||||
// append a newline so that the data is always consistent (json.Encoder
|
||||
// adds a newline after each object)
|
||||
buf = append(buf, '\n')
|
||||
|
||||
id := restic.Hash(buf)
|
||||
if r.idx.Has(id, restic.TreeBlob) {
|
||||
return id, nil
|
||||
}
|
||||
|
||||
_, err = r.SaveBlob(restic.TreeBlob, buf, id)
|
||||
return id, err
|
||||
}
|
||||
|
@ -4,7 +4,6 @@ import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"io"
|
||||
mrand "math/rand"
|
||||
"path/filepath"
|
||||
@ -15,58 +14,6 @@ import (
|
||||
. "restic/test"
|
||||
)
|
||||
|
||||
type testJSONStruct struct {
|
||||
Foo uint32
|
||||
Bar string
|
||||
Baz []byte
|
||||
}
|
||||
|
||||
var repoTests = []testJSONStruct{
|
||||
testJSONStruct{Foo: 23, Bar: "Teststring", Baz: []byte("xx")},
|
||||
}
|
||||
|
||||
func TestSaveJSON(t *testing.T) {
|
||||
repo := SetupRepo()
|
||||
defer TeardownRepo(repo)
|
||||
|
||||
for _, obj := range repoTests {
|
||||
data, err := json.Marshal(obj)
|
||||
OK(t, err)
|
||||
data = append(data, '\n')
|
||||
h := sha256.Sum256(data)
|
||||
|
||||
id, err := repo.SaveJSON(restic.TreeBlob, obj)
|
||||
OK(t, err)
|
||||
|
||||
Assert(t, h == id,
|
||||
"TestSaveJSON: wrong plaintext ID: expected %02x, got %02x",
|
||||
h, id)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkSaveJSON(t *testing.B) {
|
||||
repo := SetupRepo()
|
||||
defer TeardownRepo(repo)
|
||||
|
||||
obj := repoTests[0]
|
||||
|
||||
data, err := json.Marshal(obj)
|
||||
OK(t, err)
|
||||
data = append(data, '\n')
|
||||
h := sha256.Sum256(data)
|
||||
|
||||
t.ResetTimer()
|
||||
|
||||
for i := 0; i < t.N; i++ {
|
||||
id, err := repo.SaveJSON(restic.TreeBlob, obj)
|
||||
OK(t, err)
|
||||
|
||||
Assert(t, h == id,
|
||||
"TestSaveJSON: wrong plaintext ID: expected %02x, got %02x",
|
||||
h, id)
|
||||
}
|
||||
}
|
||||
|
||||
var testSizes = []int{5, 23, 2<<18 + 23, 1 << 20}
|
||||
|
||||
func TestSave(t *testing.T) {
|
||||
|
@ -64,17 +64,16 @@ const (
|
||||
maxNodes = 32
|
||||
)
|
||||
|
||||
func (fs fakeFileSystem) treeIsKnown(tree *Tree) (bool, ID) {
|
||||
func (fs fakeFileSystem) treeIsKnown(tree *Tree) (bool, []byte, ID) {
|
||||
data, err := json.Marshal(tree)
|
||||
if err != nil {
|
||||
fs.t.Fatalf("json.Marshal(tree) returned error: %v", err)
|
||||
return false, ID{}
|
||||
return false, nil, ID{}
|
||||
}
|
||||
data = append(data, '\n')
|
||||
|
||||
id := Hash(data)
|
||||
return fs.blobIsKnown(id, TreeBlob), id
|
||||
|
||||
return fs.blobIsKnown(id, TreeBlob), data, id
|
||||
}
|
||||
|
||||
func (fs fakeFileSystem) blobIsKnown(id ID, t BlobType) bool {
|
||||
@ -132,11 +131,12 @@ func (fs fakeFileSystem) saveTree(seed int64, depth int) ID {
|
||||
tree.Nodes = append(tree.Nodes, node)
|
||||
}
|
||||
|
||||
if known, id := fs.treeIsKnown(&tree); known {
|
||||
known, buf, id := fs.treeIsKnown(&tree)
|
||||
if known {
|
||||
return id
|
||||
}
|
||||
|
||||
id, err := fs.repo.SaveJSON(TreeBlob, tree)
|
||||
_, err := fs.repo.SaveBlob(TreeBlob, buf, id)
|
||||
if err != nil {
|
||||
fs.t.Fatal(err)
|
||||
}
|
||||
|
@ -97,7 +97,7 @@ func TestLoadTree(t *testing.T) {
|
||||
|
||||
// save tree
|
||||
tree := restic.NewTree()
|
||||
id, err := repo.SaveJSON(restic.TreeBlob, tree)
|
||||
id, err := repo.SaveTree(tree)
|
||||
OK(t, err)
|
||||
|
||||
// save packs
|
||||
|
Loading…
Reference in New Issue
Block a user