Merge branch 'externalize-map-63'

Close #63
This commit is contained in:
Alexander Neumann 2014-11-23 22:33:23 +01:00
commit 2d8dc7b695
9 changed files with 57 additions and 12 deletions

View File

@ -85,7 +85,7 @@ func NewArchiver(be backend.Server, key *Key) (*Archiver, error) {
}
// load all blobs from all snapshots
err = arch.ch.LoadAllSnapshots()
err = arch.ch.LoadAllMaps()
if err != nil {
return nil, err
}
@ -371,11 +371,16 @@ func (arch *Archiver) Snapshot(dir string, t *Tree) (*Snapshot, backend.ID, erro
if err != nil {
return nil, nil, err
}
sn.Content = blob.ID
// save bloblist
blob, err = arch.SaveJSON(backend.Map, arch.bl)
if err != nil {
return nil, nil, err
}
sn.Map = blob.Storage
// save snapshot
sn.BlobList = arch.bl
blob, err = arch.SaveJSON(backend.Snapshot, sn)
if err != nil {
return nil, nil, err

View File

@ -10,6 +10,7 @@ const (
Lock = "lock"
Snapshot = "snapshot"
Tree = "tree"
Map = "map"
)
const (

View File

@ -17,6 +17,7 @@ const (
dataPath = "data"
snapshotPath = "snapshots"
treePath = "trees"
mapPath = "maps"
lockPath = "locks"
keyPath = "keys"
tempPath = "tmp"
@ -86,6 +87,7 @@ func CreateLocal(dir string) (*Local, error) {
filepath.Join(dir, dataPath),
filepath.Join(dir, snapshotPath),
filepath.Join(dir, treePath),
filepath.Join(dir, mapPath),
filepath.Join(dir, lockPath),
filepath.Join(dir, keyPath),
filepath.Join(dir, tempPath),
@ -158,6 +160,8 @@ func (b *Local) dir(t Type) string {
n = snapshotPath
case Tree:
n = treePath
case Map:
n = mapPath
case Lock:
n = lockPath
case Key:

View File

@ -45,7 +45,7 @@ func teardownBackend(t *testing.T, b *backend.Local) {
}
func testBackend(b backend.Server, t *testing.T) {
for _, tpe := range []backend.Type{backend.Data, backend.Key, backend.Lock, backend.Snapshot, backend.Tree} {
for _, tpe := range []backend.Type{backend.Data, backend.Key, backend.Lock, backend.Snapshot, backend.Tree, backend.Map} {
// detect non-existing files
for _, test := range TestStrings {
id, err := backend.ParseID(test.id)

View File

@ -238,6 +238,8 @@ func (r *SFTP) dir(t Type) string {
n = snapshotPath
case Tree:
n = treePath
case Map:
n = mapPath
case Lock:
n = lockPath
case Key:

View File

@ -6,6 +6,8 @@ import (
"errors"
"sort"
"sync"
"github.com/fd0/khepri/backend"
)
type BlobList struct {
@ -21,6 +23,16 @@ func NewBlobList() *BlobList {
}
}
func LoadBlobList(ch *ContentHandler, id backend.ID) (*BlobList, error) {
bl := &BlobList{}
err := ch.LoadJSONRaw(backend.Map, id, bl)
if err != nil {
return nil, err
}
return bl, nil
}
func (bl *BlobList) find(blob Blob) (int, Blob, error) {
pos := sort.Search(len(bl.list), func(i int) bool {
return blob.ID.Compare(bl.list[i].ID) >= 0

View File

@ -27,7 +27,7 @@ func commandCat(be backend.Server, key *khepri.Key, args []string) error {
return err
}
err = ch.LoadAllSnapshots()
err = ch.LoadAllMaps()
if err != nil {
return err
}
@ -87,6 +87,21 @@ func commandCat(be backend.Server, key *khepri.Key, args []string) error {
fmt.Println(string(buf))
return nil
case "map":
var bl khepri.BlobList
err := ch.LoadJSONRaw(backend.Map, id, &bl)
if err != nil {
return err
}
buf, err := json.MarshalIndent(&bl, "", " ")
if err != nil {
return err
}
fmt.Println(string(buf))
return nil
case "snapshot":
var sn khepri.Snapshot

View File

@ -33,22 +33,27 @@ func (ch *ContentHandler) LoadSnapshot(id backend.ID) (*Snapshot, error) {
return nil, err
}
ch.bl.Merge(sn.BlobList)
sn.bl, err = LoadBlobList(ch, sn.Map)
if err != nil {
return nil, err
}
ch.bl.Merge(sn.bl)
return sn, nil
}
// LoadAllSnapshots adds all blobs from all snapshots that can be decrypted
// LoadAllMaps adds all blobs from all snapshots that can be decrypted
// into the content handler.
func (ch *ContentHandler) LoadAllSnapshots() error {
func (ch *ContentHandler) LoadAllMaps() error {
// add all maps from all snapshots that can be decrypted to the storage map
err := backend.EachID(ch.be, backend.Snapshot, func(id backend.ID) {
sn, err := LoadSnapshot(ch, id)
err := backend.EachID(ch.be, backend.Map, func(id backend.ID) {
bl, err := LoadBlobList(ch, id)
if err != nil {
return
}
ch.bl.Merge(sn.BlobList)
ch.bl.Merge(bl)
})
if err != nil {
return err

View File

@ -13,7 +13,7 @@ import (
type Snapshot struct {
Time time.Time `json:"time"`
Content backend.ID `json:"content"`
BlobList *BlobList `json:"blobs"`
Map backend.ID `json:"map"`
Dir string `json:"dir"`
Hostname string `json:"hostname,omitempty"`
Username string `json:"username,omitempty"`
@ -21,6 +21,7 @@ type Snapshot struct {
GID string `json:"gid,omitempty"`
id backend.ID // plaintext ID, used during restore
bl *BlobList
}
func NewSnapshot(dir string) *Snapshot {