Merge branch 'externalize-map-63'

Close #63

archiver.go

@@ -85,7 +85,7 @@ func NewArchiver(be backend.Server, key *Key) (*Archiver, error) {
 	}
 
 	// load all blobs from all snapshots
-	err = arch.ch.LoadAllSnapshots()
+	err = arch.ch.LoadAllMaps()
 	if err != nil {
 		return nil, err
 	}
@@ -371,11 +371,16 @@ func (arch *Archiver) Snapshot(dir string, t *Tree) (*Snapshot, backend.ID, erro
 	if err != nil {
 		return nil, nil, err
 	}
-
 	sn.Content = blob.ID
 
+	// save bloblist
+	blob, err = arch.SaveJSON(backend.Map, arch.bl)
+	if err != nil {
+		return nil, nil, err
+	}
+	sn.Map = blob.Storage
+
 	// save snapshot
-	sn.BlobList = arch.bl
 	blob, err = arch.SaveJSON(backend.Snapshot, sn)
 	if err != nil {
 		return nil, nil, err

backend/interface.go

@@ -10,6 +10,7 @@ const (
 	Lock          = "lock"
 	Snapshot      = "snapshot"
 	Tree          = "tree"
+	Map           = "map"
 )
 
 const (

backend/local.go

@@ -17,6 +17,7 @@ const (
 	dataPath        = "data"
 	snapshotPath    = "snapshots"
 	treePath        = "trees"
+	mapPath         = "maps"
 	lockPath        = "locks"
 	keyPath         = "keys"
 	tempPath        = "tmp"
@@ -86,6 +87,7 @@ func CreateLocal(dir string) (*Local, error) {
 		filepath.Join(dir, dataPath),
 		filepath.Join(dir, snapshotPath),
 		filepath.Join(dir, treePath),
+		filepath.Join(dir, mapPath),
 		filepath.Join(dir, lockPath),
 		filepath.Join(dir, keyPath),
 		filepath.Join(dir, tempPath),
@@ -158,6 +160,8 @@ func (b *Local) dir(t Type) string {
 		n = snapshotPath
 	case Tree:
 		n = treePath
+	case Map:
+		n = mapPath
 	case Lock:
 		n = lockPath
 	case Key:

backend/local_test.go

@@ -45,7 +45,7 @@ func teardownBackend(t *testing.T, b *backend.Local) {
 }
 
 func testBackend(b backend.Server, t *testing.T) {
-	for _, tpe := range []backend.Type{backend.Data, backend.Key, backend.Lock, backend.Snapshot, backend.Tree} {
+	for _, tpe := range []backend.Type{backend.Data, backend.Key, backend.Lock, backend.Snapshot, backend.Tree, backend.Map} {
 		// detect non-existing files
 		for _, test := range TestStrings {
 			id, err := backend.ParseID(test.id)

backend/sftp.go

@@ -238,6 +238,8 @@ func (r *SFTP) dir(t Type) string {
 		n = snapshotPath
 	case Tree:
 		n = treePath
+	case Map:
+		n = mapPath
 	case Lock:
 		n = lockPath
 	case Key:

bloblist.go

@@ -6,6 +6,8 @@ import (
 	"errors"
 	"sort"
 	"sync"
+
+	"github.com/fd0/khepri/backend"
 )
 
 type BlobList struct {
@@ -21,6 +23,16 @@ func NewBlobList() *BlobList {
 	}
 }
 
+func LoadBlobList(ch *ContentHandler, id backend.ID) (*BlobList, error) {
+	bl := &BlobList{}
+	err := ch.LoadJSONRaw(backend.Map, id, bl)
+	if err != nil {
+		return nil, err
+	}
+
+	return bl, nil
+}
+
 func (bl *BlobList) find(blob Blob) (int, Blob, error) {
 	pos := sort.Search(len(bl.list), func(i int) bool {
 		return blob.ID.Compare(bl.list[i].ID) >= 0

cmd/khepri/cmd_cat.go

@@ -27,7 +27,7 @@ func commandCat(be backend.Server, key *khepri.Key, args []string) error {
 		return err
 	}
 
-	err = ch.LoadAllSnapshots()
+	err = ch.LoadAllMaps()
 	if err != nil {
 		return err
 	}
@@ -87,6 +87,21 @@ func commandCat(be backend.Server, key *khepri.Key, args []string) error {
 
 		fmt.Println(string(buf))
 
+		return nil
+	case "map":
+		var bl khepri.BlobList
+		err := ch.LoadJSONRaw(backend.Map, id, &bl)
+		if err != nil {
+			return err
+		}
+
+		buf, err := json.MarshalIndent(&bl, "", "  ")
+		if err != nil {
+			return err
+		}
+
+		fmt.Println(string(buf))
+
 		return nil
 	case "snapshot":
 		var sn khepri.Snapshot

contenthandler.go

@@ -33,22 +33,27 @@ func (ch *ContentHandler) LoadSnapshot(id backend.ID) (*Snapshot, error) {
 		return nil, err
 	}
 
-	ch.bl.Merge(sn.BlobList)
+	sn.bl, err = LoadBlobList(ch, sn.Map)
+	if err != nil {
+		return nil, err
+	}
+
+	ch.bl.Merge(sn.bl)
 
 	return sn, nil
 }
 
-// LoadAllSnapshots adds all blobs from all snapshots that can be decrypted
+// LoadAllMaps adds all blobs from all snapshots that can be decrypted
 // into the content handler.
-func (ch *ContentHandler) LoadAllSnapshots() error {
+func (ch *ContentHandler) LoadAllMaps() error {
 	// add all maps from all snapshots that can be decrypted to the storage map
-	err := backend.EachID(ch.be, backend.Snapshot, func(id backend.ID) {
+	err := backend.EachID(ch.be, backend.Map, func(id backend.ID) {
-		sn, err := LoadSnapshot(ch, id)
+		bl, err := LoadBlobList(ch, id)
 		if err != nil {
 			return
 		}
 
-		ch.bl.Merge(sn.BlobList)
+		ch.bl.Merge(bl)
 	})
 	if err != nil {
 		return err

snapshot.go

@@ -13,7 +13,7 @@ import (
 type Snapshot struct {
 	Time     time.Time  `json:"time"`
 	Content  backend.ID `json:"content"`
-	BlobList *BlobList  `json:"blobs"`
+	Map      backend.ID `json:"map"`
 	Dir      string     `json:"dir"`
 	Hostname string     `json:"hostname,omitempty"`
 	Username string     `json:"username,omitempty"`
@@ -21,6 +21,7 @@ type Snapshot struct {
 	GID      string     `json:"gid,omitempty"`
 
 	id backend.ID // plaintext ID, used during restore
+	bl *BlobList
 }
 
 func NewSnapshot(dir string) *Snapshot {