Scan directory first, then backup. Add stats.

archiver.go

@@ -13,8 +13,20 @@ type Archiver struct {
 	ch   *ContentHandler
 	smap *StorageMap // blobs used for the current snapshot
 
+	Stats Stats
+
 	Error  func(dir string, fi os.FileInfo, err error) error
 	Filter func(item string, fi os.FileInfo) bool
+
+	ScannerUpdate func(stats Stats)
+	SaveUpdate    func(stats Stats)
+}
+
+type Stats struct {
+	Files       int
+	Directories int
+	Other       int
+	Bytes       uint64
 }
 
 func NewArchiver(be backend.Server, key *Key) (*Archiver, error) {
@@ -25,6 +37,9 @@ func NewArchiver(be backend.Server, key *Key) (*Archiver, error) {
 	arch.Error = func(string, os.FileInfo, error) error { return err }
 	// allow all files
 	arch.Filter = func(string, os.FileInfo) bool { return true }
+	// do nothing
+	arch.ScannerUpdate = func(Stats) {}
+	arch.SaveUpdate = func(Stats) {}
 
 	arch.smap = NewStorageMap()
 	arch.ch, err = NewContentHandler(be, key)
@@ -65,10 +80,10 @@ func (arch *Archiver) SaveJSON(t backend.Type, item interface{}) (*Blob, error)
 	return blob, nil
 }
 
-func (arch *Archiver) SaveFile(node *Node) (Blobs, error) {
+func (arch *Archiver) SaveFile(node *Node) error {
 	blobs, err := arch.ch.SaveFile(node.path, uint(node.Size))
 	if err != nil {
-		return nil, arch.Error(node.path, nil, err)
+		return arch.Error(node.path, nil, err)
 	}
 
 	node.Content = make([]backend.ID, len(blobs))
@@ -77,23 +92,20 @@ func (arch *Archiver) SaveFile(node *Node) (Blobs, error) {
 		arch.smap.Insert(blob)
 	}
 
-	return blobs, err
+	return err
 }
 
-func (arch *Archiver) ImportDir(dir string) (Tree, error) {
+func (arch *Archiver) loadTree(dir string) (*Tree, error) {
+	// open and list path
 	fd, err := os.Open(dir)
 	defer fd.Close()
 	if err != nil {
-		return nil, arch.Error(dir, nil, err)
+		return nil, err
 	}
 
 	entries, err := fd.Readdir(-1)
 	if err != nil {
-		return nil, arch.Error(dir, nil, err)
+		return nil, err
-	}
-
-	if len(entries) == 0 {
-		return nil, nil
 	}
 
 	tree := Tree{}
@@ -107,71 +119,97 @@ func (arch *Archiver) ImportDir(dir string) (Tree, error) {
 
 		node, err := NodeFromFileInfo(path, entry)
 		if err != nil {
-			return nil, arch.Error(dir, entry, err)
+			// TODO: error processing
+			return nil, err
 		}
 
 		tree = append(tree, node)
 
 		if entry.IsDir() {
-			subtree, err := arch.ImportDir(path)
+			node.Tree, err = arch.loadTree(path)
 			if err != nil {
 				return nil, err
 			}
-
-			blob, err := arch.SaveJSON(backend.Tree, subtree)
-			if err != nil {
-				return nil, err
-			}
-
-			node.Subtree = blob.ID
-
-			continue
 		}
 
-		if node.Type == "file" {
+		switch node.Type {
-			_, err := arch.SaveFile(node)
+		case "file":
-			if err != nil {
+			arch.Stats.Files++
-				return nil, arch.Error(path, entry, err)
+			arch.Stats.Bytes += node.Size
-			}
+		case "dir":
+			arch.Stats.Directories++
+		default:
+			arch.Stats.Other++
 		}
 	}
 
-	return tree, nil
+	arch.ScannerUpdate(arch.Stats)
+
+	return &tree, nil
 }
 
-func (arch *Archiver) Import(dir string) (*Snapshot, *Blob, error) {
+func (arch *Archiver) LoadTree(path string) (*Tree, error) {
+	fi, err := os.Lstat(path)
+	if err != nil {
+		return nil, err
+	}
+
+	node, err := NodeFromFileInfo(path, fi)
+	if err != nil {
+		return nil, err
+	}
+
+	if node.Type != "dir" {
+		arch.Stats.Files = 1
+		arch.Stats.Bytes = node.Size
+		arch.ScannerUpdate(arch.Stats)
+		return &Tree{node}, nil
+	}
+
+	arch.Stats.Directories = 1
+	node.Tree, err = arch.loadTree(path)
+	if err != nil {
+		return nil, err
+	}
+
+	arch.ScannerUpdate(arch.Stats)
+
+	return &Tree{node}, nil
+}
+
+func (arch *Archiver) saveTree(t *Tree) (*Blob, error) {
+	for _, node := range *t {
+		if node.Tree != nil && node.Subtree == nil {
+			b, err := arch.saveTree(node.Tree)
+			if err != nil {
+				return nil, err
+			}
+			node.Subtree = b.ID
+			arch.SaveUpdate(Stats{Directories: 1})
+		} else if node.Type == "file" && len(node.Content) == 0 {
+			err := arch.SaveFile(node)
+			if err != nil {
+				return nil, err
+			}
+
+			arch.SaveUpdate(Stats{Files: 1, Bytes: node.Size})
+		} else {
+			arch.SaveUpdate(Stats{Other: 1})
+		}
+	}
+
+	blob, err := arch.SaveJSON(backend.Tree, t)
+	if err != nil {
+		return nil, err
+	}
+
+	return blob, nil
+}
+
+func (arch *Archiver) Snapshot(dir string, t *Tree) (*Snapshot, backend.ID, error) {
 	sn := NewSnapshot(dir)
 
-	fi, err := os.Lstat(dir)
+	blob, err := arch.saveTree(t)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	node, err := NodeFromFileInfo(dir, fi)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	if node.Type == "dir" {
-		tree, err := arch.ImportDir(dir)
-		if err != nil {
-			return nil, nil, err
-		}
-
-		blob, err := arch.SaveJSON(backend.Tree, tree)
-		if err != nil {
-			return nil, nil, err
-		}
-
-		node.Subtree = blob.ID
-	} else if node.Type == "file" {
-		_, err := arch.SaveFile(node)
-		if err != nil {
-			return nil, nil, err
-		}
-	}
-
-	blob, err := arch.SaveJSON(backend.Tree, &Tree{node})
 	if err != nil {
 		return nil, nil, err
 	}
@@ -185,5 +223,5 @@ func (arch *Archiver) Import(dir string) (*Snapshot, *Blob, error) {
 		return nil, nil, err
 	}
 
-	return sn, blob, nil
+	return sn, blob.Storage, nil
 }

cmd/khepri/cmd_backup.go

@@ -4,11 +4,41 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"strings"
 
 	"github.com/fd0/khepri"
 	"github.com/fd0/khepri/backend"
+	"golang.org/x/crypto/ssh/terminal"
 )
 
+func format_bytes(c uint64) string {
+	b := float64(c)
+
+	switch {
+	case c > 1<<40:
+		return fmt.Sprintf("%.3f TiB", b/(1<<40))
+	case c > 1<<30:
+		return fmt.Sprintf("%.3f GiB", b/(1<<30))
+	case c > 1<<20:
+		return fmt.Sprintf("%.3f MiB", b/(1<<20))
+	case c > 1<<10:
+		return fmt.Sprintf("%.3f KiB", b/(1<<10))
+	default:
+		return fmt.Sprintf("%d B", c)
+	}
+}
+
+func print_tree2(indent int, t *khepri.Tree) {
+	for _, node := range *t {
+		if node.Tree != nil {
+			fmt.Printf("%s%s/\n", strings.Repeat("  ", indent), node.Name)
+			print_tree2(indent+1, node.Tree)
+		} else {
+			fmt.Printf("%s%s\n", strings.Repeat("  ", indent), node.Name)
+		}
+	}
+}
+
 func commandBackup(be backend.Server, key *khepri.Key, args []string) error {
 	if len(args) != 1 {
 		return errors.New("usage: backup [dir|file]")
@@ -25,12 +55,49 @@ func commandBackup(be backend.Server, key *khepri.Key, args []string) error {
 		return err
 	}
 
-	_, blob, err := arch.Import(target)
+	fmt.Printf("scanning %s\n", target)
+
+	if terminal.IsTerminal(int(os.Stdout.Fd())) {
+		arch.ScannerUpdate = func(stats khepri.Stats) {
+			fmt.Printf("\r%6d directories, %6d files, %14s", stats.Directories, stats.Files, format_bytes(stats.Bytes))
+		}
+	}
+
+	// TODO: add filter
+	// arch.Filter = func(dir string, fi os.FileInfo) bool {
+	// 	return true
+	// }
+
+	t, err := arch.LoadTree(target)
 	if err != nil {
+		fmt.Fprintf(os.Stderr, "error: %v\n", err)
 		return err
 	}
 
-	fmt.Printf("snapshot %s saved\n", blob.Storage)
+	fmt.Printf("\r%6d directories, %6d files, %14s\n", arch.Stats.Directories, arch.Stats.Files, format_bytes(arch.Stats.Bytes))
+
+	stats := khepri.Stats{}
+	if terminal.IsTerminal(int(os.Stdout.Fd())) {
+		arch.SaveUpdate = func(s khepri.Stats) {
+			stats.Files += s.Files
+			stats.Directories += s.Directories
+			stats.Other += s.Other
+			stats.Bytes += s.Bytes
+
+			fmt.Printf("\r%3.2f%% %d/%d directories, %d/%d files, %s/%s",
+				float64(stats.Bytes)/float64(arch.Stats.Bytes)*100,
+				stats.Directories, arch.Stats.Directories,
+				stats.Files, arch.Stats.Files,
+				format_bytes(stats.Bytes), format_bytes(arch.Stats.Bytes))
+		}
+	}
+
+	sn, id, err := arch.Snapshot(target, t)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error: %v\n", err)
+	}
+
+	fmt.Printf("\nsnapshot %s saved: %v\n", id, sn)
 
 	return nil
 }

tree.go

@@ -34,6 +34,8 @@ type Node struct {
 	Content    []backend.ID `json:"content,omitempty"`
 	Subtree    backend.ID   `json:"subtree,omitempty"`
 
+	Tree *Tree `json:"-"`
+
 	path string
 }