Refactor

cmd/khepri/cmd_backup.go

@@ -1,117 +1,13 @@
 package main
 
 import (
-	"crypto/sha256"
 	"errors"
 	"fmt"
-	"io"
 	"log"
-	"os"
-	"path/filepath"
 
 	"github.com/fd0/khepri"
 )
 
-func hash(filename string) (khepri.ID, error) {
-	h := sha256.New()
-	f, err := os.Open(filename)
-	if err != nil {
-		return nil, err
-	}
-
-	io.Copy(h, f)
-	return h.Sum([]byte{}), nil
-}
-
-func store_file(repo *khepri.Repository, path string) (khepri.ID, error) {
-	obj, idch, err := repo.Create(khepri.TYPE_BLOB)
-	if err != nil {
-		return nil, err
-	}
-
-	file, err := os.Open(path)
-	defer func() {
-		file.Close()
-	}()
-
-	_, err = io.Copy(obj, file)
-	if err != nil {
-		return nil, err
-	}
-
-	err = obj.Close()
-	if err != nil {
-		return nil, err
-	}
-
-	return <-idch, nil
-}
-
-func archive_dir(repo *khepri.Repository, path string) (khepri.ID, error) {
-	log.Printf("archiving dir %q", path)
-
-	dir, err := os.Open(path)
-	if err != nil {
-		log.Printf("open(%q): %v\n", path, err)
-		return nil, err
-	}
-
-	entries, err := dir.Readdir(-1)
-	if err != nil {
-		log.Printf("readdir(%q): %v\n", path, err)
-		return nil, err
-	}
-
-	// use nil ID for empty directories
-	if len(entries) == 0 {
-		return nil, nil
-	}
-
-	t := khepri.NewTree()
-	for _, e := range entries {
-		node := khepri.NodeFromFileInfo(e)
-
-		var id khepri.ID
-		var err error
-
-		if e.IsDir() {
-			id, err = archive_dir(repo, filepath.Join(path, e.Name()))
-		} else {
-			id, err = store_file(repo, filepath.Join(path, e.Name()))
-		}
-
-		node.Content = id
-
-		t.Nodes = append(t.Nodes, node)
-
-		if err != nil {
-			log.Printf("  error storing %q: %v\n", e.Name(), err)
-			continue
-		}
-	}
-
-	log.Printf("  dir %q: %v entries", path, len(t.Nodes))
-
-	obj, idch, err := repo.Create(khepri.TYPE_BLOB)
-
-	if err != nil {
-		log.Printf("error creating object for tree: %v", err)
-		return nil, err
-	}
-
-	err = t.Save(obj)
-	if err != nil {
-		log.Printf("error saving tree to repo: %v", err)
-	}
-
-	obj.Close()
-
-	id := <-idch
-	log.Printf("tree for %q saved at %s", path, id)
-
-	return id, nil
-}
-
 func commandBackup(repo *khepri.Repository, args []string) error {
 	if len(args) != 1 {
 		return errors.New("usage: backup dir")
@@ -119,13 +15,18 @@ func commandBackup(repo *khepri.Repository, args []string) error {
 
 	target := args[0]
 
-	id, err := archive_dir(repo, target)
+	tree, err := khepri.NewTreeFromPath(repo, target)
+	if err != nil {
+		return err
+	}
+
+	id, err := tree.Save(repo)
 	if err != nil {
 		return err
 	}
 
 	sn := khepri.NewSnapshot(target)
-	sn.TreeID = id
+	sn.Content = id
 	snid, err := sn.Save(repo)
 
 	if err != nil {

cmd/khepri/cmd_dump.go

@@ -0,0 +1,90 @@
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"os"
+
+	"github.com/fd0/khepri"
+)
+
+func dump_tree(repo *khepri.Repository, id khepri.ID) error {
+	tree, err := khepri.NewTreeFromRepo(repo, id)
+	if err != nil {
+		return err
+	}
+
+	buf, err := json.MarshalIndent(tree, "", "  ")
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf("tree %s\n%s\n", id, buf)
+
+	for _, node := range tree.Nodes {
+		if node.Type == "dir" {
+			err = dump_tree(repo, node.Subtree)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "error: %v\n", err)
+			}
+		}
+	}
+
+	return nil
+}
+
+func dump_snapshot(repo *khepri.Repository, id khepri.ID) error {
+	sn, err := khepri.LoadSnapshot(repo, id)
+	if err != nil {
+		log.Fatalf("error loading snapshot %s", id)
+	}
+
+	buf, err := json.MarshalIndent(sn, "", "  ")
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf("%s\n%s\n", sn, buf)
+
+	return dump_tree(repo, sn.Content)
+}
+
+func dump_file(repo *khepri.Repository, id khepri.ID) error {
+	rd, err := repo.Get(khepri.TYPE_BLOB, id)
+	if err != nil {
+		return err
+	}
+
+	io.Copy(os.Stdout, rd)
+
+	return nil
+}
+
+func commandDump(repo *khepri.Repository, args []string) error {
+	if len(args) != 2 {
+		return errors.New("usage: dump [snapshot|tree|file] ID")
+	}
+
+	tpe := args[0]
+
+	id, err := khepri.ParseID(args[1])
+	if err != nil {
+		errx(1, "invalid id %q: %v", args[0], err)
+	}
+
+	switch tpe {
+	case "snapshot":
+		return dump_snapshot(repo, id)
+	case "tree":
+		return dump_tree(repo, id)
+	case "file":
+		return dump_file(repo, id)
+	default:
+		return fmt.Errorf("invalid type %q", tpe)
+	}
+
+	return nil
+}

cmd/khepri/cmd_fsck.go

@@ -41,7 +41,7 @@ func fsck_snapshot(repo *khepri.Repository, id khepri.ID) (bool, error) {
 		return false, err
 	}
 
-	return fsck_tree(repo, sn.TreeID)
+	return fsck_tree(repo, sn.Content)
 }
 
 func commandFsck(repo *khepri.Repository, args []string) error {

cmd/khepri/cmd_restore.go

@@ -2,23 +2,26 @@ package main
 
 import (
 	"errors"
+	"fmt"
 	"io"
 	"log"
 	"os"
-	"path"
+	"path/filepath"
+	"syscall"
 
 	"github.com/fd0/khepri"
 )
 
-func restore_file(repo *khepri.Repository, node khepri.Node, target string) error {
+func restore_file(repo *khepri.Repository, node *khepri.Node, path string) (err error) {
-	log.Printf("  restore file %q\n", target)
+	switch node.Type {
-
+	case "file":
+		// TODO: handle hard links
 		rd, err := repo.Get(khepri.TYPE_BLOB, node.Content)
 		if err != nil {
 			return err
 		}
 
-	f, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY, 0600)
+		f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0600)
 		defer f.Close()
 		if err != nil {
 			return err
@@ -29,17 +32,65 @@ func restore_file(repo *khepri.Repository, node khepri.Node, target string) erro
 			return err
 		}
 
-	err = f.Chmod(node.Mode)
+	case "symlink":
+		err = os.Symlink(node.LinkTarget, path)
 		if err != nil {
 			return err
 		}
 
-	err = f.Chown(int(node.User), int(node.Group))
+		err = os.Lchown(path, int(node.UID), int(node.GID))
 		if err != nil {
 			return err
 		}
 
-	err = os.Chtimes(target, node.AccessTime, node.ModTime)
+		f, err := os.OpenFile(path, khepri.O_PATH|syscall.O_NOFOLLOW, 0600)
+		defer f.Close()
+		if err != nil {
+			return err
+		}
+
+		var utimes = []syscall.Timeval{
+			syscall.NsecToTimeval(node.AccessTime.UnixNano()),
+			syscall.NsecToTimeval(node.ModTime.UnixNano()),
+		}
+		err = syscall.Futimes(int(f.Fd()), utimes)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	case "dev":
+		err = syscall.Mknod(path, syscall.S_IFBLK|0600, int(node.Device))
+		if err != nil {
+			return err
+		}
+	case "chardev":
+		err = syscall.Mknod(path, syscall.S_IFCHR|0600, int(node.Device))
+		if err != nil {
+			return err
+		}
+	case "fifo":
+		err = syscall.Mkfifo(path, 0600)
+		if err != nil {
+			return err
+		}
+	case "socket":
+		// nothing to do, we do not restore sockets
+	default:
+		return fmt.Errorf("filetype %q not implemented!\n", node.Type)
+	}
+
+	err = os.Chmod(path, node.Mode)
+	if err != nil {
+		return err
+	}
+
+	err = os.Chown(path, int(node.UID), int(node.GID))
+	if err != nil {
+		return err
+	}
+
+	err = os.Chtimes(path, node.AccessTime, node.ModTime)
 	if err != nil {
 		return err
 	}
@@ -47,61 +98,48 @@ func restore_file(repo *khepri.Repository, node khepri.Node, target string) erro
 	return nil
 }
 
-func restore_dir(repo *khepri.Repository, id khepri.ID, target string) error {
+func restore_subtree(repo *khepri.Repository, tree *khepri.Tree, path string) {
-	log.Printf("  restore dir %q\n", target)
+	fmt.Printf("restore_subtree(%s)\n", path)
-	rd, err := repo.Get(khepri.TYPE_BLOB, id)
-	if err != nil {
-		return err
-	}
 
-	t := khepri.NewTree()
+	for _, node := range tree.Nodes {
-	err = t.Restore(rd)
+		nodepath := filepath.Join(path, node.Name)
-	if err != nil {
+		// fmt.Printf("%s:%s\n", node.Type, nodepath)
-		return err
-	}
 
-	for _, node := range t.Nodes {
+		if node.Type == "dir" {
-		name := path.Base(node.Name)
+			err := os.Mkdir(nodepath, 0700)
-		if name == "." || name == ".." {
-			return errors.New("invalid path")
-		}
-
-		nodepath := path.Join(target, name)
-		if node.Mode.IsDir() {
-			err = os.Mkdir(nodepath, 0700)
 			if err != nil {
-				return err
+				fmt.Fprintf(os.Stderr, "%s\n", err)
+				continue
 			}
 
 			err = os.Chmod(nodepath, node.Mode)
 			if err != nil {
-				return err
+				fmt.Fprintf(os.Stderr, "%s\n", err)
+				continue
 			}
 
-			err = os.Chown(nodepath, int(node.User), int(node.Group))
+			err = os.Chown(nodepath, int(node.UID), int(node.GID))
 			if err != nil {
-				return err
+				fmt.Fprintf(os.Stderr, "%s\n", err)
+				continue
 			}
 
-			err = restore_dir(repo, node.Content, nodepath)
+			restore_subtree(repo, node.Tree, filepath.Join(path, node.Name))
-			if err != nil {
-				return err
-			}
 
 			err = os.Chtimes(nodepath, node.AccessTime, node.ModTime)
 			if err != nil {
-				return err
+				fmt.Fprintf(os.Stderr, "%s\n", err)
+				continue
 			}
 
 		} else {
-			err = restore_file(repo, node, nodepath)
+			err := restore_file(repo, node, nodepath)
 			if err != nil {
-				return err
+				fmt.Fprintf(os.Stderr, "%s\n", err)
+				continue
 			}
 		}
 	}
-
-	return nil
 }
 
 func commandRestore(repo *khepri.Repository, args []string) error {
@@ -126,11 +164,13 @@ func commandRestore(repo *khepri.Repository, args []string) error {
 		log.Fatalf("error loading snapshot %s", id)
 	}
 
-	err = restore_dir(repo, sn.TreeID, target)
+	tree, err := khepri.NewTreeFromRepo(repo, sn.Content)
 	if err != nil {
-		return err
+		log.Fatalf("error loading tree %s", sn.Content)
 	}
 
+	restore_subtree(repo, tree, target)
+
 	log.Printf("%q restored to %q\n", id, target)
 
 	return nil

cmd/khepri/main.go

@@ -34,6 +34,7 @@ func init() {
 	commands["list"] = commandList
 	commands["snapshots"] = commandSnapshots
 	commands["fsck"] = commandFsck
+	commands["dump"] = commandDump
 }
 
 func main() {

cmd/stat/stat.go

@@ -0,0 +1,37 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/fd0/khepri"
+)
+
+func main() {
+	if len(os.Args) == 1 {
+		fmt.Printf("usage: %s [file] [file] [...]\n", os.Args[0])
+		os.Exit(1)
+	}
+
+	for _, path := range os.Args[1:] {
+		fmt.Printf("lstat %s\n", path)
+
+		fi, err := os.Lstat(path)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "%v", err)
+			continue
+		}
+
+		node, err := khepri.NodeFromFileInfo(path, fi)
+		if err != nil {
+			fmt.Printf("err: %v\n", err)
+		}
+
+		buf, err := json.MarshalIndent(node, "", "  ")
+		if err != nil {
+			panic(err)
+		}
+		fmt.Printf("%s\n", string(buf))
+	}
+}

cmd/tree_serialise/main.go

@@ -0,0 +1,222 @@
+package main
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+func check(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+// References content within a repository.
+type ID []byte
+
+func (id ID) String() string {
+	return hex.EncodeToString(id)
+}
+
+func (id ID) MarshalJSON() ([]byte, error) {
+	return json.Marshal(id.String())
+}
+
+func (id *ID) UnmarshalJSON(b []byte) error {
+	var s string
+	err := json.Unmarshal(b, &s)
+	if err != nil {
+		return err
+	}
+
+	*id = make([]byte, len(s)/2)
+	_, err = hex.Decode(*id, []byte(s))
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// ParseID converts the given string to an ID.
+func ParseID(s string) ID {
+	b, err := hex.DecodeString(s)
+
+	if err != nil {
+		panic(err)
+	}
+
+	return ID(b)
+}
+
+type Repository interface {
+	Store([]byte) ID
+	Get(ID) []byte
+}
+
+type Repo map[string][]byte
+
+func (r Repo) Store(buf []byte) ID {
+	hash := sha256.New()
+	_, err := hash.Write(buf)
+	check(err)
+
+	id := ID(hash.Sum([]byte{}))
+	r[id.String()] = buf
+
+	return id
+}
+
+func (r Repo) Get(id ID) []byte {
+	buf, ok := r[id.String()]
+	if !ok {
+		panic("no such id")
+	}
+
+	return buf
+}
+
+func (r Repo) Dump(wr io.Writer) {
+	for k, v := range r {
+		_, err := wr.Write([]byte(k))
+		check(err)
+		_, err = wr.Write([]byte(":"))
+		check(err)
+		_, err = wr.Write(v)
+		check(err)
+		_, err = wr.Write([]byte("\n"))
+		check(err)
+	}
+}
+
+type Tree struct {
+	Nodes []*Node `json:"nodes,omitempty"`
+}
+
+type Node struct {
+	Name    string `json:"name"`
+	Tree    *Tree  `json:"tree,omitempty"`
+	Subtree ID     `json:"subtree,omitempty"`
+	Content ID     `json:"content,omitempty"`
+}
+
+func (tree Tree) Save(repo Repository) ID {
+	// fmt.Printf("nodes: %#v\n", tree.Nodes)
+	for _, node := range tree.Nodes {
+		if node.Tree != nil {
+			node.Subtree = node.Tree.Save(repo)
+			node.Tree = nil
+		}
+	}
+
+	buf, err := json.Marshal(tree)
+	check(err)
+
+	return repo.Store(buf)
+}
+
+func (tree Tree) PP(wr io.Writer) {
+	tree.pp(0, wr)
+}
+
+func (tree Tree) pp(indent int, wr io.Writer) {
+	for _, node := range tree.Nodes {
+		if node.Tree != nil {
+			fmt.Printf("%s%s/\n", strings.Repeat("    ", indent), node.Name)
+			node.Tree.pp(indent+1, wr)
+		} else {
+			fmt.Printf("%s%s [%s]\n", strings.Repeat("    ", indent), node.Name, node.Content)
+		}
+
+	}
+}
+
+func create_tree(path string) *Tree {
+	dir, err := os.Open(path)
+	check(err)
+
+	entries, err := dir.Readdir(-1)
+	check(err)
+
+	tree := &Tree{
+		Nodes: make([]*Node, 0, len(entries)),
+	}
+
+	for _, entry := range entries {
+		node := &Node{}
+		node.Name = entry.Name()
+
+		if !entry.Mode().IsDir() && entry.Mode()&os.ModeType != 0 {
+			fmt.Fprintf(os.Stderr, "skipping %q\n", filepath.Join(path, entry.Name()))
+			continue
+		}
+
+		tree.Nodes = append(tree.Nodes, node)
+
+		if entry.IsDir() {
+			node.Tree = create_tree(filepath.Join(path, entry.Name()))
+			continue
+		}
+
+		file, err := os.Open(filepath.Join(path, entry.Name()))
+		defer file.Close()
+		check(err)
+
+		hash := sha256.New()
+		io.Copy(hash, file)
+
+		node.Content = hash.Sum([]byte{})
+	}
+
+	return tree
+}
+
+func load_tree(repo Repository, id ID) *Tree {
+	tree := &Tree{}
+
+	buf := repo.Get(id)
+	json.Unmarshal(buf, tree)
+
+	for _, node := range tree.Nodes {
+		if node.Subtree != nil {
+			node.Tree = load_tree(repo, node.Subtree)
+			node.Subtree = nil
+		}
+	}
+
+	return tree
+}
+
+func main() {
+	repo := make(Repo)
+
+	tree := create_tree(os.Args[1])
+	// encoder := json.NewEncoder(os.Stdout)
+	// fmt.Println("---------------------------")
+	// encoder.Encode(tree)
+	// fmt.Println("---------------------------")
+
+	id := tree.Save(repo)
+
+	// for k, v := range repo {
+	// 	fmt.Printf("%s: %s\n", k, v)
+	// }
+
+	// fmt.Println("---------------------------")
+
+	tree2 := load_tree(repo, id)
+	tree2.PP(os.Stdout)
+	// encoder.Encode(tree2)
+
+	// dumpfile, err := os.Create("dump")
+	// defer dumpfile.Close()
+	// check(err)
+
+	// repo.Dump(dumpfile)
+}

cmd/tree_test/khepri-repo/blobs/44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a

@@ -0,0 +1 @@
+{}

cmd/tree_test/khepri-repo/blobs/655ee6b9ff7fb7324f1e8566eca49c67d251459787765f9c5c6af513e3db0c72

@@ -0,0 +1 @@
+{"nodes":[{"name":"blobs","type":"dir","mode":2147484096,"mtime":"2014-08-11T19:51:11.894770622+02:00","atime":"2014-08-11T19:51:11.894770622+02:00","ctime":"2014-08-11T19:51:11.894770622+02:00","uid":1000,"gid":100,"user":"fd0","inode":428349318,"subtree":"44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a"},{"name":"refs","type":"dir","mode":2147484096,"mtime":"2014-08-11T19:51:11.894770622+02:00","atime":"2014-08-11T19:51:11.894770622+02:00","ctime":"2014-08-11T19:51:11.894770622+02:00","uid":1000,"gid":100,"user":"fd0","inode":4883656,"subtree":"44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a"},{"name":"tmp","type":"dir","mode":2147484096,"mtime":"2014-08-11T19:51:11.894770622+02:00","atime":"2014-08-11T19:51:11.894770622+02:00","ctime":"2014-08-11T19:51:11.894770622+02:00","uid":1000,"gid":100,"user":"fd0","inode":169971890,"subtree":"44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a"}]}

cmd/tree_test/khepri-repo/blobs/9e3528071fbfda7702010f2b711174c17df3156b77c14015ce633c9722c0e146

@@ -0,0 +1 @@
+{"nodes":[{"name":"main.go","type":"file","mode":420,"mtime":"2014-08-10T23:18:15.815999516+02:00","atime":"2014-08-10T23:18:15.815999516+02:00","ctime":"2014-08-10T23:18:15.819332884+02:00","uid":1000,"gid":100,"user":"fd0","inode":193352413,"size":1181,"links":1,"content":"c45d3975908245296eb5730cf4bbbe9d8c39b0736658b5df9beada0739d40569"},{"name":"khepri-repo","type":"dir","mode":2147484096,"mtime":"2014-08-11T19:51:11.894770622+02:00","atime":"2014-08-11T19:51:11.894770622+02:00","ctime":"2014-08-11T19:51:11.894770622+02:00","uid":1000,"gid":100,"user":"fd0","inode":275443328,"subtree":"655ee6b9ff7fb7324f1e8566eca49c67d251459787765f9c5c6af513e3db0c72"}]}

cmd/tree_test/main.go

@@ -0,0 +1,69 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/fd0/khepri"
+)
+
+func check(err error) {
+	if err == nil {
+		return
+	}
+
+	fmt.Fprintf(os.Stderr, "error: %v\n", err)
+	os.Exit(1)
+}
+
+func save(repo *khepri.Repository, path string) {
+	tree, err := khepri.NewTreeFromPath(repo, path)
+
+	check(err)
+
+	id, err := tree.Save(repo)
+
+	fmt.Printf("saved tree as %s\n", id)
+}
+
+func restore(repo *khepri.Repository, idstr string) {
+	id, err := khepri.ParseID(idstr)
+	check(err)
+
+	tree, err := khepri.NewTreeFromRepo(repo, id)
+	check(err)
+
+	walk(0, tree)
+}
+
+func walk(indent int, tree *khepri.Tree) {
+	for _, node := range tree.Nodes {
+		if node.Type == "dir" {
+			fmt.Printf("%s%s:%s/\n", strings.Repeat(" ", indent), node.Type, node.Name)
+			walk(indent+1, node.Tree)
+		} else {
+			fmt.Printf("%s%s:%s\n", strings.Repeat(" ", indent), node.Type, node.Name)
+		}
+	}
+}
+
+func main() {
+	if len(os.Args) != 3 {
+		fmt.Fprintf(os.Stderr, "usage: %s [save|restore] DIR\n", os.Args[0])
+		os.Exit(1)
+	}
+
+	command := os.Args[1]
+	arg := os.Args[2]
+
+	repo, err := khepri.NewRepository("khepri-repo")
+	check(err)
+
+	switch command {
+	case "save":
+		save(repo, arg)
+	case "restore":
+		restore(repo, arg)
+	}
+}

repository.go

@@ -31,9 +31,11 @@ func (n Name) Encode() string {
 	return url.QueryEscape(string(n))
 }
 
+type HashFunc func() hash.Hash
+
 type Repository struct {
 	path string
-	hash func() hash.Hash
+	hash HashFunc
 }
 
 type Type int
@@ -99,6 +101,11 @@ func (r *Repository) create() error {
 	return nil
 }
 
+// SetHash changes the hash function used for deriving IDs. Default is SHA256.
+func (r *Repository) SetHash(h HashFunc) {
+	r.hash = h
+}
+
 // Path returns the directory used for this repository.
 func (r *Repository) Path() string {
 	return r.path

snapshot.go

@@ -2,6 +2,7 @@ package khepri
 
 import (
 	"encoding/json"
+	"fmt"
 	"os"
 	"os/user"
 	"time"
@@ -9,12 +10,14 @@ import (
 
 type Snapshot struct {
 	Time     time.Time `json:"time"`
-	TreeID   ID        `json:"tree"`
+	Content  ID        `json:"content"`
+	Tree     *Tree     `json:"-"`
 	Dir      string    `json:"dir"`
 	Hostname string    `json:"hostname,omitempty"`
 	Username string    `json:"username,omitempty"`
 	UID      string    `json:"uid,omitempty"`
 	GID      string    `json:"gid,omitempty"`
+	id       ID        `json:omit`
 }
 
 func NewSnapshot(dir string) *Snapshot {
@@ -39,7 +42,7 @@ func NewSnapshot(dir string) *Snapshot {
 }
 
 func (sn *Snapshot) Save(repo *Repository) (ID, error) {
-	if sn.TreeID == nil {
+	if sn.Content == nil {
 		panic("Snapshot.Save() called with nil tree id")
 	}
 
@@ -59,7 +62,9 @@ func (sn *Snapshot) Save(repo *Repository) (ID, error) {
 		return nil, err
 	}
 
-	return <-id_ch, nil
+	sn.id = <-id_ch
+
+	return sn.id, nil
 }
 
 func LoadSnapshot(repo *Repository, id ID) (*Snapshot, error) {
@@ -78,5 +83,15 @@ func LoadSnapshot(repo *Repository, id ID) (*Snapshot, error) {
 		return nil, err
 	}
 
+	sn.id = id
+
 	return sn, nil
 }
+
+func (sn *Snapshot) ID() ID {
+	return sn.id
+}
+
+func (sn *Snapshot) String() string {
+	return fmt.Sprintf("<Snapshot of %q at %s>", sn.Dir, sn.Time.Format(time.RFC822Z))
+}

snapshot_test.go

@@ -17,7 +17,7 @@ func TestSnapshot(t *testing.T) {
 	}()
 
 	sn := khepri.NewSnapshot("/home/foobar")
-	sn.TreeID, err = khepri.ParseID("c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2")
+	sn.Content, err = khepri.ParseID("c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2")
 	ok(t, err)
 	sn.Time, err = time.Parse(time.RFC3339Nano, "2014-08-03T17:49:05.378595539+02:00")
 	ok(t, err)

tree.go

@@ -2,54 +2,238 @@ package khepri
 
 import (
 	"encoding/json"
+	"fmt"
 	"io"
 	"os"
+	"os/user"
+	"path/filepath"
+	"strconv"
 	"syscall"
 	"time"
 )
 
 type Tree struct {
-	Nodes []Node `json:"nodes"`
+	Nodes []*Node `json:"nodes,omitempty"`
 }
 
 type Node struct {
 	Name       string      `json:"name"`
-	Mode       os.FileMode `json:"mode"`
+	Type       string      `json:"type"`
-	ModTime    time.Time   `json:"mtime"`
+	Mode       os.FileMode `json:"mode,omitempty"`
-	AccessTime time.Time   `json:"atime"`
+	ModTime    time.Time   `json:"mtime,omitempty"`
-	User       uint32      `json:"user"`
+	AccessTime time.Time   `json:"atime,omitempty"`
-	Group      uint32      `json:"group"`
+	ChangeTime time.Time   `json:"ctime,omitempty"`
+	UID        uint32      `json:"uid"`
+	GID        uint32      `json:"gid"`
+	User       string      `json:"user,omitempty"`
+	Group      string      `json:"group,omitempty"`
+	Inode      uint64      `json:"inode,omitempty"`
+	Size       uint64      `json:"size,omitempty"`
+	Links      uint64      `json:"links,omitempty"`
+	LinkTarget string      `json:"linktarget,omitempty"`
+	Device     uint64      `json:"device,omitempty"`
 	Content    ID          `json:"content,omitempty"`
+	Subtree    ID          `json:"subtree,omitempty"`
+	Tree       *Tree       `json:"-"`
 }
 
 func NewTree() *Tree {
 	return &Tree{
-		Nodes: []Node{},
+		Nodes: []*Node{},
 	}
 }
 
-func (t *Tree) Restore(r io.Reader) error {
+func NewTreeFromPath(repo *Repository, dir string) (*Tree, error) {
-	dec := json.NewDecoder(r)
+	fd, err := os.Open(dir)
-	return dec.Decode(t)
+	defer fd.Close()
+	if err != nil {
+		return nil, err
 	}
 
-func (t *Tree) Save(w io.Writer) error {
+	entries, err := fd.Readdir(-1)
-	enc := json.NewEncoder(w)
+	if err != nil {
-	return enc.Encode(t)
+		return nil, err
 	}
 
-func NodeFromFileInfo(fi os.FileInfo) Node {
+	tree := &Tree{
-	node := Node{
+		Nodes: make([]*Node, 0, len(entries)),
+	}
+
+	for _, entry := range entries {
+		path := filepath.Join(dir, entry.Name())
+		node, err := NodeFromFileInfo(path, entry)
+		if err != nil {
+			return nil, err
+		}
+
+		tree.Nodes = append(tree.Nodes, node)
+
+		if entry.IsDir() {
+			node.Tree, err = NewTreeFromPath(repo, path)
+			if err != nil {
+				return nil, err
+			}
+			continue
+		}
+
+		if node.Type == "file" {
+			file, err := os.Open(path)
+			defer file.Close()
+			if err != nil {
+				return nil, err
+			}
+
+			wr, idch, err := repo.Create(TYPE_BLOB)
+			if err != nil {
+				return nil, err
+			}
+
+			io.Copy(wr, file)
+			err = wr.Close()
+			if err != nil {
+				return nil, err
+			}
+
+			node.Content = <-idch
+		}
+	}
+
+	return tree, nil
+}
+
+func (tree *Tree) Save(repo *Repository) (ID, error) {
+	for _, node := range tree.Nodes {
+		if node.Tree != nil {
+			var err error
+			node.Subtree, err = node.Tree.Save(repo)
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	buf, err := json.Marshal(tree)
+	if err != nil {
+		return nil, err
+	}
+
+	wr, idch, err := repo.Create(TYPE_BLOB)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = wr.Write(buf)
+	if err != nil {
+		return nil, err
+	}
+
+	err = wr.Close()
+	if err != nil {
+		return nil, err
+	}
+
+	return <-idch, nil
+}
+
+func NewTreeFromRepo(repo *Repository, id ID) (*Tree, error) {
+	tree := NewTree()
+
+	rd, err := repo.Get(TYPE_BLOB, id)
+	defer rd.Close()
+	if err != nil {
+		return nil, err
+	}
+
+	decoder := json.NewDecoder(rd)
+
+	err = decoder.Decode(tree)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, node := range tree.Nodes {
+		if node.Subtree != nil {
+			node.Tree, err = NewTreeFromRepo(repo, node.Subtree)
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	return tree, nil
+}
+
+// TODO: make sure that node.Type is valid
+
+func (node *Node) fill_extra(path string, fi os.FileInfo) (err error) {
+	stat, ok := fi.Sys().(*syscall.Stat_t)
+	if !ok {
+		return
+	}
+
+	node.ChangeTime = time.Unix(stat.Ctim.Unix())
+	node.AccessTime = time.Unix(stat.Atim.Unix())
+	node.UID = stat.Uid
+	node.GID = stat.Gid
+
+	if u, nil := user.LookupId(strconv.Itoa(int(stat.Uid))); err == nil {
+		node.User = u.Username
+	}
+
+	// TODO: implement getgrnam()
+	// if g, nil := user.LookupId(strconv.Itoa(int(stat.Uid))); err == nil {
+	// 	node.User = u.Username
+	// }
+
+	node.Inode = stat.Ino
+
+	switch node.Type {
+	case "file":
+		node.Size = uint64(stat.Size)
+		node.Links = stat.Nlink
+	case "dir":
+		// nothing to do
+	case "symlink":
+		node.LinkTarget, err = os.Readlink(path)
+	case "dev":
+		node.Device = stat.Rdev
+	case "chardev":
+		node.Device = stat.Rdev
+	case "fifo":
+		// nothing to do
+	case "socket":
+		// nothing to do
+	default:
+		panic(fmt.Sprintf("invalid node type %q", node.Type))
+	}
+
+	return err
+}
+
+func NodeFromFileInfo(path string, fi os.FileInfo) (*Node, error) {
+	node := &Node{
 		Name:    fi.Name(),
-		Mode:    fi.Mode(),
+		Mode:    fi.Mode() & os.ModePerm,
 		ModTime: fi.ModTime(),
 	}
 
-	if stat, ok := fi.Sys().(*syscall.Stat_t); ok {
+	switch fi.Mode() & (os.ModeType | os.ModeCharDevice) {
-		node.User = stat.Uid
+	case 0:
-		node.Group = stat.Gid
+		node.Type = "file"
-		node.AccessTime = time.Unix(stat.Atim.Unix())
+	case os.ModeDir:
+		node.Type = "dir"
+	case os.ModeSymlink:
+		node.Type = "symlink"
+	case os.ModeDevice | os.ModeCharDevice:
+		node.Type = "chardev"
+	case os.ModeDevice:
+		node.Type = "dev"
+	case os.ModeNamedPipe:
+		node.Type = "fifo"
+	case os.ModeSocket:
+		node.Type = "socket"
 	}
 
-	return node
+	err := node.fill_extra(path, fi)
+	return node, err
 }

tree_test.go

@@ -1,72 +0,0 @@
-package khepri_test
-
-import (
-	"bytes"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/fd0/khepri"
-)
-
-func parseTime(str string) time.Time {
-	t, err := time.Parse(time.RFC3339Nano, str)
-	if err != nil {
-		panic(err)
-	}
-
-	return t
-}
-
-func TestTree(t *testing.T) {
-	var tree = &khepri.Tree{
-		Nodes: []khepri.Node{
-			khepri.Node{
-				Name:       "foobar",
-				Mode:       0755,
-				ModTime:    parseTime("2014-04-20T22:16:54.161401+02:00"),
-				AccessTime: parseTime("2014-04-21T22:16:54.161401+02:00"),
-				User:       1000,
-				Group:      1001,
-				Content:    []byte{0x41, 0x42, 0x43},
-			},
-			khepri.Node{
-				Name:       "baz",
-				Mode:       0755,
-				User:       1000,
-				ModTime:    parseTime("2014-04-20T22:16:54.161401+02:00"),
-				AccessTime: parseTime("2014-04-21T22:16:54.161401+02:00"),
-				Group:      1001,
-				Content:    []byte("\xde\xad\xbe\xef\xba\xdc\x0d\xe0"),
-			},
-		},
-	}
-
-	const raw = `{"nodes":[{"name":"foobar","mode":493,"mtime":"2014-04-20T22:16:54.161401+02:00","atime":"2014-04-21T22:16:54.161401+02:00","user":1000,"group":1001,"content":"414243"},{"name":"baz","mode":493,"mtime":"2014-04-20T22:16:54.161401+02:00","atime":"2014-04-21T22:16:54.161401+02:00","user":1000,"group":1001,"content":"deadbeefbadc0de0"}]}`
-
-	// test save
-	buf := &bytes.Buffer{}
-
-	tree.Save(buf)
-	equals(t, raw, strings.TrimRight(buf.String(), "\n"))
-
-	tree2 := new(khepri.Tree)
-	err := tree2.Restore(buf)
-	ok(t, err)
-	equals(t, tree, tree2)
-
-	// test nodes for equality
-	for i, n := range tree.Nodes {
-		equals(t, n.Content, tree2.Nodes[i].Content)
-	}
-
-	// test restore
-	buf = bytes.NewBufferString(raw)
-
-	tree2 = new(khepri.Tree)
-	err = tree2.Restore(buf)
-	ok(t, err)
-
-	// test if tree has correctly been restored
-	equals(t, tree, tree2)
-}

zerrors_linux.go

@@ -0,0 +1,5 @@
+package khepri
+
+// Add constant O_PATH missing from Go1.3, will be added to Go1.4 according to
+// https://code.google.com/p/go/issues/detail?id=7830
+const O_PATH = 010000000