From 4f3a54dc4077c1da890eb067883f83dbb9eb1ba8 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Mon, 21 Apr 2014 23:25:31 +0200 Subject: [PATCH] First test implementation --- storage/id.go | 59 +++++++++++++++++++++++++++++ storage/repository.go | 58 ++++++++++++++++------------ storage/repository_test.go | 12 +++++- storage/tree.go | 53 ++++++++++++++++++++++++++ storage/tree_test.go | 77 ++++++++++++++++++++++++++++++++++++++ storagetest/main.go | 76 ++++++++++++++++++++++++++++--------- 6 files changed, 293 insertions(+), 42 deletions(-) create mode 100644 storage/id.go create mode 100644 storage/tree.go create mode 100644 storage/tree_test.go diff --git a/storage/id.go b/storage/id.go new file mode 100644 index 000000000..759dacefe --- /dev/null +++ b/storage/id.go @@ -0,0 +1,59 @@ +package storage + +import ( + "bytes" + "encoding/hex" + "encoding/json" +) + +// References content within a repository. +type ID []byte + +// ParseID converts the given string to an ID. +func ParseID(s string) (ID, error) { + b, err := hex.DecodeString(s) + + if err != nil { + return nil, err + } + + return ID(b), nil +} +func (id ID) String() string { + return hex.EncodeToString(id) +} + +// Equal compares an ID to another other. +func (id ID) Equal(other ID) bool { + return bytes.Equal(id, other) +} + +// EqualString compares this ID to another one, given as a string. +func (id ID) EqualString(other string) (bool, error) { + s, err := hex.DecodeString(other) + if err != nil { + return false, err + } + + return id.Equal(ID(s)), nil +} + +func (id ID) MarshalJSON() ([]byte, error) { + return json.Marshal(id.String()) +} + +func (id *ID) UnmarshalJSON(b []byte) error { + var s string + err := json.Unmarshal(b, &s) + if err != nil { + return err + } + + *id = make([]byte, len(s)/2) + _, err = hex.Decode(*id, []byte(s)) + if err != nil { + return err + } + + return nil +} diff --git a/storage/repository.go b/storage/repository.go index a0caec1aa..7c95607f0 100644 --- a/storage/repository.go +++ b/storage/repository.go @@ -1,7 +1,6 @@ package storage import ( - "bytes" "crypto/sha256" "encoding/hex" "errors" @@ -25,6 +24,7 @@ const ( type Repository interface { Put(reader io.Reader) (ID, error) PutFile(path string) (ID, error) + PutRaw([]byte) (ID, error) Get(ID) (io.Reader, error) Test(ID) (bool, error) Remove(ID) error @@ -37,28 +37,6 @@ var ( ErrIDDoesNotExist = errors.New("ID does not exist") ) -// References content within a repository. -type ID []byte - -func (id ID) String() string { - return hex.EncodeToString(id) -} - -// Equal compares an ID to another other. -func (id ID) Equal(other ID) bool { - return bytes.Equal(id, other) -} - -// EqualString compares this ID to another one, given as a string. -func (id ID) EqualString(other string) (bool, error) { - s, err := hex.DecodeString(other) - if err != nil { - return false, err - } - - return id.Equal(ID(s)), nil -} - // Name stands for the alias given to an ID. type Name string @@ -156,6 +134,40 @@ func (r *DirRepository) PutFile(path string) (ID, error) { return r.Put(f) } +// PutRaw saves a []byte's content to the repository and returns the ID. +func (r *DirRepository) PutRaw(buf []byte) (ID, error) { + // save contents to tempfile, hash while writing + file, err := ioutil.TempFile(path.Join(r.path, tempPath), "temp-") + if err != nil { + return nil, err + } + + wr := hashing.NewWriter(file, r.hash) + n, err := wr.Write(buf) + if err != nil { + return nil, err + } + + if n != len(buf) { + return nil, errors.New("not all bytes written") + } + + err = file.Close() + if err != nil { + return nil, err + } + + // move file to final name using hash of contents + id := ID(wr.Hash()) + filename := path.Join(r.path, objectPath, id.String()) + err = os.Rename(file.Name(), filename) + if err != nil { + return nil, err + } + + return id, nil +} + // Test returns true if the given ID exists in the repository. func (r *DirRepository) Test(id ID) (bool, error) { // try to open file diff --git a/storage/repository_test.go b/storage/repository_test.go index 2f549bae7..ec091ef52 100644 --- a/storage/repository_test.go +++ b/storage/repository_test.go @@ -2,7 +2,6 @@ package storage_test import ( "bytes" - "encoding/hex" "io" "io/ioutil" "os" @@ -55,7 +54,7 @@ var _ = Describe("Storage", func() { Context("File Operations", func() { It("Should detect non-existing file", func() { for _, test := range TestStrings { - id, err := hex.DecodeString(test.id) + id, err := storage.ParseID(test.id) Expect(err).NotTo(HaveOccurred()) // try to get string out, should fail @@ -96,6 +95,15 @@ var _ = Describe("Storage", func() { Expect(repo.Remove(id)) } }) + + It("Should Add Buffer", func() { + for _, test := range TestStrings { + // store buf in repository + id, err := repo.PutRaw([]byte(test.data)) + Expect(err).NotTo(HaveOccurred()) + Expect(id.String()).To(Equal(test.id)) + } + }) }) }) }) diff --git a/storage/tree.go b/storage/tree.go new file mode 100644 index 000000000..818dd741c --- /dev/null +++ b/storage/tree.go @@ -0,0 +1,53 @@ +package storage + +import ( + "encoding/json" + "io" + "os" + "syscall" + "time" +) + +type Tree struct { + Nodes []Node `json:"nodes"` +} + +type Node struct { + Name string `json:"name"` + Mode os.FileMode `json:"mode"` + ModTime time.Time `json:"mtime"` + User uint32 `json:"user"` + Group uint32 `json:"group"` + Content ID `json:"content"` +} + +func NewTree() *Tree { + return &Tree{ + Nodes: []Node{}, + } +} + +func (t *Tree) Restore(r io.Reader) error { + dec := json.NewDecoder(r) + return dec.Decode(t) +} + +func (t *Tree) Save(w io.Writer) error { + enc := json.NewEncoder(w) + return enc.Encode(t) +} + +func NodeFromFileInfo(fi os.FileInfo) Node { + node := Node{ + Name: fi.Name(), + Mode: fi.Mode(), + ModTime: fi.ModTime(), + } + + if stat, ok := fi.Sys().(*syscall.Stat_t); ok { + node.User = stat.Uid + node.Group = stat.Gid + } + + return node +} diff --git a/storage/tree_test.go b/storage/tree_test.go new file mode 100644 index 000000000..b025700d4 --- /dev/null +++ b/storage/tree_test.go @@ -0,0 +1,77 @@ +package storage_test + +import ( + "bytes" + "strings" + "time" + + "github.com/fd0/khepri/storage" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func parseTime(str string) time.Time { + t, err := time.Parse(time.RFC3339Nano, str) + if err != nil { + panic(err) + } + + return t +} + +var _ = Describe("Tree", func() { + var t *storage.Tree + var raw string + + BeforeEach(func() { + t = new(storage.Tree) + t.Nodes = []storage.Node{ + storage.Node{ + Name: "foobar", + Mode: 0755, + ModTime: parseTime("2014-04-20T22:16:54.161401+02:00"), + User: 1000, + Group: 1001, + Content: []byte{0x41, 0x42, 0x43}, + }, + storage.Node{ + Name: "baz", + Mode: 0755, + User: 1000, + ModTime: parseTime("2014-04-20T22:16:54.161401+02:00"), + Group: 1001, + Content: []byte("\xde\xad\xbe\xef\xba\xdc\x0d\xe0"), + }, + } + + raw = `{"nodes":[{"name":"foobar","mode":493,"mtime":"2014-04-20T22:16:54.161401+02:00","user":1000,"group":1001,"content":"414243"},{"name":"baz","mode":493,"mtime":"2014-04-20T22:16:54.161401+02:00","user":1000,"group":1001,"content":"deadbeefbadc0de0"}]}` + }) + + It("Should save", func() { + var buf bytes.Buffer + t.Save(&buf) + Expect(strings.TrimRight(buf.String(), "\n")).To(Equal(raw)) + + t2 := new(storage.Tree) + err := t2.Restore(&buf) + Expect(err).NotTo(HaveOccurred()) + + // test tree for equality + Expect(t2).To(Equal(t)) + + // test nodes for equality + for i, n := range t.Nodes { + Expect(n.Content).To(Equal(t2.Nodes[i].Content)) + } + }) + + It("Should restore", func() { + buf := bytes.NewBufferString(raw) + t2 := new(storage.Tree) + err := t2.Restore(buf) + Expect(err).NotTo(HaveOccurred()) + + // test if tree has correctly been restored + Expect(t2).To(Equal(t)) + }) +}) diff --git a/storagetest/main.go b/storagetest/main.go index c6a150684..e17ed1fdc 100644 --- a/storagetest/main.go +++ b/storagetest/main.go @@ -1,10 +1,12 @@ package main import ( + "bytes" "crypto/sha256" "io" "log" "os" + "path/filepath" "github.com/fd0/khepri/storage" ) @@ -20,31 +22,71 @@ func hash(filename string) (storage.ID, error) { return h.Sum([]byte{}), nil } -func archive_dir(repo storage.Repository, path string) { +func archive_dir(repo storage.Repository, path string) (storage.ID, error) { log.Printf("archiving dir %q", path) - // items := make() - // filepath.Walk(path, func(item string, info os.FileInfo, err error) error { - // log.Printf(" archiving %q", item) + dir, err := os.Open(path) + if err != nil { + log.Printf("open(%q): %v\n", path, err) + return nil, err + } - // if item != path && info.IsDir() { - // archive_dir(repo, item) - // } else { + entries, err := dir.Readdir(-1) + if err != nil { + log.Printf("readdir(%q): %v\n", path, err) + return nil, err + } - // } + t := storage.NewTree() + for _, e := range entries { + node := storage.NodeFromFileInfo(e) - // return nil - // }) + var id storage.ID + var err error + + if e.IsDir() { + id, err = archive_dir(repo, filepath.Join(path, e.Name())) + } else { + id, err = repo.PutFile(filepath.Join(path, e.Name())) + } + + node.Content = id + + t.Nodes = append(t.Nodes, node) + + if err != nil { + log.Printf(" error storing %q: %v\n", e.Name(), err) + continue + } + } + + log.Printf(" dir %q: %v entries", path, len(t.Nodes)) + + var buf bytes.Buffer + t.Save(&buf) + id, err := repo.PutRaw(buf.Bytes()) + + if err != nil { + log.Printf("error saving tree to repo: %v", err) + } + + log.Printf("tree for %q saved at %s", path, id) + + return id, nil } func main() { - repo, err := storage.NewDir("repo") + if len(os.Args) <= 2 { + log.Fatalf("usage: %s repo [add|link|putdir] ...", os.Args[0]) + } + + repo, err := storage.NewDirRepository(os.Args[1]) if err != nil { log.Fatalf("error: %v", err) } - switch os.Args[1] { + switch os.Args[2] { case "add": - for _, file := range os.Args[2:] { + for _, file := range os.Args[3:] { f, err := os.Open(file) if err != nil { log.Printf("error opening file %q: %v", file, err) @@ -59,8 +101,8 @@ func main() { log.Printf("archived file %q as ID %v", file, id) } case "link": - file := os.Args[2] - name := os.Args[3] + file := os.Args[3] + name := os.Args[4] id, err := hash(file) if err != nil { @@ -85,10 +127,10 @@ func main() { log.Fatalf("error linking name %q to id %v", name, id) } case "putdir": - for _, dir := range os.Args[2:] { + for _, dir := range os.Args[3:] { archive_dir(repo, dir) } default: - log.Fatalf("unknown command: %q", os.Args[1]) + log.Fatalf("unknown command: %q", os.Args[2]) } }