Introduce Type for DirRepository objects

repository.go

@@ -2,19 +2,20 @@ package khepri
 
 import (
 	"crypto/sha256"
-	"encoding/hex"
 	"errors"
+	"fmt"
 	"hash"
 	"io"
 	"io/ioutil"
 	"net/url"
 	"os"
 	"path"
+	"path/filepath"
 )
 
 const (
 	dirMode  = 0700
-	objectPath = "objects"
+	blobPath = "blobs"
 	refPath  = "refs"
 	tempPath = "tmp"
 )
@@ -35,6 +36,36 @@ type DirRepository struct {
 	hash func() hash.Hash
 }
 
+type Type int
+
+const (
+	TypeUnknown = iota
+	TypeBlob
+	TypeRef
+)
+
+func NewTypeFromString(s string) Type {
+	switch s {
+	case "blob":
+		return TypeBlob
+	case "ref":
+		return TypeRef
+	}
+
+	panic(fmt.Sprintf("unknown type %q", s))
+}
+
+func (t Type) String() string {
+	switch t {
+	case TypeBlob:
+		return "blob"
+	case TypeRef:
+		return "ref"
+	}
+
+	panic(fmt.Sprintf("unknown type %d", t))
+}
+
 // NewDirRepository creates a new dir-baked repository at the given path.
 func NewDirRepository(path string) (*DirRepository, error) {
 	d := &DirRepository{
@@ -54,7 +85,7 @@ func NewDirRepository(path string) (*DirRepository, error) {
 func (r *DirRepository) create() error {
 	dirs := []string{
 		r.path,
-		path.Join(r.path, objectPath),
+		path.Join(r.path, blobPath),
 		path.Join(r.path, refPath),
 		path.Join(r.path, tempPath),
 	}
@@ -79,10 +110,21 @@ func (r *DirRepository) Path() string {
 	return r.path
 }
 
+// Return temp directory in correct directory for this repository.
+func (r *DirRepository) tempFile() (*os.File, error) {
+	return ioutil.TempFile(path.Join(r.path, tempPath), "temp-")
+}
+
+// Rename temp file to final name according to type and ID.
+func (r *DirRepository) renameFile(file *os.File, t Type, id ID) error {
+	filename := path.Join(r.dir(t), id.String())
+	return os.Rename(file.Name(), filename)
+}
+
 // Put saves content and returns the ID.
-func (r *DirRepository) Put(reader io.Reader) (ID, error) {
+func (r *DirRepository) Put(t Type, reader io.Reader) (ID, error) {
 	// save contents to tempfile, hash while writing
-	file, err := ioutil.TempFile(path.Join(r.path, tempPath), "temp-")
+	file, err := r.tempFile()
 	if err != nil {
 		return nil, err
 	}
@@ -100,8 +142,7 @@ func (r *DirRepository) Put(reader io.Reader) (ID, error) {
 
 	// move file to final name using hash of contents
 	id := ID(rd.Hash())
-	filename := path.Join(r.path, objectPath, id.String())
+	err = r.renameFile(file, t, id)
-	err = os.Rename(file.Name(), filename)
 	if err != nil {
 		return nil, err
 	}
@@ -109,6 +150,23 @@ func (r *DirRepository) Put(reader io.Reader) (ID, error) {
 	return id, nil
 }
 
+// Construct directory for given Type.
+func (r *DirRepository) dir(t Type) string {
+	switch t {
+	case TypeBlob:
+		return path.Join(r.path, blobPath)
+	case TypeRef:
+		return path.Join(r.path, refPath)
+	}
+
+	panic(fmt.Sprintf("unknown type %d", t))
+}
+
+// Construct path for given Type and ID.
+func (r *DirRepository) filename(t Type, id ID) string {
+	return path.Join(r.dir(t), id.String())
+}
+
 // PutFile saves a file's content to the repository and returns the ID.
 func (r *DirRepository) PutFile(path string) (ID, error) {
 	f, err := os.Open(path)
@@ -117,13 +175,13 @@ func (r *DirRepository) PutFile(path string) (ID, error) {
 		return nil, err
 	}
 
-	return r.Put(f)
+	return r.Put(TypeBlob, f)
 }
 
 // PutRaw saves a []byte's content to the repository and returns the ID.
-func (r *DirRepository) PutRaw(buf []byte) (ID, error) {
+func (r *DirRepository) PutRaw(t Type, buf []byte) (ID, error) {
 	// save contents to tempfile, hash while writing
-	file, err := ioutil.TempFile(path.Join(r.path, tempPath), "temp-")
+	file, err := r.tempFile()
 	if err != nil {
 		return nil, err
 	}
@@ -140,8 +198,7 @@ func (r *DirRepository) PutRaw(buf []byte) (ID, error) {
 
 	// move file to final name using hash of contents
 	id := ID(wr.Hash())
-	filename := path.Join(r.path, objectPath, id.String())
+	err = r.renameFile(file, t, id)
-	err = os.Rename(file.Name(), filename)
 	if err != nil {
 		return nil, err
 	}
@@ -150,9 +207,9 @@ func (r *DirRepository) PutRaw(buf []byte) (ID, error) {
 }
 
 // Test returns true if the given ID exists in the repository.
-func (r *DirRepository) Test(id ID) (bool, error) {
+func (r *DirRepository) Test(t Type, id ID) (bool, error) {
 	// try to open file
-	file, err := os.Open(path.Join(r.path, objectPath, id.String()))
+	file, err := os.Open(r.filename(t, id))
 	defer func() {
 		file.Close()
 	}()
@@ -168,9 +225,9 @@ func (r *DirRepository) Test(id ID) (bool, error) {
 }
 
 // Get returns a reader for the content stored under the given ID.
-func (r *DirRepository) Get(id ID) (io.Reader, error) {
+func (r *DirRepository) Get(t Type, id ID) (io.Reader, error) {
 	// try to open file
-	file, err := os.Open(path.Join(r.path, objectPath, id.String()))
+	file, err := os.Open(r.filename(t, id))
 	if err != nil {
 		return nil, err
 	}
@@ -179,60 +236,66 @@ func (r *DirRepository) Get(id ID) (io.Reader, error) {
 }
 
 // Remove removes the content stored at ID.
-func (r *DirRepository) Remove(id ID) error {
+func (r *DirRepository) Remove(t Type, id ID) error {
-	return os.Remove(path.Join(r.path, objectPath, id.String()))
+	return os.Remove(r.filename(t, id))
 }
 
-// Unlink removes a named ID.
+type IDs []ID
-func (r *DirRepository) Unlink(name string) error {
-	return os.Remove(path.Join(r.path, refPath, Name(name).Encode()))
-}
 
-// Link assigns a name to an ID. Name must be unique in this repository and ID must exist.
+// Lists all objects of a given type.
-func (r *DirRepository) Link(name string, id ID) error {
+func (r *DirRepository) ListIDs(t Type) (IDs, error) {
-	exist, err := r.Test(id)
+	// TODO: use os.Open() and d.Readdirnames() instead of Glob()
-	if err != nil {
+	pattern := path.Join(r.dir(t), "*")
-		return err
-	}
 
-	if !exist {
+	matches, err := filepath.Glob(pattern)
-		return ErrIDDoesNotExist
-	}
-
-	// create file, write id
-	f, err := os.Create(path.Join(r.path, refPath, Name(name).Encode()))
-	defer f.Close()
-
-	if err != nil {
-		return err
-	}
-
-	f.Write([]byte(hex.EncodeToString(id)))
-	return nil
-}
-
-// Resolve returns the ID associated with the given name.
-func (r *DirRepository) Resolve(name string) (ID, error) {
-	f, err := os.Open(path.Join(r.path, refPath, Name(name).Encode()))
-	defer f.Close()
 	if err != nil {
 		return nil, err
 	}
 
-	// read hex string
+	ids := make(IDs, 0, len(matches))
-	l := r.hash().Size()
+
-	buf := make([]byte, l*2)
+	for _, m := range matches {
-	_, err = io.ReadFull(f, buf)
+		base := filepath.Base(m)
+
+		if base == "" {
+			continue
+		}
+		id, err := ParseID(base)
 
 		if err != nil {
-		return nil, err
+			continue
 		}
 
-	id := make([]byte, l)
+		ids = append(ids, id)
-	_, err = hex.Decode(id, buf)
-	if err != nil {
-		return nil, err
 	}
 
-	return ID(id), nil
+	return ids, nil
+}
+
+func (ids IDs) Len() int {
+	return len(ids)
+}
+
+func (ids IDs) Less(i, j int) bool {
+	if len(ids[i]) < len(ids[j]) {
+		return true
+	}
+
+	for k, b := range ids[i] {
+		if b == ids[j][k] {
+			continue
+		}
+
+		if b < ids[j][k] {
+			return true
+		} else {
+			return false
+		}
+	}
+
+	return false
+}
+
+func (ids IDs) Swap(i, j int) {
+	ids[i], ids[j] = ids[j], ids[i]
 }

repository_test.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
+	"sort"
 	"strings"
 
 	"github.com/fd0/khepri"
@@ -14,12 +15,13 @@ import (
 
 var TestStrings = []struct {
 	id   string
+	t    khepri.Type
 	data string
 }{
-	{"c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2", "foobar"},
+	{"c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2", khepri.TypeBlob, "foobar"},
-	{"248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"},
+	{"248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1", khepri.TypeBlob, "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"},
-	{"cc5d46bdb4991c6eae3eb739c9c8a7a46fe9654fab79c47b4fe48383b5b25e1c", "foo/bar"},
+	{"cc5d46bdb4991c6eae3eb739c9c8a7a46fe9654fab79c47b4fe48383b5b25e1c", khepri.TypeRef, "foo/bar"},
-	{"4e54d2c721cbdb730f01b10b62dec622962b36966ec685880effa63d71c808f2", "foo/../../baz"},
+	{"4e54d2c721cbdb730f01b10b62dec622962b36966ec685880effa63d71c808f2", khepri.TypeBlob, "foo/../../baz"},
 }
 
 var _ = Describe("Storage", func() {
@@ -30,7 +32,7 @@ var _ = Describe("Storage", func() {
 		id      khepri.ID
 	)
 
-	BeforeEach(func() {
+	var _ = BeforeSuite(func() {
 		tempdir, err = ioutil.TempDir("", "khepri-test-")
 		if err != nil {
 			panic(err)
@@ -41,7 +43,7 @@ var _ = Describe("Storage", func() {
 		}
 	})
 
-	AfterEach(func() {
+	AfterSuite(func() {
 		err = os.RemoveAll(tempdir)
 		if err != nil {
 			panic(err)
@@ -58,7 +60,7 @@ var _ = Describe("Storage", func() {
 					Expect(err).NotTo(HaveOccurred())
 
 					// try to get string out, should fail
-					ret, err := repo.Test(id)
+					ret, err := repo.Test(test.t, id)
 					Expect(ret).Should(Equal(false))
 				}
 			})
@@ -66,44 +68,62 @@ var _ = Describe("Storage", func() {
 			It("Should Add File", func() {
 				for _, test := range TestStrings {
 					// store string in repository
-					id, err = repo.Put(strings.NewReader(test.data))
+					id, err = repo.Put(test.t, strings.NewReader(test.data))
 
 					Expect(err).NotTo(HaveOccurred())
 					Expect(id.String()).Should(Equal(test.id))
 
 					// try to get it out again
 					var buf bytes.Buffer
-					rd, err := repo.Get(id)
+					rd, err := repo.Get(test.t, id)
 					Expect(err).NotTo(HaveOccurred())
 					Expect(rd).ShouldNot(BeNil())
 
 					// compare content
 					Expect(io.Copy(&buf, rd)).Should(Equal(int64(len(test.data))))
 					Expect(buf.Bytes()).Should(Equal([]byte(test.data)))
-
-					// store id under name
-					err = repo.Link(test.data, id)
-					Expect(err).NotTo(HaveOccurred())
-
-					// resolve again
-					Expect(repo.Resolve(test.data)).Should(Equal(id))
-
-					// remove link
-					Expect(repo.Unlink(test.data)).NotTo(HaveOccurred())
-
-					// remove string
-					Expect(repo.Remove(id))
 				}
 			})
 
 			It("Should Add Buffer", func() {
 				for _, test := range TestStrings {
 					// store buf in repository
-					id, err := repo.PutRaw([]byte(test.data))
+					id, err := repo.PutRaw(test.t, []byte(test.data))
 					Expect(err).NotTo(HaveOccurred())
 					Expect(id.String()).To(Equal(test.id))
 				}
 			})
+
+			It("Should List IDs", func() {
+				for _, t := range []khepri.Type{khepri.TypeBlob, khepri.TypeRef} {
+					IDs := khepri.IDs{}
+					for _, test := range TestStrings {
+						if test.t == t {
+							id, err := khepri.ParseID(test.id)
+							Expect(err).NotTo(HaveOccurred())
+							IDs = append(IDs, id)
+						}
+					}
+
+					ids, err := repo.ListIDs(t)
+
+					sort.Sort(ids)
+					sort.Sort(IDs)
+					Expect(err).NotTo(HaveOccurred())
+					Expect(ids).Should(Equal(IDs))
+				}
+			})
+
+			It("Should Remove Content", func() {
+				for _, test := range TestStrings {
+					id, err := khepri.ParseID(test.id)
+					Expect(err).ShouldNot(HaveOccurred())
+					Expect(repo.Test(test.t, id)).To(Equal(true))
+					Expect(repo.Remove(test.t, id))
+					Expect(repo.Test(test.t, id)).To(Equal(false))
+				}
 			})
 		})
+	})
+
 })