diff --git a/storage/storage.go b/storage/storage.go new file mode 100644 index 000000000..eb44ef210 --- /dev/null +++ b/storage/storage.go @@ -0,0 +1,231 @@ +package storage + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "errors" + "hash" + "io" + "io/ioutil" + "net/url" + "os" + "path" + + "github.com/fd0/khepri/hashing" +) + +const ( + dirMode = 0700 + objectPath = "objects" + refPath = "refs" + tempPath = "tmp" +) + +type Repository interface { + Put(reader io.Reader) (ID, error) + PutFile(path string) (ID, error) + Get(ID) (io.Reader, error) + Test(ID) (bool, error) + Link(name string, id ID) error + Unlink(name string) error + Resolve(name string) (ID, error) +} + +var ( + ErrIDDoesNotExist = errors.New("ID does not exist") +) + +// References content within a repository. +type ID []byte + +func (id ID) String() string { + return hex.EncodeToString(id) +} + +// Equal compares an ID to another other. +func (id ID) Equal(other ID) bool { + return bytes.Equal(id, other) +} + +// EqualString compares this ID to another one, given as a string. +func (id ID) EqualString(other string) (bool, error) { + s, err := hex.DecodeString(other) + if err != nil { + return false, err + } + + return id.Equal(ID(s)), nil +} + +// Name stands for the alias given to an ID. +type Name string + +func (n Name) Encode() string { + return url.QueryEscape(string(n)) +} + +type Dir struct { + path string + hash func() hash.Hash +} + +// NewDir creates a new dir-baked repository at the given path. +func NewDir(path string) (*Dir, error) { + d := &Dir{ + path: path, + hash: sha256.New, + } + + err := d.create() + + if err != nil { + return nil, err + } + + return d, nil +} + +func (r *Dir) create() error { + dirs := []string{ + r.path, + path.Join(r.path, objectPath), + path.Join(r.path, refPath), + path.Join(r.path, tempPath), + } + + for _, dir := range dirs { + err := os.MkdirAll(dir, dirMode) + if err != nil { + return err + } + } + + return nil +} + +// SetHash changes the hash function used for deriving IDs. Default is SHA256. +func (r *Dir) SetHash(h func() hash.Hash) { + r.hash = h +} + +// Path returns the directory used for this repository. +func (r *Dir) Path() string { + return r.path +} + +// Put saves content and returns the ID. +func (r *Dir) Put(reader io.Reader) (ID, error) { + // save contents to tempfile, hash while writing + file, err := ioutil.TempFile(path.Join(r.path, tempPath), "temp-") + if err != nil { + return nil, err + } + + rd := hashing.NewReader(reader, r.hash) + _, err = io.Copy(file, rd) + if err != nil { + return nil, err + } + + err = file.Close() + if err != nil { + return nil, err + } + + // move file to final name using hash of contents + id := ID(rd.Hash()) + filename := path.Join(r.path, objectPath, id.String()) + err = os.Rename(file.Name(), filename) + if err != nil { + return nil, err + } + + return id, nil +} + +// PutFile saves a file's content to the repository and returns the ID. +func (r *Dir) PutFile(path string) (ID, error) { + f, err := os.Open(path) + defer f.Close() + if err != nil { + return nil, err + } + + return r.Put(f) +} + +// Test returns true if the given ID exists in the repository. +func (r *Dir) Test(id ID) (bool, error) { + // try to open file + file, err := os.Open(path.Join(r.path, objectPath, id.String())) + defer func() { + file.Close() + }() + + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + + return true, nil +} + +// Get returns a reader for the content stored under the given ID. +func (r *Dir) Get(id ID) (io.Reader, error) { + // try to open file + file, err := os.Open(path.Join(r.path, objectPath, id.String())) + if err != nil { + return nil, err + } + + return file, nil +} + +// Unlink removes a named ID. +func (r *Dir) Unlink(name string) error { + return os.Remove(path.Join(r.path, refPath, Name(name).Encode())) +} + +// Link assigns a name to an ID. Name must be unique in this repository and ID must exist. +func (r *Dir) Link(name string, id ID) error { + exist, err := r.Test(id) + if err != nil { + return err + } + + if !exist { + return ErrIDDoesNotExist + } + + // create file, write id + f, err := os.Create(path.Join(r.path, refPath, Name(name).Encode())) + defer f.Close() + + if err != nil { + return err + } + + f.Write(id) + return nil +} + +// Resolve returns the ID associated with the given name. +func (r *Dir) Resolve(name string) (ID, error) { + f, err := os.Open(path.Join(r.path, refPath, Name(name).Encode())) + defer f.Close() + if err != nil { + return nil, err + } + + id := make([]byte, r.hash().Size()) + _, err = io.ReadFull(f, id) + + if err != nil { + return nil, err + } + + return ID(id), nil +} diff --git a/storage/storage_suite_test.go b/storage/storage_suite_test.go new file mode 100644 index 000000000..c368e3f22 --- /dev/null +++ b/storage/storage_suite_test.go @@ -0,0 +1,13 @@ +package storage_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "testing" +) + +func TestStorage(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Storage Suite") +} diff --git a/storage/storage_test.go b/storage/storage_test.go new file mode 100644 index 000000000..7bd2414bd --- /dev/null +++ b/storage/storage_test.go @@ -0,0 +1,98 @@ +package storage_test + +import ( + "bytes" + "encoding/hex" + "io" + "io/ioutil" + "os" + "strings" + + "github.com/fd0/khepri/storage" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var TestStrings = []struct { + id string + data string +}{ + {"c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2", "foobar"}, + {"248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, + {"cc5d46bdb4991c6eae3eb739c9c8a7a46fe9654fab79c47b4fe48383b5b25e1c", "foo/bar"}, + {"4e54d2c721cbdb730f01b10b62dec622962b36966ec685880effa63d71c808f2", "foo/../../baz"}, +} + +var _ = Describe("Storage", func() { + var ( + tempdir string + repo storage.Repository + err error + id storage.ID + ) + + BeforeEach(func() { + tempdir, err = ioutil.TempDir("", "khepri-test-") + if err != nil { + panic(err) + } + repo, err = storage.NewDir(tempdir) + if err != nil { + panic(err) + } + }) + + AfterEach(func() { + err = os.RemoveAll(tempdir) + if err != nil { + panic(err) + } + // fmt.Fprintf(os.Stderr, "leaving tempdir %s", tempdir) + tempdir = "" + }) + + Describe("Repository", func() { + Context("File Operations", func() { + It("Should detect non-existing file", func() { + for _, test := range TestStrings { + id, err := hex.DecodeString(test.id) + Expect(err).NotTo(HaveOccurred()) + + // try to get string out, should fail + ret, err := repo.Test(id) + Expect(ret).Should(Equal(false)) + } + }) + + It("Should Add File", func() { + for _, test := range TestStrings { + // store string in repository + id, err = repo.Put(strings.NewReader(test.data)) + + Expect(err).NotTo(HaveOccurred()) + Expect(id.String()).Should(Equal(test.id)) + + // try to get it out again + var buf bytes.Buffer + rd, err := repo.Get(id) + Expect(err).NotTo(HaveOccurred()) + Expect(rd).ShouldNot(BeNil()) + + // compare content + Expect(io.Copy(&buf, rd)).Should(Equal(int64(len(test.data)))) + Expect(buf.Bytes()).Should(Equal([]byte(test.data))) + + // store id under name + err = repo.Link(test.data, id) + Expect(err).NotTo(HaveOccurred()) + + // resolve again + Expect(repo.Resolve(test.data)).Should(Equal(id)) + + // remove link + Expect(repo.Unlink(test.data)).NotTo(HaveOccurred()) + } + }) + }) + }) +})