octoleo/restic
2
2
Fork 0
mirror of https://github.com/octoleo/restic.git synced 2024-12-22 10:58:55 +00:00
Code Issues Releases Wiki Activity

Add decrypt, refactor

Browse Source
This commit is contained in:
Alexander Neumann 2014-09-23 22:39:12 +02:00
parent 83ea81d8c3
commit 30ab03b7b7
37 changed files with 2572 additions and 1046 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

189
archiver.go Normal file
View File

@ -0,0 +1,189 @@
package khepri
import (
"os"
"path/filepath"
"github.com/fd0/khepri/backend"
)
type Archiver struct {
be backend.Server
key *Key
ch *ContentHandler
smap *StorageMap // blobs used for the current snapshot
Error func(dir string, fi os.FileInfo, err error) error
Filter func(item string, fi os.FileInfo) bool
}
func NewArchiver(be backend.Server, key *Key) (*Archiver, error) {
var err error
arch := &Archiver{be: be, key: key}
// abort on all errors
arch.Error = func(string, os.FileInfo, error) error { return err }
// allow all files
arch.Filter = func(string, os.FileInfo) bool { return true }
arch.smap = NewStorageMap()
arch.ch, err = NewContentHandler(be, key)
if err != nil {
return nil, err
}
// load all blobs from all snapshots
err = arch.ch.LoadAllSnapshots()
if err != nil {
return nil, err
}
return arch, nil
}
func (arch *Archiver) Save(t backend.Type, data []byte) (*Blob, error) {
blob, err := arch.ch.Save(t, data)
if err != nil {
return nil, err
}
// store blob in storage map for current snapshot
arch.smap.Insert(blob)
return blob, nil
}
func (arch *Archiver) SaveJSON(t backend.Type, item interface{}) (*Blob, error) {
blob, err := arch.ch.SaveJSON(t, item)
if err != nil {
return nil, err
}
// store blob in storage map for current snapshot
arch.smap.Insert(blob)
return blob, nil
}
func (arch *Archiver) SaveFile(node *Node) (Blobs, error) {
blobs, err := arch.ch.SaveFile(node.path, uint(node.Size))
if err != nil {
return nil, arch.Error(node.path, nil, err)
}
node.Content = make([]backend.ID, len(blobs))
for i, blob := range blobs {
node.Content[i] = blob.ID
arch.smap.Insert(blob)
}
return blobs, err
}
func (arch *Archiver) ImportDir(dir string) (Tree, error) {
fd, err := os.Open(dir)
defer fd.Close()
if err != nil {
return nil, arch.Error(dir, nil, err)
}
entries, err := fd.Readdir(-1)
if err != nil {
return nil, arch.Error(dir, nil, err)
}
if len(entries) == 0 {
return nil, nil
}
tree := Tree{}
for _, entry := range entries {
path := filepath.Join(dir, entry.Name())
if !arch.Filter(path, entry) {
continue
}
node, err := NodeFromFileInfo(path, entry)
if err != nil {
return nil, arch.Error(dir, entry, err)
}
tree = append(tree, node)
if entry.IsDir() {
subtree, err := arch.ImportDir(path)
if err != nil {
return nil, err
}
blob, err := arch.SaveJSON(backend.Tree, subtree)
if err != nil {
return nil, err
}
node.Subtree = blob.ID
continue
}
if node.Type == "file" {
_, err := arch.SaveFile(node)
if err != nil {
return nil, arch.Error(path, entry, err)
}
}
}
return tree, nil
}
func (arch *Archiver) Import(dir string) (*Snapshot, *Blob, error) {
sn := NewSnapshot(dir)
fi, err := os.Lstat(dir)
if err != nil {
return nil, nil, err
}
node, err := NodeFromFileInfo(dir, fi)
if err != nil {
return nil, nil, err
}
if node.Type == "dir" {
tree, err := arch.ImportDir(dir)
if err != nil {
return nil, nil, err
}
blob, err := arch.SaveJSON(backend.Tree, tree)
if err != nil {
return nil, nil, err
}
node.Subtree = blob.ID
} else if node.Type == "file" {
_, err := arch.SaveFile(node)
if err != nil {
return nil, nil, err
}
}
blob, err := arch.SaveJSON(backend.Tree, &Tree{node})
if err != nil {
return nil, nil, err
}
sn.Content = blob.ID
// save snapshot
sn.StorageMap = arch.smap
blob, err = arch.SaveJSON(backend.Snapshot, sn)
if err != nil {
return nil, nil, err
}
return sn, blob, nil
}

2
backend/doc.go Normal file
View File

@ -0,0 +1,2 @@
// Package backend provides local and remote storage for khepri backups.
package backend

84
backend/generic.go Normal file
View File

@ -0,0 +1,84 @@
package backend
import (
"bytes"
"compress/zlib"
"crypto/sha256"
"io/ioutil"
)
// Each lists all entries of type t in the backend and calls function f() with
// the id and data.
func Each(be Server, t Type, f func(id ID, data []byte, err error)) error {
ids, err := be.List(t)
if err != nil {
return err
}
for _, id := range ids {
data, err := be.Get(t, id)
if err != nil {
f(id, nil, err)
continue
}
f(id, data, nil)
}
return nil
}
// Each lists all entries of type t in the backend and calls function f() with
// the id.
func EachID(be Server, t Type, f func(ID)) error {
ids, err := be.List(t)
if err != nil {
return err
}
for _, id := range ids {
f(id)
}
return nil
}
// Compress applies zlib compression to data.
func Compress(data []byte) []byte {
// apply zlib compression
var b bytes.Buffer
w := zlib.NewWriter(&b)
_, err := w.Write(data)
if err != nil {
panic(err)
}
w.Close()
return b.Bytes()
}
// Uncompress reverses zlib compression on data.
func Uncompress(data []byte) []byte {
b := bytes.NewBuffer(data)
r, err := zlib.NewReader(b)
if err != nil {
panic(err)
}
buf, err := ioutil.ReadAll(r)
if err != nil {
panic(err)
}
r.Close()
return buf
}
// Hash returns the ID for data.
func Hash(data []byte) ID {
h := sha256.Sum256(data)
id := make(ID, 32)
copy(id, h[:])
return id
}

36
backend/generic_test.go Normal file
View File

@ -0,0 +1,36 @@
package backend_test
import (
"fmt"
"path/filepath"
"reflect"
"runtime"
"testing"
)
// assert fails the test if the condition is false.
func assert(tb testing.TB, condition bool, msg string, v ...interface{}) {
if !condition {
_, file, line, _ := runtime.Caller(1)
fmt.Printf("\033[31m%s:%d: "+msg+"\033[39m\n\n", append([]interface{}{filepath.Base(file), line}, v...)...)
tb.FailNow()
}
}
// ok fails the test if an err is not nil.
func ok(tb testing.TB, err error) {
if err != nil {
_, file, line, _ := runtime.Caller(1)
fmt.Printf("\033[31m%s:%d: unexpected error: %s\033[39m\n\n", filepath.Base(file), line, err.Error())
tb.FailNow()
}
}
// equals fails the test if exp is not equal to act.
func equals(tb testing.TB, exp, act interface{}) {
if !reflect.DeepEqual(exp, act) {
_, file, line, _ := runtime.Caller(1)
fmt.Printf("\033[31m%s:%d:\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", filepath.Base(file), line, exp, act)
tb.FailNow()
}
}

41
id.go → backend/id.go
View File

@ -1,12 +1,15 @@
package khepri
package backend
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
)
const sha256_length = 32 // in bytes
// References content within a repository.
type ID []byte
@ -18,6 +21,10 @@ func ParseID(s string) (ID, error) {
return nil, err
}
if len(b) != sha256_length {
return nil, errors.New("invalid length for sha256 hash")
}
return ID(b), nil
}
@ -62,7 +69,37 @@ func (id *ID) UnmarshalJSON(b []byte) error {
func IDFromData(d []byte) ID {
hash := sha256.Sum256(d)
id := make([]byte, 32)
id := make([]byte, sha256_length)
copy(id, hash[:])
return id
}
type IDs []ID
func (ids IDs) Len() int {
return len(ids)
}
func (ids IDs) Less(i, j int) bool {
if len(ids[i]) < len(ids[j]) {
return true
}
for k, b := range ids[i] {
if b == ids[j][k] {
continue
}
if b < ids[j][k] {
return true
} else {
return false
}
}
return false
}
func (ids IDs) Swap(i, j int) {
ids[i], ids[j] = ids[j], ids[i]
}

21
backend/interface.go Normal file
View File

@ -0,0 +1,21 @@
package backend
type Type string
const (
Blob Type = "blob"
Key = "key"
Lock = "lock"
Snapshot = "snapshot"
Tree = "tree"
)
type Server interface {
Create(Type, []byte) (ID, error)
Get(Type, ID) ([]byte, error)
List(Type) (IDs, error)
Test(Type, ID) (bool, error)
Remove(Type, ID) error
Location() string
}

213
backend/local.go Normal file
View File

@ -0,0 +1,213 @@
package backend
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
)
const (
dirMode = 0700
blobPath = "blobs"
snapshotPath = "snapshots"
treePath = "trees"
lockPath = "locks"
keyPath = "keys"
tempPath = "tmp"
)
type Local struct {
p string
}
// OpenLocal opens the local backend at dir.
func OpenLocal(dir string) (*Local, error) {
items := []string{
dir,
filepath.Join(dir, blobPath),
filepath.Join(dir, snapshotPath),
filepath.Join(dir, treePath),
filepath.Join(dir, lockPath),
filepath.Join(dir, keyPath),
filepath.Join(dir, tempPath),
}
// test if all necessary dirs and files are there
for _, d := range items {
if _, err := os.Stat(d); err != nil {
return nil, fmt.Errorf("%s does not exist", d)
}
}
return &Local{p: dir}, nil
}
// CreateLocal creates all the necessary files and directories for a new local
// backend at dir.
func CreateLocal(dir string) (*Local, error) {
dirs := []string{
dir,
filepath.Join(dir, blobPath),
filepath.Join(dir, snapshotPath),
filepath.Join(dir, treePath),
filepath.Join(dir, lockPath),
filepath.Join(dir, keyPath),
filepath.Join(dir, tempPath),
}
// test if directories already exist
for _, d := range dirs[1:] {
if _, err := os.Stat(d); err == nil {
return nil, fmt.Errorf("dir %s already exists", d)
}
}
// create paths for blobs, refs and temp
for _, d := range dirs {
err := os.MkdirAll(d, dirMode)
if err != nil {
return nil, err
}
}
// open repository
return OpenLocal(dir)
}
// Location returns this backend's location (the directory name).
func (b *Local) Location() string {
return b.p
}
// Return temp directory in correct directory for this backend.
func (b *Local) tempFile() (*os.File, error) {
return ioutil.TempFile(filepath.Join(b.p, tempPath), "temp-")
}
// Rename temp file to final name according to type and ID.
func (b *Local) renameFile(file *os.File, t Type, id ID) error {
filename := filepath.Join(b.dir(t), id.String())
return os.Rename(file.Name(), filename)
}
// Construct directory for given Type.
func (b *Local) dir(t Type) string {
var n string
switch t {
case Blob:
n = blobPath
case Snapshot:
n = snapshotPath
case Tree:
n = treePath
case Lock:
n = lockPath
case Key:
n = keyPath
}
return filepath.Join(b.p, n)
}
// Create stores new content of type t and data and returns the ID.
func (b *Local) Create(t Type, data []byte) (ID, error) {
// TODO: make sure that tempfile is removed upon error
// create tempfile in repository
var err error
file, err := b.tempFile()
if err != nil {
return nil, err
}
// write data to tempfile
_, err = file.Write(data)
if err != nil {
return nil, err
}
// close tempfile, return id
id := IDFromData(data)
err = b.renameFile(file, t, id)
if err != nil {
return nil, err
}
return id, nil
}
// Construct path for given Type and ID.
func (b *Local) filename(t Type, id ID) string {
return filepath.Join(b.dir(t), id.String())
}
// Get returns the content stored under the given ID.
func (b *Local) Get(t Type, id ID) ([]byte, error) {
// try to open file
file, err := os.Open(b.filename(t, id))
defer file.Close()
if err != nil {
return nil, err
}
// read all
buf, err := ioutil.ReadAll(file)
if err != nil {
return nil, err
}
return buf, nil
}
// Test returns true if a blob of the given type and ID exists in the backend.
func (b *Local) Test(t Type, id ID) (bool, error) {
// try to open file
file, err := os.Open(b.filename(t, id))
defer func() {
file.Close()
}()
if err != nil {
if os.IsNotExist(err) {
return false, nil
}
return false, err
}
return true, nil
}
// Remove removes the content stored at ID.
func (b *Local) Remove(t Type, id ID) error {
return os.Remove(b.filename(t, id))
}
// List lists all objects of a given type.
func (b *Local) List(t Type) (IDs, error) {
// TODO: use os.Open() and d.Readdirnames() instead of Glob()
pattern := filepath.Join(b.dir(t), "*")
matches, err := filepath.Glob(pattern)
if err != nil {
return nil, err
}
ids := make(IDs, 0, len(matches))
for _, m := range matches {
base := filepath.Base(m)
if base == "" {
continue
}
id, err := ParseID(base)
if err != nil {
continue
}
ids = append(ids, id)
}
return ids, nil
}

171
backend/local_test.go Normal file
View File

@ -0,0 +1,171 @@
package backend_test
import (
"flag"
"fmt"
"io/ioutil"
"os"
"sort"
"testing"
"github.com/fd0/khepri/backend"
)
var testCleanup = flag.Bool("test.cleanup", true, "clean up after running tests (remove local backend directory with all content)")
var TestStrings = []struct {
id string
data string
}{
{"c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2", "foobar"},
{"248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"},
{"cc5d46bdb4991c6eae3eb739c9c8a7a46fe9654fab79c47b4fe48383b5b25e1c", "foo/bar"},
{"4e54d2c721cbdb730f01b10b62dec622962b36966ec685880effa63d71c808f2", "foo/../../baz"},
}
func setupBackend(t *testing.T) *backend.Local {
tempdir, err := ioutil.TempDir("", "khepri-test-")
ok(t, err)
b, err := backend.CreateLocal(tempdir)
ok(t, err)
t.Logf("created local backend at %s", tempdir)
return b
}
func teardownBackend(t *testing.T, b *backend.Local) {
if !*testCleanup {
t.Logf("leaving local backend at %s\n", b.Location())
return
}
ok(t, os.RemoveAll(b.Location()))
}
func testBackend(b backend.Server, t *testing.T) {
for _, tpe := range []backend.Type{backend.Blob, backend.Key, backend.Lock, backend.Snapshot, backend.Tree} {
// detect non-existing files
for _, test := range TestStrings {
id, err := backend.ParseID(test.id)
ok(t, err)
// test if blob is already in repository
ret, err := b.Test(tpe, id)
ok(t, err)
assert(t, !ret, "blob was found to exist before creating")
// try to open not existing blob
d, err := b.Get(tpe, id)
assert(t, err != nil && d == nil, "blob data could be extracted befor creation")
// try to get string out, should fail
ret, err = b.Test(tpe, id)
ok(t, err)
assert(t, !ret, fmt.Sprintf("id %q was found (but should not have)", test.id))
}
// add files
for _, test := range TestStrings {
// store string in backend
id, err := b.Create(tpe, []byte(test.data))
ok(t, err)
equals(t, test.id, id.String())
// try to get it out again
buf, err := b.Get(tpe, id)
ok(t, err)
assert(t, buf != nil, "Get() returned nil")
// compare content
equals(t, test.data, string(buf))
}
// list items
IDs := backend.IDs{}
for _, test := range TestStrings {
id, err := backend.ParseID(test.id)
ok(t, err)
IDs = append(IDs, id)
}
ids, err := b.List(tpe)
ok(t, err)
sort.Sort(ids)
sort.Sort(IDs)
equals(t, IDs, ids)
// remove content if requested
if *testCleanup {
for _, test := range TestStrings {
id, err := backend.ParseID(test.id)
ok(t, err)
found, err := b.Test(tpe, id)
ok(t, err)
assert(t, found, fmt.Sprintf("id %q was not found before removal"))
ok(t, b.Remove(tpe, id))
found, err = b.Test(tpe, id)
ok(t, err)
assert(t, !found, fmt.Sprintf("id %q was not found before removal"))
}
}
}
}
func TestBackend(t *testing.T) {
// test for non-existing backend
b, err := backend.OpenLocal("/invalid-khepri-test")
assert(t, err != nil, "opening invalid repository at /invalid-khepri-test should have failed, but err is nil")
assert(t, b == nil, fmt.Sprintf("opening invalid repository at /invalid-khepri-test should have failed, but b is not nil: %v", b))
b = setupBackend(t)
defer teardownBackend(t, b)
testBackend(b, t)
}
func TestLocalBackendCreationFailures(t *testing.T) {
b := setupBackend(t)
defer teardownBackend(t, b)
// test failure to create a new repository at the same location
b2, err := backend.CreateLocal(b.Location())
assert(t, err != nil && b2 == nil, fmt.Sprintf("creating a repository at %s for the second time should have failed", b.Location()))
// test failure to create a new repository at the same location without a config file
b2, err = backend.CreateLocal(b.Location())
assert(t, err != nil && b2 == nil, fmt.Sprintf("creating a repository at %s for the second time should have failed", b.Location()))
}
func TestID(t *testing.T) {
for _, test := range TestStrings {
id, err := backend.ParseID(test.id)
ok(t, err)
id2, err := backend.ParseID(test.id)
ok(t, err)
assert(t, id.Equal(id2), "ID.Equal() does not work as expected")
ret, err := id.EqualString(test.id)
ok(t, err)
assert(t, ret, "ID.EqualString() returned wrong value")
// test json marshalling
buf, err := id.MarshalJSON()
ok(t, err)
equals(t, "\""+test.id+"\"", string(buf))
var id3 backend.ID
err = id3.UnmarshalJSON(buf)
ok(t, err)
equals(t, id, id3)
}
}

81
cmd/archive/main.go Normal file
View File

@ -0,0 +1,81 @@
package main
import (
"fmt"
"os"
"github.com/fd0/khepri"
"github.com/fd0/khepri/backend"
)
const pass = "foobar"
func main() {
if len(os.Args) != 3 {
fmt.Fprintf(os.Stderr, "usage: archive REPO DIR\n")
os.Exit(1)
}
repo := os.Args[1]
dir := os.Args[2]
// fmt.Printf("import %s into backend %s\n", dir, repo)
var (
be backend.Server
key *khepri.Key
)
be, err := backend.OpenLocal(repo)
if err != nil {
fmt.Printf("creating %s\n", repo)
be, err = backend.CreateLocal(repo)
if err != nil {
fmt.Fprintf(os.Stderr, "failed: %v\n", err)
os.Exit(2)
}
key, err = khepri.CreateKey(be, pass)
if err != nil {
fmt.Fprintf(os.Stderr, "failed: %v\n", err)
os.Exit(2)
}
}
key, err = khepri.SearchKey(be, pass)
if err != nil {
fmt.Fprintf(os.Stderr, "failed: %v\n", err)
os.Exit(2)
}
arch, err := khepri.NewArchiver(be, key)
if err != nil {
fmt.Fprintf(os.Stderr, "err: %v\n", err)
}
arch.Error = func(dir string, fi os.FileInfo, err error) error {
fmt.Fprintf(os.Stderr, "error for %s: %v\n%s\n", dir, err, fi)
return err
}
arch.Filter = func(item string, fi os.FileInfo) bool {
// if fi.IsDir() {
// if fi.Name() == ".svn" {
// return false
// }
// } else {
// if filepath.Ext(fi.Name()) == ".bz2" {
// return false
// }
// }
fmt.Printf("%s\n", item)
return true
}
_, blob, err := arch.Import(dir)
if err != nil {
fmt.Fprintf(os.Stderr, "Import() error: %v\n", err)
os.Exit(2)
}
fmt.Printf("saved as %+v\n", blob)
}

126
cmd/cat/main.go Normal file
View File

@ -0,0 +1,126 @@
package main
import (
"bytes"
"encoding/json"
"fmt"
"os"
"path/filepath"
"time"
"code.google.com/p/go.crypto/ssh/terminal"
"github.com/fd0/khepri"
"github.com/fd0/khepri/backend"
)
func read_password(prompt string) string {
p := os.Getenv("KHEPRI_PASSWORD")
if p != "" {
return p
}
fmt.Print(prompt)
pw, err := terminal.ReadPassword(int(os.Stdin.Fd()))
if err != nil {
fmt.Fprintf(os.Stderr, "unable to read password: %v", err)
os.Exit(2)
}
fmt.Println()
return string(pw)
}
func json_pp(data []byte) error {
var buf bytes.Buffer
err := json.Indent(&buf, data, "", " ")
if err != nil {
return err
}
fmt.Println(string(buf.Bytes()))
return nil
}
type StopWatch struct {
start, last time.Time
}
func NewStopWatch() *StopWatch {
return &StopWatch{
start: time.Now(),
last: time.Now(),
}
}
func (s *StopWatch) Next(format string, data ...interface{}) {
t := time.Now()
d := t.Sub(s.last)
s.last = t
arg := make([]interface{}, len(data)+1)
arg[0] = d
copy(arg[1:], data)
fmt.Printf("[%s]: "+format+"\n", arg...)
}
func main() {
if len(os.Args) != 3 {
fmt.Fprintf(os.Stderr, "usage: cat REPO ID\n")
os.Exit(1)
}
repo := os.Args[1]
id, err := backend.ParseID(filepath.Base(os.Args[2]))
if err != nil {
panic(err)
}
s := NewStopWatch()
be, err := backend.OpenLocal(repo)
if err != nil {
fmt.Fprintf(os.Stderr, "failed: %v\n", err)
os.Exit(1)
}
s.Next("OpenLocal()")
key, err := khepri.SearchKey(be, read_password("Enter Password for Repository: "))
if err != nil {
fmt.Fprintf(os.Stderr, "failed: %v\n", err)
os.Exit(2)
}
s.Next("SearchKey()")
// try all possible types
for _, t := range []backend.Type{backend.Blob, backend.Snapshot, backend.Lock, backend.Tree, backend.Key} {
buf, err := be.Get(t, id)
if err != nil {
continue
}
s.Next("Get(%s, %s)", t, id)
if t == backend.Key {
json_pp(buf)
}
buf2, err := key.Decrypt(buf)
if err != nil {
panic(err)
}
if t == backend.Blob {
// directly output blob
fmt.Println(string(buf2))
} else {
// try to uncompress and print as idented json
err = json_pp(backend.Uncompress(buf2))
if err != nil {
fmt.Fprintf(os.Stderr, "failed: %v\n", err)
}
}
break
}
}

233
cmd/decrypt/main.go Normal file
View File

@ -0,0 +1,233 @@
package main
import (
"bytes"
"encoding/hex"
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"crypto/aes"
"crypto/cipher"
"crypto/hmac"
"crypto/rand"
"crypto/sha256"
"code.google.com/p/go.crypto/scrypt"
"code.google.com/p/go.crypto/ssh/terminal"
"github.com/jessevdk/go-flags"
)
const (
scrypt_N = 65536
scrypt_r = 8
scrypt_p = 1
aesKeySize = 32 // for AES256
)
var Opts struct {
Password string `short:"p" long:"password" description:"Password for the file"`
Keys string `short:"k" long:"keys" description:"Keys for the file (encryption_key || sign_key, hex-encoded)"`
Salt string `short:"s" long:"salt" description:"Salt to use (hex-encoded)"`
}
func newIV() ([]byte, error) {
buf := make([]byte, aes.BlockSize)
_, err := io.ReadFull(rand.Reader, buf)
if err != nil {
return nil, err
}
return buf, nil
}
func pad(plaintext []byte) []byte {
l := aes.BlockSize - (len(plaintext) % aes.BlockSize)
if l == 0 {
l = aes.BlockSize
}
if l <= 0 || l > aes.BlockSize {
panic("invalid padding size")
}
return append(plaintext, bytes.Repeat([]byte{byte(l)}, l)...)
}
func unpad(plaintext []byte) []byte {
l := len(plaintext)
pad := plaintext[l-1]
if pad > aes.BlockSize {
panic(errors.New("padding > BlockSize"))
}
if pad == 0 {
panic(errors.New("invalid padding 0"))
}
for i := l - int(pad); i < l; i++ {
if plaintext[i] != pad {
panic(errors.New("invalid padding!"))
}
}
return plaintext[:l-int(pad)]
}
// Encrypt encrypts and signs data. Returned is IV || Ciphertext || HMAC. For
// the hash function, SHA256 is used, so the overhead is 16+32=48 byte.
func Encrypt(ekey, skey []byte, plaintext []byte) ([]byte, error) {
iv, err := newIV()
if err != nil {
panic(fmt.Sprintf("unable to generate new random iv: %v", err))
}
c, err := aes.NewCipher(ekey)
if err != nil {
panic(fmt.Sprintf("unable to create cipher: %v", err))
}
e := cipher.NewCBCEncrypter(c, iv)
p := pad(plaintext)
ciphertext := make([]byte, len(p))
e.CryptBlocks(ciphertext, p)
ciphertext = append(iv, ciphertext...)
hm := hmac.New(sha256.New, skey)
n, err := hm.Write(ciphertext)
if err != nil || n != len(ciphertext) {
panic(fmt.Sprintf("unable to calculate hmac of ciphertext: %v", err))
}
return hm.Sum(ciphertext), nil
}
// Decrypt verifes and decrypts the ciphertext. Ciphertext must be in the form
// IV || Ciphertext || HMAC.
func Decrypt(ekey, skey []byte, ciphertext []byte) ([]byte, error) {
hm := hmac.New(sha256.New, skey)
// extract hmac
l := len(ciphertext) - hm.Size()
ciphertext, mac := ciphertext[:l], ciphertext[l:]
// calculate new hmac
n, err := hm.Write(ciphertext)
if err != nil || n != len(ciphertext) {
panic(fmt.Sprintf("unable to calculate hmac of ciphertext, err %v", err))
}
// verify hmac
mac2 := hm.Sum(nil)
if !hmac.Equal(mac, mac2) {
panic("HMAC verification failed")
}
// extract iv
iv, ciphertext := ciphertext[:aes.BlockSize], ciphertext[aes.BlockSize:]
// decrypt data
c, err := aes.NewCipher(ekey)
if err != nil {
panic(fmt.Sprintf("unable to create cipher: %v", err))
}
// decrypt
e := cipher.NewCBCDecrypter(c, iv)
plaintext := make([]byte, len(ciphertext))
e.CryptBlocks(plaintext, ciphertext)
// remove padding and return
return unpad(plaintext), nil
}
func errx(code int, format string, data ...interface{}) {
if len(format) > 0 && format[len(format)-1] != '\n' {
format += "\n"
}
fmt.Fprintf(os.Stderr, format, data...)
os.Exit(code)
}
func read_password(prompt string) string {
p := os.Getenv("KHEPRI_PASSWORD")
if p != "" {
return p
}
fmt.Print(prompt)
pw, err := terminal.ReadPassword(int(os.Stdin.Fd()))
if err != nil {
errx(2, "unable to read password: %v", err)
}
fmt.Println()
return string(pw)
}
func main() {
args, err := flags.Parse(&Opts)
if e, ok := err.(*flags.Error); ok && e.Type == flags.ErrHelp {
os.Exit(0)
}
var keys []byte
if Opts.Password == "" && Opts.Keys == "" {
Opts.Password = read_password("password: ")
salt, err := hex.DecodeString(Opts.Salt)
if err != nil {
errx(1, "unable to hex-decode salt: %v", err)
}
keys, err = scrypt.Key([]byte(Opts.Password), salt, scrypt_N, scrypt_r, scrypt_p, 2*aesKeySize)
if err != nil {
errx(1, "scrypt: %v", err)
}
}
if Opts.Keys != "" {
keys, err = hex.DecodeString(Opts.Keys)
if err != nil {
errx(1, "unable to hex-decode keys: %v", err)
}
}
if len(keys) != 2*aesKeySize {
errx(2, "key length is not 512")
}
encrypt_key := keys[:aesKeySize]
sign_key := keys[aesKeySize:]
for _, filename := range args {
f, err := os.Open(filename)
defer f.Close()
if err != nil {
errx(3, "%v\n", err)
}
buf, err := ioutil.ReadAll(f)
if err != nil {
errx(3, "%v\n", err)
}
buf, err = Decrypt(encrypt_key, sign_key, buf)
if err != nil {
errx(3, "%v\n", err)
}
_, err = os.Stdout.Write(buf)
if err != nil {
errx(3, "%v\n", err)
}
}
}

3
cmd/dirdiff/main.go
View File

@ -53,14 +53,17 @@ func walk(dir string) <-chan *entry {
func (e *entry) equals(other *entry) bool {
if e.path != other.path {
fmt.Printf("path does not match\n")
return false
}
if e.fi.Mode() != other.fi.Mode() {
fmt.Printf("mode does not match\n")
return false
}
if e.fi.ModTime() != other.fi.ModTime() {
fmt.Printf("ModTime does not match\n")
return false
}

33
cmd/khepri/cmd_backup.go
View File

@ -3,37 +3,34 @@ package main
import (
"errors"
"fmt"
"log"
"os"
"github.com/fd0/khepri"
"github.com/fd0/khepri/backend"
)
func commandBackup(repo *khepri.Repository, args []string) error {
func commandBackup(be backend.Server, key *khepri.Key, args []string) error {
if len(args) != 1 {
return errors.New("usage: backup dir")
return errors.New("usage: backup [dir|file]")
}
target := args[0]
tree, err := khepri.NewTreeFromPath(repo, target)
arch, err := khepri.NewArchiver(be, key)
if err != nil {
fmt.Fprintf(os.Stderr, "err: %v\n", err)
}
arch.Error = func(dir string, fi os.FileInfo, err error) error {
fmt.Fprintf(os.Stderr, "error for %s: %v\n%s\n", dir, err, fi)
return err
}
_, blob, err := arch.Import(target)
if err != nil {
return err
}
id, err := tree.Save(repo)
if err != nil {
return err
}
sn := khepri.NewSnapshot(target)
sn.Content = id
snid, err := sn.Save(repo)
if err != nil {
log.Printf("error saving snapshopt: %v", err)
}
fmt.Printf("%q archived as %v\n", target, snid)
fmt.Printf("snapshot %s saved\n", blob.Storage)
return nil
}

90
cmd/khepri/cmd_dump.go
View File

@ -1,90 +0,0 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"io"
"log"
"os"
"github.com/fd0/khepri"
)
func dump_tree(repo *khepri.Repository, id khepri.ID) error {
tree, err := khepri.NewTreeFromRepo(repo, id)
if err != nil {
return err
}
buf, err := json.MarshalIndent(tree, "", " ")
if err != nil {
return err
}
fmt.Printf("tree %s\n%s\n", id, buf)
for _, node := range tree.Nodes {
if node.Type == "dir" {
err = dump_tree(repo, node.Subtree)
if err != nil {
fmt.Fprintf(os.Stderr, "error: %v\n", err)
}
}
}
return nil
}
func dump_snapshot(repo *khepri.Repository, id khepri.ID) error {
sn, err := khepri.LoadSnapshot(repo, id)
if err != nil {
log.Fatalf("error loading snapshot %s", id)
}
buf, err := json.MarshalIndent(sn, "", " ")
if err != nil {
return err
}
fmt.Printf("%s\n%s\n", sn, buf)
return dump_tree(repo, sn.Content)
}
func dump_file(repo *khepri.Repository, id khepri.ID) error {
rd, err := repo.Get(khepri.TYPE_BLOB, id)
if err != nil {
return err
}
io.Copy(os.Stdout, rd)
return nil
}
func commandDump(repo *khepri.Repository, args []string) error {
if len(args) != 2 {
return errors.New("usage: dump [snapshot|tree|file] ID")
}
tpe := args[0]
id, err := khepri.ParseID(args[1])
if err != nil {
errx(1, "invalid id %q: %v", args[0], err)
}
switch tpe {
case "snapshot":
return dump_snapshot(repo, id)
case "tree":
return dump_tree(repo, id)
case "file":
return dump_file(repo, id)
default:
return fmt.Errorf("invalid type %q", tpe)
}
return nil
}

116
cmd/khepri/cmd_fsck.go
View File

@ -1,84 +1,76 @@
package main
import (
"encoding/json"
"io/ioutil"
"log"
import "github.com/fd0/khepri/backend"
"github.com/fd0/khepri"
)
// func fsck_tree(be backend.Server, id backend.ID) (bool, error) {
// log.Printf(" checking dir %s", id)
func fsck_tree(repo *khepri.Repository, id khepri.ID) (bool, error) {
log.Printf(" checking dir %s", id)
// buf, err := be.GetBlob(id)
// if err != nil {
// return false, err
// }
rd, err := repo.Get(khepri.TYPE_BLOB, id)
if err != nil {
return false, err
}
// tree := &khepri.Tree{}
// err = json.Unmarshal(buf, tree)
// if err != nil {
// return false, err
// }
buf, err := ioutil.ReadAll(rd)
// if !id.Equal(backend.IDFromData(buf)) {
// return false, nil
// }
tree := &khepri.Tree{}
err = json.Unmarshal(buf, tree)
if err != nil {
return false, err
}
// return true, nil
// }
if !id.Equal(khepri.IDFromData(buf)) {
return false, nil
}
// func fsck_snapshot(be backend.Server, id backend.ID) (bool, error) {
// log.Printf("checking snapshot %s", id)
return true, nil
}
// sn, err := khepri.LoadSnapshot(be, id)
// if err != nil {
// return false, err
// }
func fsck_snapshot(repo *khepri.Repository, id khepri.ID) (bool, error) {
log.Printf("checking snapshot %s", id)
// return fsck_tree(be, sn.Content)
// }
sn, err := khepri.LoadSnapshot(repo, id)
if err != nil {
return false, err
}
func commandFsck(be backend.Server, args []string) error {
// var snapshots backend.IDs
// var err error
return fsck_tree(repo, sn.Content)
}
// if len(args) != 0 {
// snapshots = make(backend.IDs, 0, len(args))
func commandFsck(repo *khepri.Repository, args []string) error {
var snapshots khepri.IDs
var err error
// for _, arg := range args {
// id, err := backend.ParseID(arg)
// if err != nil {
// log.Fatal(err)
// }
if len(args) != 0 {
snapshots = make(khepri.IDs, 0, len(args))
// snapshots = append(snapshots, id)
// }
// } else {
// snapshots, err = be.ListRefs()
for _, arg := range args {
id, err := khepri.ParseID(arg)
if err != nil {
log.Fatal(err)
}
// if err != nil {
// log.Fatalf("error reading list of snapshot IDs: %v", err)
// }
// }
snapshots = append(snapshots, id)
}
} else {
snapshots, err = repo.List(khepri.TYPE_REF)
// log.Printf("checking %d snapshots", len(snapshots))
if err != nil {
log.Fatalf("error reading list of snapshot IDs: %v", err)
}
}
// for _, id := range snapshots {
// ok, err := fsck_snapshot(be, id)
log.Printf("checking %d snapshots", len(snapshots))
// if err != nil {
// log.Printf("error checking snapshot %s: %v", id, err)
// continue
// }
for _, id := range snapshots {
ok, err := fsck_snapshot(repo, id)
if err != nil {
log.Printf("error checking snapshot %s: %v", id, err)
continue
}
if !ok {
log.Printf("snapshot %s failed", id)
}
}
// if !ok {
// log.Printf("snapshot %s failed", id)
// }
// }
return nil
}

20
cmd/khepri/cmd_init.go
View File

@ -5,16 +5,30 @@ import (
"os"
"github.com/fd0/khepri"
"github.com/fd0/khepri/backend"
)
func commandInit(path string) error {
repo, err := khepri.CreateRepository(path)
pw := read_password("enter password for new backend: ")
pw2 := read_password("enter password again: ")
if pw != pw2 {
errx(1, "passwords do not match")
}
be, err := backend.CreateLocal(path)
if err != nil {
fmt.Fprintf(os.Stderr, "creating repository at %s failed: %v\n", path, err)
fmt.Fprintf(os.Stderr, "creating local backend at %s failed: %v\n", path, err)
os.Exit(1)
}
fmt.Printf("created khepri repository at %s\n", repo.Path())
_, err = khepri.CreateKey(be, pw)
if err != nil {
fmt.Fprintf(os.Stderr, "creating key in local backend at %s failed: %v\n", path, err)
os.Exit(1)
}
fmt.Printf("created khepri backend at %s\n", be.Location())
return nil
}

28
cmd/khepri/cmd_list.go
View File

@ -1,29 +1,21 @@
package main
import (
"errors"
"fmt"
"os"
"github.com/fd0/khepri"
"github.com/fd0/khepri/backend"
)
func commandList(repo *khepri.Repository, args []string) error {
if len(args) != 1 {
return errors.New("usage: list [blob|ref]")
}
func commandList(be backend.Server, key *khepri.Key, args []string) error {
tpe := khepri.NewTypeFromString(args[0])
// ids, err := be.ListRefs()
// if err != nil {
// fmt.Fprintf(os.Stderr, "error: %v\n", err)
// return nil
// }
ids, err := repo.List(tpe)
if err != nil {
fmt.Fprintf(os.Stderr, "error: %v\n", err)
return nil
}
for _, id := range ids {
fmt.Printf("%v\n", id)
}
// for _, id := range ids {
// fmt.Printf("%v\n", id)
// }
return nil
}

38
cmd/khepri/cmd_restore.go
View File

@ -2,34 +2,54 @@ package main
import (
"errors"
"log"
"fmt"
"os"
"github.com/fd0/khepri"
"github.com/fd0/khepri/backend"
)
func commandRestore(repo *khepri.Repository, args []string) error {
func commandRestore(be backend.Server, key *khepri.Key, args []string) error {
if len(args) != 2 {
return errors.New("usage: restore ID dir")
}
id, err := khepri.ParseID(args[0])
id, err := backend.ParseID(args[0])
if err != nil {
errx(1, "invalid id %q: %v", args[0], err)
}
target := args[1]
sn, err := khepri.LoadSnapshot(repo, id)
// create restorer
res, err := khepri.NewRestorer(be, key, id)
if err != nil {
log.Fatalf("error loading snapshot %s: %v", id, err)
fmt.Fprintf(os.Stderr, "creating restorer failed: %v\n", err)
os.Exit(2)
}
err = sn.RestoreAt(target)
if err != nil {
log.Fatalf("error restoring snapshot %s: %v", id, err)
res.Error = func(dir string, node *khepri.Node, err error) error {
fmt.Fprintf(os.Stderr, "error for %s: %+v\n", dir, err)
// if node.Type == "dir" {
// if e, ok := err.(*os.PathError); ok {
// if errn, ok := e.Err.(syscall.Errno); ok {
// if errn == syscall.EEXIST {
// fmt.Printf("ignoring already existing directory %s\n", dir)
// return nil
// }
// }
// }
// }
return err
}
log.Printf("%q restored to %q\n", id, target)
fmt.Printf("restoring %s to %s\n", res.Snapshot(), target)
err = res.RestoreTo(target)
if err != nil {
return err
}
return nil
}

37
cmd/khepri/cmd_snapshots.go
View File

@ -3,39 +3,34 @@ package main
import (
"errors"
"fmt"
"log"
"github.com/fd0/khepri"
"github.com/fd0/khepri/backend"
)
const TimeFormat = "02.01.2006 15:04:05 -0700"
func commandSnapshots(repo *khepri.Repository, args []string) error {
func commandSnapshots(be backend.Server, key *khepri.Key, args []string) error {
if len(args) != 0 {
return errors.New("usage: snapshots")
}
snapshot_ids, err := repo.List(khepri.TYPE_REF)
if err != nil {
log.Fatalf("error loading list of snapshot ids: %v", err)
}
// ch, err := khepri.NewContentHandler(be, key)
// if err != nil {
// return err
// }
fmt.Printf("found snapshots:\n")
for _, id := range snapshot_ids {
snapshot, err := khepri.LoadSnapshot(repo, id)
backend.EachID(be, backend.Snapshot, func(id backend.ID) {
// sn, err := ch.LoadSnapshot(id)
// if err != nil {
// fmt.Fprintf(os.Stderr, "error loading snapshot %s: %v\n", id, err)
// return
// }
if err != nil {
log.Printf("error loading snapshot %s: %v", id, err)
continue
}
fmt.Printf("%s %s@%s %s %s\n",
snapshot.Time.Format(TimeFormat),
snapshot.Username,
snapshot.Hostname,
snapshot.Dir,
id)
}
// fmt.Printf("snapshot %s\n %s at %s by %s\n",
// id, sn.Dir, sn.Time, sn.Username)
fmt.Println(id)
})
return nil
}

48
cmd/khepri/main.go
View File

@ -4,8 +4,13 @@ import (
"fmt"
"log"
"os"
"sort"
"strings"
"code.google.com/p/go.crypto/ssh/terminal"
"github.com/fd0/khepri"
"github.com/fd0/khepri/backend"
"github.com/jessevdk/go-flags"
)
@ -21,18 +26,32 @@ func errx(code int, format string, data ...interface{}) {
os.Exit(code)
}
type commandFunc func(*khepri.Repository, []string) error
type commandFunc func(backend.Server, *khepri.Key, []string) error
var commands map[string]commandFunc
func read_password(prompt string) string {
p := os.Getenv("KHEPRI_PASSWORD")
if p != "" {
return p
}
fmt.Print(prompt)
pw, err := terminal.ReadPassword(int(os.Stdin.Fd()))
if err != nil {
errx(2, "unable to read password: %v", err)
}
fmt.Println()
return string(pw)
}
func init() {
commands = make(map[string]commandFunc)
commands["backup"] = commandBackup
commands["restore"] = commandRestore
commands["list"] = commandList
commands["snapshots"] = commandSnapshots
commands["fsck"] = commandFsck
commands["dump"] = commandDump
}
func main() {
@ -42,12 +61,22 @@ func main() {
if Opts.Repo == "" {
Opts.Repo = "khepri-backup"
}
args, err := flags.Parse(&Opts)
args, err := flags.Parse(&Opts)
if e, ok := err.(*flags.Error); ok && e.Type == flags.ErrHelp {
os.Exit(0)
}
if len(args) == 0 {
cmds := []string{"init"}
for k := range commands {
cmds = append(cmds, k)
}
sort.Strings(cmds)
fmt.Printf("nothing to do, available commands: [%v]\n", strings.Join(cmds, "|"))
os.Exit(0)
}
cmd := args[0]
if cmd == "init" {
@ -64,13 +93,18 @@ func main() {
errx(1, "unknown command: %q\n", cmd)
}
repo, err := khepri.NewRepository(Opts.Repo)
// read_password("enter password: ")
repo, err := backend.OpenLocal(Opts.Repo)
if err != nil {
errx(1, "unable to open repo: %v", err)
}
err = f(repo, args[1:])
key, err := khepri.SearchKey(repo, read_password("Enter Password for Repository: "))
if err != nil {
errx(2, "unable to open repo: %v", err)
}
err = f(repo, key, args[1:])
if err != nil {
errx(1, "error executing command %q: %v", cmd, err)
}

110
cmd/list/main.go Normal file
View File

@ -0,0 +1,110 @@
package main
import (
"encoding/json"
"fmt"
"os"
"code.google.com/p/go.crypto/ssh/terminal"
"github.com/fd0/khepri"
"github.com/fd0/khepri/backend"
)
func read_password(prompt string) string {
p := os.Getenv("KHEPRI_PASSWORD")
if p != "" {
return p
}
fmt.Print(prompt)
pw, err := terminal.ReadPassword(int(os.Stdin.Fd()))
if err != nil {
fmt.Fprintf(os.Stderr, "unable to read password: %v", err)
os.Exit(2)
}
fmt.Println()
return string(pw)
}
func list(be backend.Server, key *khepri.Key, t backend.Type) {
ids, err := be.List(t)
if err != nil {
fmt.Fprintf(os.Stderr, "failed: %v\n", err)
os.Exit(3)
}
for _, id := range ids {
buf, err := be.Get(t, id)
if err != nil {
fmt.Fprintf(os.Stderr, "unable to get snapshot %s: %v\n", id, err)
continue
}
if t != backend.Key && t != backend.Blob {
buf, err = key.Decrypt(buf)
if err != nil {
fmt.Fprintf(os.Stderr, "error: %v\n", err)
continue
}
}
if t == backend.Snapshot {
var sn khepri.Snapshot
err = json.Unmarshal(backend.Uncompress(buf), &sn)
if err != nil {
fmt.Fprintf(os.Stderr, "error: %v\n", err)
continue
}
fmt.Printf("%s %s\n", id, sn.String())
} else if t == backend.Blob {
fmt.Printf("%s %d bytes (encrypted)\n", id, len(buf))
} else if t == backend.Tree {
fmt.Printf("%s\n", backend.Hash(buf))
} else if t == backend.Key {
k := &khepri.Key{}
err = json.Unmarshal(buf, k)
if err != nil {
fmt.Fprintf(os.Stderr, "unable to unmashal key: %v\n", err)
continue
}
fmt.Println(key)
} else if t == backend.Lock {
fmt.Printf("lock: %v\n", id)
}
}
}
func main() {
if len(os.Args) != 2 {
fmt.Fprintf(os.Stderr, "usage: archive REPO\n")
os.Exit(1)
}
repo := os.Args[1]
be, err := backend.OpenLocal(repo)
if err != nil {
fmt.Fprintf(os.Stderr, "failed: %v\n", err)
os.Exit(1)
}
key, err := khepri.SearchKey(be, read_password("Enter Password for Repository: "))
if err != nil {
fmt.Fprintf(os.Stderr, "failed: %v\n", err)
os.Exit(2)
}
fmt.Printf("keys:\n")
list(be, key, backend.Key)
fmt.Printf("---\nlocks:\n")
list(be, key, backend.Lock)
fmt.Printf("---\nsnapshots:\n")
list(be, key, backend.Snapshot)
fmt.Printf("---\ntrees:\n")
list(be, key, backend.Tree)
fmt.Printf("---\nblobs:\n")
list(be, key, backend.Blob)
}

242
contenthandler.go Normal file
View File

@ -0,0 +1,242 @@
package khepri
import (
"encoding/json"
"errors"
"io"
"io/ioutil"
"os"
"github.com/fd0/khepri/backend"
"github.com/fd0/khepri/chunker"
)
type ContentHandler struct {
be backend.Server
key *Key
content *StorageMap
}
// NewContentHandler creates a new content handler.
func NewContentHandler(be backend.Server, key *Key) (*ContentHandler, error) {
ch := &ContentHandler{
be: be,
key: key,
content: NewStorageMap(),
}
return ch, nil
}
// LoadSnapshotadds all blobs from a snapshot into the content handler and returns the snapshot.
func (ch *ContentHandler) LoadSnapshot(id backend.ID) (*Snapshot, error) {
sn, err := LoadSnapshot(ch, id)
if err != nil {
return nil, err
}
ch.content.Merge(sn.StorageMap)
return sn, nil
}
// LoadAllSnapshots adds all blobs from all snapshots that can be decrypted
// into the content handler.
func (ch *ContentHandler) LoadAllSnapshots() error {
// add all maps from all snapshots that can be decrypted to the storage map
err := backend.EachID(ch.be, backend.Snapshot, func(id backend.ID) {
sn, err := LoadSnapshot(ch, id)
if err != nil {
return
}
ch.content.Merge(sn.StorageMap)
})
if err != nil {
return err
}
return nil
}
// Save encrypts data and stores it to the backend as type t. If the data was
// already saved before, the blob is returned.
func (ch *ContentHandler) Save(t backend.Type, data []byte) (*Blob, error) {
// compute plaintext hash
id := backend.Hash(data)
// test if the hash is already in the backend
blob := ch.content.Find(id)
if blob != nil {
return blob, nil
}
// else create a new blob
blob = &Blob{
ID: id,
Size: uint64(len(data)),
}
// encrypt blob
ciphertext, err := ch.key.Encrypt(data)
if err != nil {
return nil, err
}
// save blob
sid, err := ch.be.Create(t, ciphertext)
if err != nil {
return nil, err
}
blob.Storage = sid
blob.StorageSize = uint64(len(ciphertext))
// insert blob into the storage map
ch.content.Insert(blob)
return blob, nil
}
// SaveJSON serialises item as JSON and uses Save() to store it to the backend as type t.
func (ch *ContentHandler) SaveJSON(t backend.Type, item interface{}) (*Blob, error) {
// convert to json
data, err := json.Marshal(item)
if err != nil {
return nil, err
}
// compress and save data
return ch.Save(t, backend.Compress(data))
}
// SaveFile stores the content of the file on the backend as a Blob by calling
// Save for each chunk.
func (ch *ContentHandler) SaveFile(filename string, size uint) (Blobs, error) {
file, err := os.Open(filename)
defer file.Close()
if err != nil {
return nil, err
}
// if the file is small enough, store it directly
if size < chunker.MinSize {
buf, err := ioutil.ReadAll(file)
if err != nil {
return nil, err
}
blob, err := ch.Save(backend.Blob, buf)
if err != nil {
return nil, err
}
return Blobs{blob}, nil
}
// else store all chunks
blobs := Blobs{}
chunker := chunker.New(file)
for {
chunk, err := chunker.Next()
if err == io.EOF {
break
}
if err != nil {
return nil, err
}
blob, err := ch.Save(backend.Blob, chunk.Data)
if err != nil {
return nil, err
}
blobs = append(blobs, blob)
}
return blobs, nil
}
// Load tries to load and decrypt content identified by t and id from the backend.
func (ch *ContentHandler) Load(t backend.Type, id backend.ID) ([]byte, error) {
if t == backend.Snapshot {
// load data
buf, err := ch.be.Get(t, id)
if err != nil {
return nil, err
}
// decrypt
buf, err = ch.key.Decrypt(buf)
if err != nil {
return nil, err
}
return buf, nil
}
// lookup storage hash
blob := ch.content.Find(id)
if blob == nil {
return nil, errors.New("Storage ID not found")
}
// load data
buf, err := ch.be.Get(t, blob.Storage)
if err != nil {
return nil, err
}
// check length
if len(buf) != int(blob.StorageSize) {
return nil, errors.New("Invalid storage length")
}
// decrypt
buf, err = ch.key.Decrypt(buf)
if err != nil {
return nil, err
}
// check length
if len(buf) != int(blob.Size) {
return nil, errors.New("Invalid length")
}
return buf, nil
}
// LoadJSON calls Load() to get content from the backend and afterwards calls
// json.Unmarshal on the item.
func (ch *ContentHandler) LoadJSON(t backend.Type, id backend.ID, item interface{}) error {
// load from backend
buf, err := ch.Load(t, id)
if err != nil {
return err
}
// inflate and unmarshal
err = json.Unmarshal(backend.Uncompress(buf), item)
return err
}
// LoadJSONRaw loads data with the given storage id and type from the backend,
// decrypts it and calls json.Unmarshal on the item.
func (ch *ContentHandler) LoadJSONRaw(t backend.Type, id backend.ID, item interface{}) error {
// load data
buf, err := ch.be.Get(t, id)
if err != nil {
return err
}
// decrypt
buf, err = ch.key.Decrypt(buf)
if err != nil {
return err
}
// inflate and unmarshal
err = json.Unmarshal(backend.Uncompress(buf), item)
return err
}

388
key.go Normal file
View File

@ -0,0 +1,388 @@
package khepri
import (
"bytes"
"crypto/aes"
"crypto/cipher"
"crypto/hmac"
"crypto/rand"
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"os/user"
"time"
"github.com/fd0/khepri/backend"
"code.google.com/p/go.crypto/scrypt"
)
var (
ErrUnauthenticated = errors.New("Ciphertext verification failed")
ErrNoKeyFound = errors.New("No key could be found")
)
// TODO: figure out scrypt values on the fly depending on the current
// hardware.
const (
scrypt_N = 65536
scrypt_r = 8
scrypt_p = 1
scrypt_saltsize = 64
aesKeysize = 32 // for AES256
hmacKeysize = 32 // for HMAC with SHA256
)
type Key struct {
Created time.Time `json:"created"`
Username string `json:"username"`
Hostname string `json:"hostname"`
Comment string `json:"comment,omitempty"`
KDF string `json:"kdf"`
N int `json:"N"`
R int `json:"r"`
P int `json:"p"`
Salt []byte `json:"salt"`
Data []byte `json:"data"`
user *keys
master *keys
}
type keys struct {
Sign []byte
Encrypt []byte
}
func CreateKey(be backend.Server, password string) (*Key, error) {
// fill meta data about key
k := &Key{
Created: time.Now(),
KDF: "scrypt",
N: scrypt_N,
R: scrypt_r,
P: scrypt_p,
}
hn, err := os.Hostname()
if err == nil {
k.Hostname = hn
}
usr, err := user.Current()
if err == nil {
k.Username = usr.Username
}
// generate random salt
k.Salt = make([]byte, scrypt_saltsize)
n, err := rand.Read(k.Salt)
if n != scrypt_saltsize || err != nil {
panic("unable to read enough random bytes for salt")
}
// call scrypt() to derive user key
k.user, err = k.scrypt(password)
if err != nil {
return nil, err
}
// generate new random master keys
k.master, err = k.newKeys()
if err != nil {
return nil, err
}
// encrypt master keys (as json) with user key
buf, err := json.Marshal(k.master)
if err != nil {
return nil, err
}
k.Data, err = k.EncryptUser(buf)
// dump as json
buf, err = json.Marshal(k)
if err != nil {
return nil, err
}
// store in repository and return
_, err = be.Create(backend.Key, buf)
if err != nil {
return nil, err
}
return k, nil
}
func OpenKey(be backend.Server, id backend.ID, password string) (*Key, error) {
// extract data from repo
data, err := be.Get(backend.Key, id)
if err != nil {
return nil, err
}
// restore json
k := &Key{}
err = json.Unmarshal(data, k)
if err != nil {
return nil, err
}
// check KDF
if k.KDF != "scrypt" {
return nil, errors.New("only supported KDF is scrypt()")
}
// derive user key
k.user, err = k.scrypt(password)
if err != nil {
return nil, err
}
// decrypt master keys
buf, err := k.DecryptUser(k.Data)
if err != nil {
return nil, err
}
// restore json
k.master = &keys{}
err = json.Unmarshal(buf, k.master)
if err != nil {
return nil, err
}
return k, nil
}
func SearchKey(be backend.Server, password string) (*Key, error) {
// list all keys
ids, err := be.List(backend.Key)
if err != nil {
panic(err)
}
// try all keys in repo
var key *Key
for _, id := range ids {
key, err = OpenKey(be, id, password)
if err != nil {
continue
}
return key, nil
}
return nil, ErrNoKeyFound
}
func (k *Key) scrypt(password string) (*keys, error) {
if len(k.Salt) == 0 {
return nil, fmt.Errorf("scrypt() called with empty salt")
}
keybytes := hmacKeysize + aesKeysize
scrypt_keys, err := scrypt.Key([]byte(password), k.Salt, k.N, k.R, k.P, keybytes)
if err != nil {
return nil, fmt.Errorf("error deriving keys from password: %v", err)
}
if len(scrypt_keys) != keybytes {
return nil, fmt.Errorf("invalid numbers of bytes expanded from scrypt(): %d", len(scrypt_keys))
}
ks := &keys{
Encrypt: scrypt_keys[:aesKeysize],
Sign: scrypt_keys[aesKeysize:],
}
return ks, nil
}
func (k *Key) newKeys() (*keys, error) {
ks := &keys{
Encrypt: make([]byte, aesKeysize),
Sign: make([]byte, hmacKeysize),
}
n, err := rand.Read(ks.Encrypt)
if n != aesKeysize || err != nil {
panic("unable to read enough random bytes for encryption key")
}
n, err = rand.Read(ks.Sign)
if n != hmacKeysize || err != nil {
panic("unable to read enough random bytes for signing key")
}
return ks, nil
}
func (k *Key) newIV() ([]byte, error) {
buf := make([]byte, aes.BlockSize)
_, err := io.ReadFull(rand.Reader, buf)
if err != nil {
return nil, err
}
return buf, nil
}
func (k *Key) pad(plaintext []byte) []byte {
l := aes.BlockSize - (len(plaintext) % aes.BlockSize)
if l == 0 {
l = aes.BlockSize
}
if l <= 0 || l > aes.BlockSize {
panic("invalid padding size")
}
return append(plaintext, bytes.Repeat([]byte{byte(l)}, l)...)
}
func (k *Key) unpad(plaintext []byte) []byte {
l := len(plaintext)
pad := plaintext[l-1]
if pad > aes.BlockSize {
panic(errors.New("padding > BlockSize"))
}
if pad == 0 {
panic(errors.New("invalid padding 0"))
}
for i := l - int(pad); i < l; i++ {
if plaintext[i] != pad {
panic(errors.New("invalid padding!"))
}
}
return plaintext[:l-int(pad)]
}
// Encrypt encrypts and signs data. Returned is IV || Ciphertext || HMAC. For
// the hash function, SHA256 is used, so the overhead is 16+32=48 byte.
func (k *Key) encrypt(ks *keys, plaintext []byte) ([]byte, error) {
iv, err := k.newIV()
if err != nil {
panic(fmt.Sprintf("unable to generate new random iv: %v", err))
}
c, err := aes.NewCipher(ks.Encrypt)
if err != nil {
panic(fmt.Sprintf("unable to create cipher: %v", err))
}
e := cipher.NewCBCEncrypter(c, iv)
p := k.pad(plaintext)
ciphertext := make([]byte, len(p))
e.CryptBlocks(ciphertext, p)
ciphertext = append(iv, ciphertext...)
hm := hmac.New(sha256.New, ks.Sign)
n, err := hm.Write(ciphertext)
if err != nil || n != len(ciphertext) {
panic(fmt.Sprintf("unable to calculate hmac of ciphertext: %v", err))
}
return hm.Sum(ciphertext), nil
}
// EncryptUser encrypts and signs data with the user key. Returned is IV ||
// Ciphertext || HMAC. For the hash function, SHA256 is used, so the overhead
// is 16+32=48 byte.
func (k *Key) EncryptUser(plaintext []byte) ([]byte, error) {
return k.encrypt(k.user, plaintext)
}
// Encrypt encrypts and signs data with the master key. Returned is IV ||
// Ciphertext || HMAC. For the hash function, SHA256 is used, so the overhead
// is 16+32=48 byte.
func (k *Key) Encrypt(plaintext []byte) ([]byte, error) {
return k.encrypt(k.master, plaintext)
}
// Decrypt verifes and decrypts the ciphertext. Ciphertext must be in the form
// IV || Ciphertext || HMAC.
func (k *Key) decrypt(ks *keys, ciphertext []byte) ([]byte, error) {
hm := hmac.New(sha256.New, ks.Sign)
// extract hmac
l := len(ciphertext) - hm.Size()
ciphertext, mac := ciphertext[:l], ciphertext[l:]
// calculate new hmac
n, err := hm.Write(ciphertext)
if err != nil || n != len(ciphertext) {
panic(fmt.Sprintf("unable to calculate hmac of ciphertext, err %v", err))
}
// verify hmac
mac2 := hm.Sum(nil)
if !hmac.Equal(mac, mac2) {
return nil, ErrUnauthenticated
}
// extract iv
iv, ciphertext := ciphertext[:aes.BlockSize], ciphertext[aes.BlockSize:]
// decrypt data
c, err := aes.NewCipher(ks.Encrypt)
if err != nil {
panic(fmt.Sprintf("unable to create cipher: %v", err))
}
// decrypt
e := cipher.NewCBCDecrypter(c, iv)
plaintext := make([]byte, len(ciphertext))
e.CryptBlocks(plaintext, ciphertext)
// remove padding and return
return k.unpad(plaintext), nil
}
// Decrypt verifes and decrypts the ciphertext with the master key. Ciphertext
// must be in the form IV || Ciphertext || HMAC.
func (k *Key) Decrypt(ciphertext []byte) ([]byte, error) {
return k.decrypt(k.master, ciphertext)
}
// DecryptUser verifes and decrypts the ciphertext with the master key. Ciphertext
// must be in the form IV || Ciphertext || HMAC.
func (k *Key) DecryptUser(ciphertext []byte) ([]byte, error) {
return k.decrypt(k.user, ciphertext)
}
// Each calls backend.Each() with the given parameters, Decrypt() on the
// ciphertext and, on successful decryption, f with the plaintext.
func (k *Key) Each(be backend.Server, t backend.Type, f func(backend.ID, []byte, error)) error {
return backend.Each(be, t, func(id backend.ID, data []byte, e error) {
if e != nil {
f(id, nil, e)
return
}
buf, err := k.Decrypt(data)
if err != nil {
f(id, nil, err)
return
}
f(id, buf, nil)
})
}
func (k *Key) String() string {
if k == nil {
return "<Key nil>"
}
return fmt.Sprintf("<Key of %s@%s, created on %s>", k.Username, k.Hostname, k.Created)
}

79
key_int_test.go Normal file
View File

@ -0,0 +1,79 @@
package khepri
import (
"bytes"
"encoding/hex"
"testing"
)
var test_values = []struct {
ekey, skey []byte
ciphertext []byte
plaintext []byte
should_panic bool
}{
{
ekey: decode_hex("303e8687b1d7db18421bdc6bb8588ccadac4d59ee87b8ff70c44e635790cafef"),
skey: decode_hex("cc8d4b948ee0ebfe1d415de921d10353ef4d8824cb80b2bcc5fbff8a9b12a42c"),
ciphertext: decode_hex("154f582d77e6430409da392c3a09aa38e00a78bcc8919557fe18dd17f83e7b0b3053def59f4215b6e1c6b72ceb5acdddd8511ce3a853e054218de1e9f34637470d68f1f93ba8228e4d9817d7c9acfcd2"),
plaintext: []byte("Dies ist ein Test!"),
},
}
func decode_hex(s string) []byte {
d, _ := hex.DecodeString(s)
return d
}
// returns true if function called panic
func should_panic(f func()) (did_panic bool) {
defer func() {
if r := recover(); r != nil {
did_panic = true
}
}()
f()
return false
}
func TestCrypto(t *testing.T) {
r := &Key{}
for _, tv := range test_values {
// test encryption
r.master = &keys{
Encrypt: tv.ekey,
Sign: tv.skey,
}
msg, err := r.encrypt(r.master, tv.plaintext)
if err != nil {
t.Fatal(err)
}
// decrypt message
_, err = r.decrypt(r.master, msg)
if err != nil {
t.Fatal(err)
}
// change hmac, this must fail
msg[len(msg)-8] ^= 0x23
if _, err = r.decrypt(r.master, msg); err != ErrUnauthenticated {
t.Fatal("wrong HMAC value not detected")
}
// test decryption
p, err := r.decrypt(r.master, tv.ciphertext)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(p, tv.plaintext) {
t.Fatalf("wrong plaintext: expected %q but got %q\n", tv.plaintext, p)
}
}
}

50
key_test.go Normal file
View File

@ -0,0 +1,50 @@
package khepri_test
import (
"flag"
"io/ioutil"
"os"
"testing"
"github.com/fd0/khepri"
"github.com/fd0/khepri/backend"
)
var test_password = "foobar"
var testCleanup = flag.Bool("test.cleanup", true, "clean up after running tests (remove local backend directory with all content)")
func setupBackend(t *testing.T) *backend.Local {
tempdir, err := ioutil.TempDir("", "khepri-test-")
ok(t, err)
b, err := backend.CreateLocal(tempdir)
ok(t, err)
t.Logf("created local backend at %s", tempdir)
return b
}
func teardownBackend(t *testing.T, b *backend.Local) {
if !*testCleanup {
t.Logf("leaving local backend at %s\n", b.Location())
return
}
ok(t, os.RemoveAll(b.Location()))
}
func setupKey(t *testing.T, be backend.Server, password string) *khepri.Key {
c, err := khepri.CreateKey(be, password)
ok(t, err)
t.Logf("created Safe at %s", be.Location())
return c
}
func TestSafe(t *testing.T) {
be := setupBackend(t)
defer teardownBackend(t, be)
_ = setupKey(t, be, test_password)
}

27
object.go
View File

@ -1,27 +0,0 @@
package khepri
func (repo *Repository) Create(t Type, data []byte) (ID, error) {
// TODO: make sure that tempfile is removed upon error
// create tempfile in repository
var err error
file, err := repo.tempFile()
if err != nil {
return nil, err
}
// write data to tempfile
_, err = file.Write(data)
if err != nil {
return nil, err
}
// close tempfile, return id
id := IDFromData(data)
err = repo.renameFile(file, t, id)
if err != nil {
return nil, err
}
return id, nil
}

27
object_test.go
View File

@ -1,27 +0,0 @@
package khepri_test
import (
"testing"
"github.com/fd0/khepri"
)
func TestObjects(t *testing.T) {
repo, err := setupRepo()
ok(t, err)
defer func() {
err = teardownRepo(repo)
ok(t, err)
}()
for _, test := range TestStrings {
id, err := repo.Create(khepri.TYPE_BLOB, []byte(test.data))
ok(t, err)
id2, err := khepri.ParseID(test.id)
ok(t, err)
equals(t, id2, id)
}
}

326
repository.go
View File

@ -1,326 +0,0 @@
package khepri
import (
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"hash"
"io"
"io/ioutil"
"net/url"
"os"
"path"
"path/filepath"
)
const (
dirMode = 0700
blobPath = "blobs"
refPath = "refs"
tempPath = "tmp"
configFileName = "config.json"
)
var (
ErrIDDoesNotExist = errors.New("ID does not exist")
)
// Name stands for the alias given to an ID.
type Name string
func (n Name) Encode() string {
return url.QueryEscape(string(n))
}
type HashFunc func() hash.Hash
type Repository struct {
path string
hash HashFunc
config *Config
}
type Config struct {
Salt string
N uint
R uint `json:"r"`
P uint `json:"p"`
}
// TODO: figure out scrypt values on the fly depending on the current
// hardware.
const (
scrypt_N = 65536
scrypt_r = 8
scrypt_p = 1
scrypt_saltsize = 64
)
type Type int
const (
TYPE_BLOB = iota
TYPE_REF
)
func NewTypeFromString(s string) Type {
switch s {
case "blob":
return TYPE_BLOB
case "ref":
return TYPE_REF
}
panic(fmt.Sprintf("unknown type %q", s))
}
func (t Type) String() string {
switch t {
case TYPE_BLOB:
return "blob"
case TYPE_REF:
return "ref"
}
panic(fmt.Sprintf("unknown type %d", t))
}
// NewRepository opens a dir-baked repository at the given path.
func NewRepository(path string) (*Repository, error) {
var err error
d := &Repository{
path: path,
hash: sha256.New,
}
d.config, err = d.read_config()
if err != nil {
return nil, err
}
return d, nil
}
func (r *Repository) read_config() (*Config, error) {
// try to open config file
f, err := os.Open(path.Join(r.path, configFileName))
if err != nil {
return nil, err
}
cfg := new(Config)
buf, err := ioutil.ReadAll(f)
if err != nil {
return nil, err
}
err = json.Unmarshal(buf, cfg)
if err != nil {
return nil, err
}
return cfg, nil
}
// CreateRepository creates all the necessary files and directories for the
// Repository.
func CreateRepository(p string) (*Repository, error) {
dirs := []string{
p,
path.Join(p, blobPath),
path.Join(p, refPath),
path.Join(p, tempPath),
}
var configfile = path.Join(p, configFileName)
// test if repository directories or config file already exist
if _, err := os.Stat(configfile); err == nil {
return nil, fmt.Errorf("config file %s already exists", configfile)
}
for _, d := range dirs[1:] {
if _, err := os.Stat(d); err == nil {
return nil, fmt.Errorf("dir %s already exists", d)
}
}
// create initial json configuration
cfg := &Config{
N: scrypt_N,
R: scrypt_r,
P: scrypt_p,
}
// generate salt
buf := make([]byte, scrypt_saltsize)
n, err := rand.Read(buf)
if n != scrypt_saltsize || err != nil {
panic("unable to read enough random bytes for salt")
}
cfg.Salt = hex.EncodeToString(buf)
// create ps for blobs, refs and temp
for _, dir := range dirs {
err := os.MkdirAll(dir, dirMode)
if err != nil {
return nil, err
}
}
// write config file
f, err := os.Create(configfile)
defer f.Close()
if err != nil {
return nil, err
}
s, err := json.Marshal(cfg)
if err != nil {
return nil, err
}
_, err = f.Write(s)
if err != nil {
return nil, err
}
// open repository
return NewRepository(p)
}
// SetHash changes the hash function used for deriving IDs. Default is SHA256.
func (r *Repository) SetHash(h HashFunc) {
r.hash = h
}
// Path returns the directory used for this repository.
func (r *Repository) Path() string {
return r.path
}
// Return temp directory in correct directory for this repository.
func (r *Repository) tempFile() (*os.File, error) {
return ioutil.TempFile(path.Join(r.path, tempPath), "temp-")
}
// Rename temp file to final name according to type and ID.
func (r *Repository) renameFile(file *os.File, t Type, id ID) error {
filename := path.Join(r.dir(t), id.String())
return os.Rename(file.Name(), filename)
}
// Construct directory for given Type.
func (r *Repository) dir(t Type) string {
switch t {
case TYPE_BLOB:
return path.Join(r.path, blobPath)
case TYPE_REF:
return path.Join(r.path, refPath)
}
panic(fmt.Sprintf("unknown type %d", t))
}
// Construct path for given Type and ID.
func (r *Repository) filename(t Type, id ID) string {
return path.Join(r.dir(t), id.String())
}
// Test returns true if the given ID exists in the repository.
func (r *Repository) Test(t Type, id ID) (bool, error) {
// try to open file
file, err := os.Open(r.filename(t, id))
defer func() {
file.Close()
}()
if err != nil {
if os.IsNotExist(err) {
return false, nil
}
return false, err
}
return true, nil
}
// Get returns a reader for the content stored under the given ID.
func (r *Repository) Get(t Type, id ID) (io.ReadCloser, error) {
// try to open file
file, err := os.Open(r.filename(t, id))
if err != nil {
return nil, err
}
return file, nil
}
// Remove removes the content stored at ID.
func (r *Repository) Remove(t Type, id ID) error {
return os.Remove(r.filename(t, id))
}
type IDs []ID
// Lists all objects of a given type.
func (r *Repository) List(t Type) (IDs, error) {
// TODO: use os.Open() and d.Readdirnames() instead of Glob()
pattern := path.Join(r.dir(t), "*")
matches, err := filepath.Glob(pattern)
if err != nil {
return nil, err
}
ids := make(IDs, 0, len(matches))
for _, m := range matches {
base := filepath.Base(m)
if base == "" {
continue
}
id, err := ParseID(base)
if err != nil {
continue
}
ids = append(ids, id)
}
return ids, nil
}
func (ids IDs) Len() int {
return len(ids)
}
func (ids IDs) Less(i, j int) bool {
if len(ids[i]) < len(ids[j]) {
return true
}
for k, b := range ids[i] {
if b == ids[j][k] {
continue
}
if b < ids[j][k] {
return true
} else {
return false
}
}
return false
}
func (ids IDs) Swap(i, j int) {
ids[i], ids[j] = ids[j], ids[i]
}

130
repository_test.go
View File

@ -1,130 +0,0 @@
package khepri_test
import (
"flag"
"fmt"
"io/ioutil"
"os"
"sort"
"testing"
"github.com/fd0/khepri"
)
var testCleanup = flag.Bool("test.cleanup", true, "clean up after running tests (remove repository directory with all content)")
var TestStrings = []struct {
id string
t khepri.Type
data string
}{
{"c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2", khepri.TYPE_BLOB, "foobar"},
{"248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1", khepri.TYPE_BLOB, "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"},
{"cc5d46bdb4991c6eae3eb739c9c8a7a46fe9654fab79c47b4fe48383b5b25e1c", khepri.TYPE_REF, "foo/bar"},
{"4e54d2c721cbdb730f01b10b62dec622962b36966ec685880effa63d71c808f2", khepri.TYPE_BLOB, "foo/../../baz"},
}
func setupRepo() (*khepri.Repository, error) {
tempdir, err := ioutil.TempDir("", "khepri-test-")
if err != nil {
return nil, err
}
repo, err := khepri.CreateRepository(tempdir)
if err != nil {
return nil, err
}
return repo, nil
}
func teardownRepo(repo *khepri.Repository) error {
if !*testCleanup {
fmt.Fprintf(os.Stderr, "leaving repository at %s\n", repo.Path())
return nil
}
err := os.RemoveAll(repo.Path())
if err != nil {
return err
}
return nil
}
func TestRepository(t *testing.T) {
repo, err := setupRepo()
ok(t, err)
defer func() {
err = teardownRepo(repo)
ok(t, err)
}()
// detect non-existing files
for _, test := range TestStrings {
id, err := khepri.ParseID(test.id)
ok(t, err)
// try to get string out, should fail
ret, err := repo.Test(test.t, id)
ok(t, err)
assert(t, !ret, fmt.Sprintf("id %q was found (but should not have)", test.id))
}
// add files
for _, test := range TestStrings {
// store string in repository
id, err := repo.Create(test.t, []byte(test.data))
ok(t, err)
equals(t, test.id, id.String())
// try to get it out again
rd, err := repo.Get(test.t, id)
ok(t, err)
assert(t, rd != nil, "Get() returned nil reader")
// compare content
buf, err := ioutil.ReadAll(rd)
equals(t, test.data, string(buf))
}
// list ids
for _, tpe := range []khepri.Type{khepri.TYPE_BLOB, khepri.TYPE_REF} {
IDs := khepri.IDs{}
for _, test := range TestStrings {
if test.t == tpe {
id, err := khepri.ParseID(test.id)
ok(t, err)
IDs = append(IDs, id)
}
}
ids, err := repo.List(tpe)
ok(t, err)
sort.Sort(ids)
sort.Sort(IDs)
equals(t, IDs, ids)
}
// remove content if requested
if *testCleanup {
for _, test := range TestStrings {
id, err := khepri.ParseID(test.id)
ok(t, err)
found, err := repo.Test(test.t, id)
ok(t, err)
assert(t, found, fmt.Sprintf("id %q was not found before removal"))
err = repo.Remove(test.t, id)
ok(t, err)
found, err = repo.Test(test.t, id)
ok(t, err)
assert(t, !found, fmt.Sprintf("id %q was not found before removal"))
}
}
}

100
restorer.go Normal file
View File

@ -0,0 +1,100 @@
package khepri
import (
"errors"
"fmt"
"os"
"path/filepath"
"github.com/fd0/khepri/backend"
)
type Restorer struct {
be backend.Server
key *Key
ch *ContentHandler
sn *Snapshot
Error func(dir string, node *Node, err error) error
Filter func(item string, node *Node) bool
}
// NewRestorer creates a restorer preloaded with the content from the snapshot snid.
func NewRestorer(be backend.Server, key *Key, snid backend.ID) (*Restorer, error) {
r := &Restorer{
be: be,
key: key,
}
var err error
r.ch, err = NewContentHandler(be, key)
if err != nil {
return nil, err
}
r.sn, err = r.ch.LoadSnapshot(snid)
if err != nil {
return nil, err
}
// abort on all errors
r.Error = func(string, *Node, error) error { return err }
// allow all files
r.Filter = func(string, *Node) bool { return true }
return r, nil
}
func (res *Restorer) to(dir string, tree_id backend.ID) error {
tree := Tree{}
err := res.ch.LoadJSON(backend.Tree, tree_id, &tree)
if err != nil {
return res.Error(dir, nil, err)
}
for _, node := range tree {
p := filepath.Join(dir, node.Name)
if !res.Filter(p, node) {
continue
}
err := node.CreateAt(res.ch, p)
if err != nil {
err = res.Error(p, node, err)
if err != nil {
return err
}
}
if node.Type == "dir" {
if node.Subtree == nil {
return errors.New(fmt.Sprintf("Dir without subtree in tree %s", tree_id))
}
err = res.to(p, node.Subtree)
if err != nil {
err = res.Error(p, node, err)
if err != nil {
return err
}
}
}
}
return nil
}
// RestoreTo creates the directories and files in the snapshot below dir.
// Before an item is created, res.Filter is called.
func (res *Restorer) RestoreTo(dir string) error {
err := os.MkdirAll(dir, 0700)
if err != nil && err != os.ErrExist {
return err
}
return res.to(dir, res.sn.Content)
}
func (res *Restorer) Snapshot() *Snapshot {
return res.sn
}

75
snapshot.go
View File

@ -1,29 +1,36 @@
package khepri
import (
"encoding/json"
"fmt"
"os"
"os/user"
"path/filepath"
"time"
"github.com/fd0/khepri/backend"
)
type Snapshot struct {
Time time.Time `json:"time"`
Content ID `json:"content"`
Tree *Tree `json:"-"`
Content backend.ID `json:"content"`
StorageMap *StorageMap `json:"map"`
Dir string `json:"dir"`
Hostname string `json:"hostname,omitempty"`
Username string `json:"username,omitempty"`
UID string `json:"uid,omitempty"`
GID string `json:"gid,omitempty"`
id ID `json:omit`
repo *Repository
id backend.ID // plaintext ID, used during restore
}
func NewSnapshot(dir string) *Snapshot {
d, err := filepath.Abs(dir)
if err != nil {
d = dir
}
sn := &Snapshot{
Dir: dir,
Dir: d,
Time: time.Now(),
}
@ -42,66 +49,16 @@ func NewSnapshot(dir string) *Snapshot {
return sn
}
func (sn *Snapshot) Save(repo *Repository) (ID, error) {
if sn.Content == nil {
panic("Snapshot.Save() called with nil tree id")
}
data, err := json.Marshal(sn)
if err != nil {
return nil, err
}
id, err := repo.Create(TYPE_REF, data)
if err != nil {
return nil, err
}
return id, nil
}
func LoadSnapshot(repo *Repository, id ID) (*Snapshot, error) {
rd, err := repo.Get(TYPE_REF, id)
if err != nil {
return nil, err
}
// TODO: maybe inject a hashing reader here and test if the given id is correct
dec := json.NewDecoder(rd)
func LoadSnapshot(ch *ContentHandler, id backend.ID) (*Snapshot, error) {
sn := &Snapshot{}
err = dec.Decode(sn)
err := ch.LoadJSON(backend.Snapshot, id, sn)
if err != nil {
return nil, err
}
sn.id = id
sn.repo = repo
return sn, nil
}
func (sn *Snapshot) RestoreAt(path string) error {
err := os.MkdirAll(path, 0700)
if err != nil {
return err
}
if sn.Tree == nil {
sn.Tree, err = NewTreeFromRepo(sn.repo, sn.Content)
if err != nil {
return err
}
}
return sn.Tree.CreateAt(path)
}
func (sn *Snapshot) ID() ID {
return sn.id
}
func (sn *Snapshot) String() string {
return fmt.Sprintf("<Snapshot of %q at %s>", sn.Dir, sn.Time.Format(time.RFC822Z))
return fmt.Sprintf("<Snapshot %q at %s>", sn.Dir, sn.Time)
}

25
snapshot_test.go
View File

@ -5,23 +5,24 @@ import (
"time"
"github.com/fd0/khepri"
"github.com/fd0/khepri/backend"
)
func TestSnapshot(t *testing.T) {
repo, err := setupRepo()
ok(t, err)
defer func() {
err = teardownRepo(repo)
ok(t, err)
}()
func testSnapshot(t *testing.T, be backend.Server) {
var err error
sn := khepri.NewSnapshot("/home/foobar")
sn.Content, err = khepri.ParseID("c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2")
sn.Content, err = backend.ParseID("c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2")
ok(t, err)
sn.Time, err = time.Parse(time.RFC3339Nano, "2014-08-03T17:49:05.378595539+02:00")
ok(t, err)
_, err = sn.Save(repo)
ok(t, err)
// _, err = sn.Save(be)
// ok(t, err)
}
func TestSnapshot(t *testing.T) {
repo := setupBackend(t)
defer teardownBackend(t, repo)
testSnapshot(t, repo)
}

51
storagemap.go Normal file
View File

@ -0,0 +1,51 @@
package khepri
import (
"bytes"
"sort"
"github.com/fd0/khepri/backend"
)
type StorageMap Blobs
func NewStorageMap() *StorageMap {
return &StorageMap{}
}
func (m StorageMap) find(id backend.ID) (int, *Blob) {
i := sort.Search(len(m), func(i int) bool {
return bytes.Compare(m[i].ID, id) >= 0
})
if i < len(m) && bytes.Equal(m[i].ID, id) {
return i, m[i]
}
return i, nil
}
func (m StorageMap) Find(id backend.ID) *Blob {
_, blob := m.find(id)
return blob
}
func (m *StorageMap) Insert(blob *Blob) {
pos, b := m.find(blob.ID)
if b != nil {
// already present
return
}
// insert blob
// https://code.google.com/p/go-wiki/wiki/SliceTricks
*m = append(*m, nil)
copy((*m)[pos+1:], (*m)[pos:])
(*m)[pos] = blob
}
func (m *StorageMap) Merge(sm *StorageMap) {
for _, blob := range *sm {
m.Insert(blob)
}
}

1
test/run.sh
View File

@ -11,6 +11,7 @@ prepare() {
export BASE="$(mktemp --tmpdir --directory khepri-testsuite-XXXXXX)"
export KHEPRI_REPOSITORY="${BASE}/khepri-backup"
export DATADIR="${BASE}/fake-data"
export KHEPRI_PASSWORD="foobar"
debug "repository is at ${KHEPRI_REPOSITORY}"
mkdir -p "$DATADIR"

4
test/test-backup.sh
View File

@ -3,6 +3,6 @@ set -e
prepare
run khepri init
run khepri backup "${BASE}/fake-data"
run khepri restore "$(khepri list ref)" "${BASE}/fake-data-restore"
dirdiff "${BASE}/fake-data" "${BASE}/fake-data-restore"
run khepri restore "$(khepri snapshots)" "${BASE}/fake-data-restore"
dirdiff "${BASE}/fake-data" "${BASE}/fake-data-restore/fake-data"
cleanup

275
tree.go
View File

@ -1,24 +1,18 @@
package khepri
import (
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
"os/user"
"path/filepath"
"strconv"
"strings"
"syscall"
"time"
"github.com/fd0/khepri/chunker"
"github.com/fd0/khepri/backend"
)
type Tree struct {
Nodes []*Node `json:"nodes,omitempty"`
}
type Tree []*Node
type Node struct {
Name string `json:"name"`
@ -36,213 +30,42 @@ type Node struct {
Links uint64 `json:"links,omitempty"`
LinkTarget string `json:"linktarget,omitempty"`
Device uint64 `json:"device,omitempty"`
Content []ID `json:"content,omitempty"`
Subtree ID `json:"subtree,omitempty"`
Tree *Tree `json:"-"`
repo *Repository
Content []backend.ID `json:"content,omitempty"`
Subtree backend.ID `json:"subtree,omitempty"`
path string
}
func NewTree() *Tree {
return &Tree{
Nodes: []*Node{},
}
type Blob struct {
ID backend.ID `json:"id,omitempty"`
Size uint64 `json:"size,omitempty"`
Storage backend.ID `json:"sid,omitempty"` // encrypted ID
StorageSize uint64 `json:"ssize,omitempty"` // encrypted Size
}
func store_chunk(repo *Repository, rd io.Reader) (ID, error) {
data, err := ioutil.ReadAll(rd)
if err != nil {
return nil, err
type Blobs []*Blob
func (n Node) String() string {
switch n.Type {
case "file":
return fmt.Sprintf("%s %5d %5d %6d %s %s",
n.Mode, n.UID, n.GID, n.Size, n.ModTime, n.Name)
case "dir":
return fmt.Sprintf("%s %5d %5d %6d %s %s",
n.Mode|os.ModeDir, n.UID, n.GID, n.Size, n.ModTime, n.Name)
}
id, err := repo.Create(TYPE_BLOB, data)
if err != nil {
return nil, err
}
return id, nil
return fmt.Sprintf("<Node(%s) %s>", n.Type, n.Name)
}
func NewTreeFromPath(repo *Repository, dir string) (*Tree, error) {
fd, err := os.Open(dir)
defer fd.Close()
if err != nil {
return nil, err
func (t Tree) String() string {
s := []string{}
for _, n := range t {
s = append(s, n.String())
}
entries, err := fd.Readdir(-1)
if err != nil {
return nil, err
}
tree := &Tree{
Nodes: make([]*Node, 0, len(entries)),
}
for _, entry := range entries {
path := filepath.Join(dir, entry.Name())
node, err := NodeFromFileInfo(path, entry)
if err != nil {
return nil, err
}
node.repo = repo
tree.Nodes = append(tree.Nodes, node)
if entry.IsDir() {
node.Tree, err = NewTreeFromPath(repo, path)
if err != nil {
return nil, err
}
continue
}
if node.Type == "file" {
file, err := os.Open(path)
defer file.Close()
if err != nil {
return nil, err
}
if node.Size < chunker.MinSize {
// if the file is small enough, store it directly
id, err := store_chunk(repo, file)
if err != nil {
return nil, err
}
node.Content = []ID{id}
} else {
// else store chunks
node.Content = []ID{}
ch := chunker.New(file)
for {
chunk, err := ch.Next()
if err == io.EOF {
break
}
if err != nil {
return nil, err
}
id, err := store_chunk(repo, bytes.NewBuffer(chunk.Data))
node.Content = append(node.Content, id)
}
}
}
}
return tree, nil
return strings.Join(s, "\n")
}
func (tree *Tree) Save(repo *Repository) (ID, error) {
for _, node := range tree.Nodes {
if node.Tree != nil {
var err error
node.Subtree, err = node.Tree.Save(repo)
if err != nil {
return nil, err
}
}
}
data, err := json.Marshal(tree)
if err != nil {
return nil, err
}
id, err := repo.Create(TYPE_BLOB, data)
if err != nil {
return nil, err
}
return id, nil
}
func NewTreeFromRepo(repo *Repository, id ID) (*Tree, error) {
tree := NewTree()
rd, err := repo.Get(TYPE_BLOB, id)
defer rd.Close()
if err != nil {
return nil, err
}
decoder := json.NewDecoder(rd)
err = decoder.Decode(tree)
if err != nil {
return nil, err
}
for _, node := range tree.Nodes {
node.repo = repo
if node.Subtree != nil {
node.Tree, err = NewTreeFromRepo(repo, node.Subtree)
if err != nil {
return nil, err
}
}
}
return tree, nil
}
func (tree *Tree) CreateAt(path string) error {
for _, node := range tree.Nodes {
nodepath := filepath.Join(path, node.Name)
if node.Type == "dir" {
err := os.Mkdir(nodepath, 0700)
if err != nil {
fmt.Fprintf(os.Stderr, "%s\n", err)
continue
}
err = os.Chmod(nodepath, node.Mode)
if err != nil {
fmt.Fprintf(os.Stderr, "%s\n", err)
continue
}
err = os.Chown(nodepath, int(node.UID), int(node.GID))
if err != nil {
fmt.Fprintf(os.Stderr, "%s\n", err)
continue
}
err = node.Tree.CreateAt(filepath.Join(path, node.Name))
if err != nil {
return err
}
err = os.Chtimes(nodepath, node.AccessTime, node.ModTime)
if err != nil {
fmt.Fprintf(os.Stderr, "%s\n", err)
continue
}
} else {
err := node.CreateAt(nodepath)
if err != nil {
fmt.Fprintf(os.Stderr, "%s\n", err)
continue
}
}
}
return nil
}
// TODO: make sure that node.Type is valid
func (node *Node) fill_extra(path string, fi os.FileInfo) (err error) {
stat, ok := fi.Sys().(*syscall.Stat_t)
if !ok {
@ -290,6 +113,7 @@ func (node *Node) fill_extra(path string, fi os.FileInfo) (err error) {
func NodeFromFileInfo(path string, fi os.FileInfo) (*Node, error) {
node := &Node{
path: path,
Name: fi.Name(),
Mode: fi.Mode() & os.ModePerm,
ModTime: fi.ModTime(),
@ -316,12 +140,27 @@ func NodeFromFileInfo(path string, fi os.FileInfo) (*Node, error) {
return node, err
}
func (node *Node) CreateAt(path string) error {
if node.repo == nil {
return fmt.Errorf("repository is nil!")
func (node *Node) CreateAt(ch *ContentHandler, path string) error {
switch node.Type {
case "dir":
err := os.Mkdir(path, node.Mode)
if err != nil {
return err
}
switch node.Type {
err = os.Lchown(path, int(node.UID), int(node.GID))
if err != nil {
return err
}
var utimes = []syscall.Timespec{
syscall.NsecToTimespec(node.AccessTime.UnixNano()),
syscall.NsecToTimespec(node.ModTime.UnixNano()),
}
err = syscall.UtimesNano(path, utimes)
if err != nil {
return err
}
case "file":
// TODO: handle hard links
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0600)
@ -331,18 +170,32 @@ func (node *Node) CreateAt(path string) error {
}
for _, blobid := range node.Content {
rd, err := node.repo.Get(TYPE_BLOB, blobid)
buf, err := ch.Load(backend.Blob, blobid)
if err != nil {
return err
}
_, err = io.Copy(f, rd)
_, err = f.Write(buf)
if err != nil {
return err
}
}
f.Close()
err = os.Lchown(path, int(node.UID), int(node.GID))
if err != nil {
return err
}
var utimes = []syscall.Timespec{
syscall.NsecToTimespec(node.AccessTime.UnixNano()),
syscall.NsecToTimespec(node.ModTime.UnixNano()),
}
err = syscall.UtimesNano(path, utimes)
if err != nil {
return err
}
case "symlink":
err := os.Symlink(node.LinkTarget, path)
if err != nil {

54
tree_test.go Normal file
View File

@ -0,0 +1,54 @@
package khepri_test
import (
"io/ioutil"
"os"
"path/filepath"
"testing"
)
var testFiles = []struct {
name string
content []byte
}{
{"foo", []byte("bar")},
{"bar/foo2", []byte("bar2")},
{"bar/bla/blubb", []byte("This is just a test!\n")},
}
// prepare directory and return temporary path
func prepare_dir(t *testing.T) string {
tempdir, err := ioutil.TempDir("", "khepri-test-")
ok(t, err)
for _, test := range testFiles {
file := filepath.Join(tempdir, test.name)
dir := filepath.Dir(file)
if dir != "." {
ok(t, os.MkdirAll(dir, 0755))
}
f, err := os.Create(file)
defer func() {
ok(t, f.Close())
}()
ok(t, err)
_, err = f.Write(test.content)
ok(t, err)
}
t.Logf("tempdir prepared at %s", tempdir)
return tempdir
}
func TestTree(t *testing.T) {
dir := prepare_dir(t)
defer func() {
if *testCleanup {
ok(t, os.RemoveAll(dir))
}
}()
}