From fe92062735f6885507ddfbbb06c77fe7d2ad1497 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Mon, 17 Nov 2014 23:28:51 +0100 Subject: [PATCH] Move SaveFile to Archiver, add blobs status --- archiver.go | 58 +++++++++++++++++++++++++++++++++++++++++------ contenthandler.go | 53 ------------------------------------------- 2 files changed, 51 insertions(+), 60 deletions(-) diff --git a/archiver.go b/archiver.go index 028e94162..182ce72cb 100644 --- a/archiver.go +++ b/archiver.go @@ -1,11 +1,14 @@ package khepri import ( + "io" + "io/ioutil" "os" "path/filepath" "sync" "github.com/fd0/khepri/backend" + "github.com/fd0/khepri/chunker" ) const ( @@ -111,10 +114,55 @@ func (arch *Archiver) SaveJSON(t backend.Type, item interface{}) (*Blob, error) return blob, nil } +// SaveFile stores the content of the file on the backend as a Blob by calling +// Save for each chunk. func (arch *Archiver) SaveFile(node *Node) error { - blobs, err := arch.ch.SaveFile(node.path, uint(node.Size)) + file, err := os.Open(node.path) + defer file.Close() if err != nil { - return arch.Error(node.path, nil, err) + return err + } + + var blobs Blobs + + // if the file is small enough, store it directly + if node.Size < chunker.MinSize { + buf, err := ioutil.ReadAll(file) + if err != nil { + return err + } + + blob, err := arch.ch.Save(backend.Data, buf) + if err != nil { + return err + } + + arch.saveUpdate(Stats{Bytes: blob.Size}) + + blobs = Blobs{blob} + } else { + // else store all chunks + chunker := chunker.New(file) + + for { + chunk, err := chunker.Next() + if err == io.EOF { + break + } + + if err != nil { + return err + } + + blob, err := arch.ch.Save(backend.Data, chunk.Data) + if err != nil { + return err + } + + arch.saveUpdate(Stats{Bytes: blob.Size}) + + blobs = append(blobs, blob) + } } node.Content = make([]backend.ID, len(blobs)) @@ -233,13 +281,9 @@ func (arch *Archiver) saveTree(t *Tree) (*Blob, error) { arch.fileToken <- token }() - // debug("start: %s", n.path) - // TODO: handle error arch.SaveFile(n) - arch.saveUpdate(Stats{Files: 1, Bytes: n.Size}) - - // debug("done: %s", n.path) + arch.saveUpdate(Stats{Files: 1}) }(node) } else { arch.saveUpdate(Stats{Other: 1}) diff --git a/contenthandler.go b/contenthandler.go index 514bc0fe3..eaf9b8bd5 100644 --- a/contenthandler.go +++ b/contenthandler.go @@ -3,13 +3,9 @@ package khepri import ( "encoding/json" "errors" - "io" - "io/ioutil" - "os" "sync" "github.com/fd0/khepri/backend" - "github.com/fd0/khepri/chunker" ) type ContentHandler struct { @@ -118,55 +114,6 @@ func (ch *ContentHandler) SaveJSON(t backend.Type, item interface{}) (*Blob, err return ch.Save(t, backend.Compress(data)) } -// SaveFile stores the content of the file on the backend as a Blob by calling -// Save for each chunk. -func (ch *ContentHandler) SaveFile(filename string, size uint) (Blobs, error) { - file, err := os.Open(filename) - defer file.Close() - if err != nil { - return nil, err - } - - // if the file is small enough, store it directly - if size < chunker.MinSize { - buf, err := ioutil.ReadAll(file) - if err != nil { - return nil, err - } - - blob, err := ch.Save(backend.Data, buf) - if err != nil { - return nil, err - } - - return Blobs{blob}, nil - } - - // else store all chunks - blobs := Blobs{} - chunker := chunker.New(file) - - for { - chunk, err := chunker.Next() - if err == io.EOF { - break - } - - if err != nil { - return nil, err - } - - blob, err := ch.Save(backend.Data, chunk.Data) - if err != nil { - return nil, err - } - - blobs = append(blobs, blob) - } - - return blobs, nil -} - // Load tries to load and decrypt content identified by t and id from the backend. func (ch *ContentHandler) Load(t backend.Type, id backend.ID) ([]byte, error) { if t == backend.Snapshot {