octoleo
/
restic
mirror of https://github.com/octoleo/restic.git
2
2
Fork 0
Code Issues Releases Wiki Activity

Scan directory first, then backup. Add stats.

This commit is contained in:
Alexander Neumann 2014-11-16 21:29:11 +01:00
parent 616a2c749d
commit 4a3a6861e2
3 changed files with 168 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

156
archiver.go
View File

@ -13,8 +13,20 @@ type Archiver struct {
ch *ContentHandler
smap *StorageMap // blobs used for the current snapshot
Stats Stats
Error func(dir string, fi os.FileInfo, err error) error
Filter func(item string, fi os.FileInfo) bool
ScannerUpdate func(stats Stats)
SaveUpdate func(stats Stats)
}
type Stats struct {
Files int
Directories int
Other int
Bytes uint64
}
func NewArchiver(be backend.Server, key *Key) (*Archiver, error) {
@ -25,6 +37,9 @@ func NewArchiver(be backend.Server, key *Key) (*Archiver, error) {
arch.Error = func(string, os.FileInfo, error) error { return err }
// allow all files
arch.Filter = func(string, os.FileInfo) bool { return true }
// do nothing
arch.ScannerUpdate = func(Stats) {}
arch.SaveUpdate = func(Stats) {}
arch.smap = NewStorageMap()
arch.ch, err = NewContentHandler(be, key)
@ -65,10 +80,10 @@ func (arch *Archiver) SaveJSON(t backend.Type, item interface{}) (*Blob, error)
return blob, nil
}
func (arch *Archiver) SaveFile(node *Node) (Blobs, error) {
func (arch *Archiver) SaveFile(node *Node) error {
blobs, err := arch.ch.SaveFile(node.path, uint(node.Size))
if err != nil {
return nil, arch.Error(node.path, nil, err)
return arch.Error(node.path, nil, err)
}
node.Content = make([]backend.ID, len(blobs))
@ -77,23 +92,20 @@ func (arch *Archiver) SaveFile(node *Node) (Blobs, error) {
arch.smap.Insert(blob)
}
return blobs, err
return err
}
func (arch *Archiver) ImportDir(dir string) (Tree, error) {
func (arch *Archiver) loadTree(dir string) (*Tree, error) {
// open and list path
fd, err := os.Open(dir)
defer fd.Close()
if err != nil {
return nil, arch.Error(dir, nil, err)
return nil, err
}
entries, err := fd.Readdir(-1)
if err != nil {
return nil, arch.Error(dir, nil, err)
}
if len(entries) == 0 {
return nil, nil
return nil, err
}
tree := Tree{}
@ -107,71 +119,97 @@ func (arch *Archiver) ImportDir(dir string) (Tree, error) {
node, err := NodeFromFileInfo(path, entry)
if err != nil {
return nil, arch.Error(dir, entry, err)
// TODO: error processing
return nil, err
}
tree = append(tree, node)
if entry.IsDir() {
subtree, err := arch.ImportDir(path)
node.Tree, err = arch.loadTree(path)
if err != nil {
return nil, err
}
blob, err := arch.SaveJSON(backend.Tree, subtree)
if err != nil {
return nil, err
}
node.Subtree = blob.ID
continue
}
if node.Type == "file" {
_, err := arch.SaveFile(node)
if err != nil {
return nil, arch.Error(path, entry, err)
}
switch node.Type {
case "file":
arch.Stats.Files++
arch.Stats.Bytes += node.Size
case "dir":
arch.Stats.Directories++
default:
arch.Stats.Other++
}
}
return tree, nil
arch.ScannerUpdate(arch.Stats)
return &tree, nil
}
func (arch *Archiver) Import(dir string) (*Snapshot, *Blob, error) {
func (arch *Archiver) LoadTree(path string) (*Tree, error) {
fi, err := os.Lstat(path)
if err != nil {
return nil, err
}
node, err := NodeFromFileInfo(path, fi)
if err != nil {
return nil, err
}
if node.Type != "dir" {
arch.Stats.Files = 1
arch.Stats.Bytes = node.Size
arch.ScannerUpdate(arch.Stats)
return &Tree{node}, nil
}
arch.Stats.Directories = 1
node.Tree, err = arch.loadTree(path)
if err != nil {
return nil, err
}
arch.ScannerUpdate(arch.Stats)
return &Tree{node}, nil
}
func (arch *Archiver) saveTree(t *Tree) (*Blob, error) {
for _, node := range *t {
if node.Tree != nil && node.Subtree == nil {
b, err := arch.saveTree(node.Tree)
if err != nil {
return nil, err
}
node.Subtree = b.ID
arch.SaveUpdate(Stats{Directories: 1})
} else if node.Type == "file" && len(node.Content) == 0 {
err := arch.SaveFile(node)
if err != nil {
return nil, err
}
arch.SaveUpdate(Stats{Files: 1, Bytes: node.Size})
} else {
arch.SaveUpdate(Stats{Other: 1})
}
}
blob, err := arch.SaveJSON(backend.Tree, t)
if err != nil {
return nil, err
}
return blob, nil
}
func (arch *Archiver) Snapshot(dir string, t *Tree) (*Snapshot, backend.ID, error) {
sn := NewSnapshot(dir)
fi, err := os.Lstat(dir)
if err != nil {
return nil, nil, err
}
node, err := NodeFromFileInfo(dir, fi)
if err != nil {
return nil, nil, err
}
if node.Type == "dir" {
tree, err := arch.ImportDir(dir)
if err != nil {
return nil, nil, err
}
blob, err := arch.SaveJSON(backend.Tree, tree)
if err != nil {
return nil, nil, err
}
node.Subtree = blob.ID
} else if node.Type == "file" {
_, err := arch.SaveFile(node)
if err != nil {
return nil, nil, err
}
}
blob, err := arch.SaveJSON(backend.Tree, &Tree{node})
blob, err := arch.saveTree(t)
if err != nil {
return nil, nil, err
}
@ -185,5 +223,5 @@ func (arch *Archiver) Import(dir string) (*Snapshot, *Blob, error) {
return nil, nil, err
}
return sn, blob, nil
return sn, blob.Storage, nil
}

71
cmd/khepri/cmd_backup.go
View File

@ -4,11 +4,41 @@ import (
"errors"
"fmt"
"os"
"strings"
"github.com/fd0/khepri"
"github.com/fd0/khepri/backend"
"golang.org/x/crypto/ssh/terminal"
)
func format_bytes(c uint64) string {
b := float64(c)
switch {
case c > 1<<40:
return fmt.Sprintf("%.3f TiB", b/(1<<40))
case c > 1<<30:
return fmt.Sprintf("%.3f GiB", b/(1<<30))
case c > 1<<20:
return fmt.Sprintf("%.3f MiB", b/(1<<20))
case c > 1<<10:
return fmt.Sprintf("%.3f KiB", b/(1<<10))
default:
return fmt.Sprintf("%d B", c)
}
}
func print_tree2(indent int, t *khepri.Tree) {
for _, node := range *t {
if node.Tree != nil {
fmt.Printf("%s%s/\n", strings.Repeat(" ", indent), node.Name)
print_tree2(indent+1, node.Tree)
} else {
fmt.Printf("%s%s\n", strings.Repeat(" ", indent), node.Name)
}
}
}
func commandBackup(be backend.Server, key *khepri.Key, args []string) error {
if len(args) != 1 {
return errors.New("usage: backup [dir|file]")
@ -25,12 +55,49 @@ func commandBackup(be backend.Server, key *khepri.Key, args []string) error {
return err
}
_, blob, err := arch.Import(target)
fmt.Printf("scanning %s\n", target)
if terminal.IsTerminal(int(os.Stdout.Fd())) {
arch.ScannerUpdate = func(stats khepri.Stats) {
fmt.Printf("\r%6d directories, %6d files, %14s", stats.Directories, stats.Files, format_bytes(stats.Bytes))
}
}
// TODO: add filter
// arch.Filter = func(dir string, fi os.FileInfo) bool {
// return true
// }
t, err := arch.LoadTree(target)
if err != nil {
fmt.Fprintf(os.Stderr, "error: %v\n", err)
return err
}
fmt.Printf("snapshot %s saved\n", blob.Storage)
fmt.Printf("\r%6d directories, %6d files, %14s\n", arch.Stats.Directories, arch.Stats.Files, format_bytes(arch.Stats.Bytes))
stats := khepri.Stats{}
if terminal.IsTerminal(int(os.Stdout.Fd())) {
arch.SaveUpdate = func(s khepri.Stats) {
stats.Files += s.Files
stats.Directories += s.Directories
stats.Other += s.Other
stats.Bytes += s.Bytes
fmt.Printf("\r%3.2f%% %d/%d directories, %d/%d files, %s/%s",
float64(stats.Bytes)/float64(arch.Stats.Bytes)*100,
stats.Directories, arch.Stats.Directories,
stats.Files, arch.Stats.Files,
format_bytes(stats.Bytes), format_bytes(arch.Stats.Bytes))
}
}
sn, id, err := arch.Snapshot(target, t)
if err != nil {
fmt.Fprintf(os.Stderr, "error: %v\n", err)
}
fmt.Printf("\nsnapshot %s saved: %v\n", id, sn)
return nil
}

2
tree.go
View File

@ -34,6 +34,8 @@ type Node struct {
Content []backend.ID `json:"content,omitempty"`
Subtree backend.ID `json:"subtree,omitempty"`
Tree *Tree `json:"-"`
path string
}