From 7c76ff3aaf94b9e6146cf5e67ad70f45efecbc17 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 8 May 2016 21:50:27 +0200 Subject: [PATCH] Allow reading backups from stdin --- src/cmds/restic/cmd_backup.go | 85 ++++++++++++++++++++++-- src/restic/archive_reader.go | 119 ++++++++++++++++++++++++++++++++++ 2 files changed, 200 insertions(+), 4 deletions(-) create mode 100644 src/restic/archive_reader.go diff --git a/src/cmds/restic/cmd_backup.go b/src/cmds/restic/cmd_backup.go index b32e7af59..5c47368bf 100644 --- a/src/cmds/restic/cmd_backup.go +++ b/src/cmds/restic/cmd_backup.go @@ -18,10 +18,12 @@ import ( ) type CmdBackup struct { - Parent string `short:"p" long:"parent" description:"use this parent snapshot (default: last snapshot in repo that has the same target)"` - Force bool `short:"f" long:"force" description:"Force re-reading the target. Overrides the \"parent\" flag"` - Excludes []string `short:"e" long:"exclude" description:"Exclude a pattern (can be specified multiple times)"` - ExcludeFile string `long:"exclude-file" description:"Read exclude-patterns from file"` + Parent string `short:"p" long:"parent" description:"use this parent snapshot (default: last snapshot in repo that has the same target)"` + Force bool `short:"f" long:"force" description:"Force re-reading the target. Overrides the \"parent\" flag"` + Excludes []string `short:"e" long:"exclude" description:"Exclude a pattern (can be specified multiple times)"` + ExcludeFile string `long:"exclude-file" description:"Read exclude-patterns from file"` + Stdin bool `long:"stdin" description:"read backup data from stdin"` + StdinFilename string `long:"stdin-filename" default:"stdin" description:"file name to use when reading from stdin"` global *GlobalOptions } @@ -175,6 +177,47 @@ func (cmd CmdBackup) newArchiveProgress(todo restic.Stat) *restic.Progress { return archiveProgress } +func (cmd CmdBackup) newArchiveStdinProgress() *restic.Progress { + if !cmd.global.ShowProgress() { + return nil + } + + archiveProgress := restic.NewProgress(time.Second) + + var bps uint64 + + archiveProgress.OnUpdate = func(s restic.Stat, d time.Duration, ticker bool) { + sec := uint64(d / time.Second) + if s.Bytes > 0 && sec > 0 && ticker { + bps = s.Bytes / sec + } + + status1 := fmt.Sprintf("[%s] %s %s/s", formatDuration(d), + formatBytes(s.Bytes), + formatBytes(bps)) + + w, _, err := terminal.GetSize(int(os.Stdout.Fd())) + if err == nil { + maxlen := w - len(status1) + + if maxlen < 4 { + status1 = "" + } else if len(status1) > maxlen { + status1 = status1[:maxlen-4] + status1 += "... " + } + } + + fmt.Printf("\x1b[2K%s\r", status1) + } + + archiveProgress.OnDone = func(s restic.Stat, d time.Duration, ticker bool) { + fmt.Printf("\nduration: %s, %s\n", formatDuration(d), formatRate(s.Bytes, d)) + } + + return archiveProgress +} + func samePaths(expected, actual []string) bool { if expected == nil || actual == nil { return true @@ -239,7 +282,41 @@ func filterExisting(items []string) (result []string, err error) { return } +func (cmd CmdBackup) readFromStdin(args []string) error { + if len(args) != 0 { + return fmt.Errorf("when reading from stdin, no additional files can be specified") + } + + repo, err := cmd.global.OpenRepository() + if err != nil { + return err + } + + lock, err := lockRepo(repo) + defer unlockRepo(lock) + if err != nil { + return err + } + + err = repo.LoadIndex() + if err != nil { + return err + } + + _, id, err := restic.ArchiveReader(repo, cmd.newArchiveStdinProgress(), os.Stdin, cmd.StdinFilename) + if err != nil { + return err + } + + fmt.Printf("archived as %v\n", id.Str()) + return nil +} + func (cmd CmdBackup) Execute(args []string) error { + if cmd.Stdin { + return cmd.readFromStdin(args) + } + if len(args) == 0 { return fmt.Errorf("wrong number of parameters, Usage: %s", cmd.Usage()) } diff --git a/src/restic/archive_reader.go b/src/restic/archive_reader.go new file mode 100644 index 000000000..02ba32f1d --- /dev/null +++ b/src/restic/archive_reader.go @@ -0,0 +1,119 @@ +package restic + +import ( + "encoding/json" + "io" + "restic/backend" + "restic/debug" + "restic/pack" + "restic/repository" + "time" + + "github.com/restic/chunker" +) + +// saveTreeJSON stores a tree in the repository. +func saveTreeJSON(repo *repository.Repository, item interface{}) (backend.ID, error) { + data, err := json.Marshal(item) + if err != nil { + return backend.ID{}, err + } + data = append(data, '\n') + + // check if tree has been saved before + id := backend.Hash(data) + if repo.Index().Has(id) { + return id, nil + } + + return repo.SaveJSON(pack.Tree, item) +} + +// ArchiveReader reads from the reader and archives the data. Returned is the +// resulting snapshot and its ID. +func ArchiveReader(repo *repository.Repository, p *Progress, rd io.Reader, name string) (*Snapshot, backend.ID, error) { + debug.Log("ArchiveReader", "start archiving %s", name) + sn, err := NewSnapshot([]string{name}) + if err != nil { + return nil, backend.ID{}, err + } + + p.Start() + defer p.Done() + + chnker := chunker.New(rd, repo.Config.ChunkerPolynomial) + + var ids backend.IDs + + for { + chunk, err := chnker.Next(getBuf()) + if err == io.EOF { + break + } + + if err != nil { + return nil, backend.ID{}, err + } + + id := backend.Hash(chunk.Data) + + if !repo.Index().Has(id) { + _, err := repo.SaveAndEncrypt(pack.Data, chunk.Data, nil) + if err != nil { + return nil, backend.ID{}, err + } + debug.Log("ArchiveReader", "saved blob %v (%d bytes)\n", id.Str(), chunk.Length) + } else { + debug.Log("ArchiveReader", "blob %v already saved in the repo\n", id.Str()) + } + + freeBuf(chunk.Data) + + ids = append(ids, id) + + p.Report(Stat{Bytes: uint64(chunk.Length)}) + } + + tree := &Tree{ + Nodes: []*Node{ + &Node{ + Name: name, + AccessTime: time.Now(), + ModTime: time.Now(), + Type: "file", + Mode: 0644, + UID: sn.UID, + GID: sn.GID, + User: sn.Username, + Content: ids, + }, + }, + } + + treeID, err := saveTreeJSON(repo, tree) + if err != nil { + return nil, backend.ID{}, err + } + sn.Tree = &treeID + debug.Log("ArchiveReader", "tree saved as %v", treeID.Str()) + + id, err := repo.SaveJSONUnpacked(backend.Snapshot, sn) + if err != nil { + return nil, backend.ID{}, err + } + + sn.id = &id + debug.Log("ArchiveReader", "snapshot saved as %v", id.Str()) + + err = repo.Flush() + if err != nil { + return nil, backend.ID{}, err + } + + err = repo.SaveIndex() + if err != nil { + return nil, backend.ID{}, err + } + + return sn, id, nil +}