Allow reading backups from stdin

src/cmds/restic/cmd_backup.go

@@ -18,10 +18,12 @@ import (
 )
 
 type CmdBackup struct {
-	Parent      string   `short:"p" long:"parent"  description:"use this parent snapshot (default: last snapshot in repo that has the same target)"`
+	Parent        string   `short:"p" long:"parent"                  description:"use this parent snapshot (default: last snapshot in repo that has the same target)"`
-	Force       bool     `short:"f" long:"force"   description:"Force re-reading the target. Overrides the \"parent\" flag"`
+	Force         bool     `short:"f" long:"force"                   description:"Force re-reading the target. Overrides the \"parent\" flag"`
-	Excludes    []string `short:"e" long:"exclude" description:"Exclude a pattern (can be specified multiple times)"`
+	Excludes      []string `short:"e" long:"exclude"                 description:"Exclude a pattern (can be specified multiple times)"`
-	ExcludeFile string   `long:"exclude-file" description:"Read exclude-patterns from file"`
+	ExcludeFile   string   `long:"exclude-file"                      description:"Read exclude-patterns from file"`
+	Stdin         bool     `long:"stdin"                             description:"read backup data from stdin"`
+	StdinFilename string   `long:"stdin-filename"    default:"stdin" description:"file name to use when reading from stdin"`
 
 	global *GlobalOptions
 }
@@ -175,6 +177,47 @@ func (cmd CmdBackup) newArchiveProgress(todo restic.Stat) *restic.Progress {
 	return archiveProgress
 }
 
+func (cmd CmdBackup) newArchiveStdinProgress() *restic.Progress {
+	if !cmd.global.ShowProgress() {
+		return nil
+	}
+
+	archiveProgress := restic.NewProgress(time.Second)
+
+	var bps uint64
+
+	archiveProgress.OnUpdate = func(s restic.Stat, d time.Duration, ticker bool) {
+		sec := uint64(d / time.Second)
+		if s.Bytes > 0 && sec > 0 && ticker {
+			bps = s.Bytes / sec
+		}
+
+		status1 := fmt.Sprintf("[%s] %s  %s/s", formatDuration(d),
+			formatBytes(s.Bytes),
+			formatBytes(bps))
+
+		w, _, err := terminal.GetSize(int(os.Stdout.Fd()))
+		if err == nil {
+			maxlen := w - len(status1)
+
+			if maxlen < 4 {
+				status1 = ""
+			} else if len(status1) > maxlen {
+				status1 = status1[:maxlen-4]
+				status1 += "... "
+			}
+		}
+
+		fmt.Printf("\x1b[2K%s\r", status1)
+	}
+
+	archiveProgress.OnDone = func(s restic.Stat, d time.Duration, ticker bool) {
+		fmt.Printf("\nduration: %s, %s\n", formatDuration(d), formatRate(s.Bytes, d))
+	}
+
+	return archiveProgress
+}
+
 func samePaths(expected, actual []string) bool {
 	if expected == nil || actual == nil {
 		return true
@@ -239,7 +282,41 @@ func filterExisting(items []string) (result []string, err error) {
 	return
 }
 
+func (cmd CmdBackup) readFromStdin(args []string) error {
+	if len(args) != 0 {
+		return fmt.Errorf("when reading from stdin, no additional files can be specified")
+	}
+
+	repo, err := cmd.global.OpenRepository()
+	if err != nil {
+		return err
+	}
+
+	lock, err := lockRepo(repo)
+	defer unlockRepo(lock)
+	if err != nil {
+		return err
+	}
+
+	err = repo.LoadIndex()
+	if err != nil {
+		return err
+	}
+
+	_, id, err := restic.ArchiveReader(repo, cmd.newArchiveStdinProgress(), os.Stdin, cmd.StdinFilename)
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf("archived as %v\n", id.Str())
+	return nil
+}
+
 func (cmd CmdBackup) Execute(args []string) error {
+	if cmd.Stdin {
+		return cmd.readFromStdin(args)
+	}
+
 	if len(args) == 0 {
 		return fmt.Errorf("wrong number of parameters, Usage: %s", cmd.Usage())
 	}

src/restic/archive_reader.go

@@ -0,0 +1,119 @@
+package restic
+
+import (
+	"encoding/json"
+	"io"
+	"restic/backend"
+	"restic/debug"
+	"restic/pack"
+	"restic/repository"
+	"time"
+
+	"github.com/restic/chunker"
+)
+
+// saveTreeJSON stores a tree in the repository.
+func saveTreeJSON(repo *repository.Repository, item interface{}) (backend.ID, error) {
+	data, err := json.Marshal(item)
+	if err != nil {
+		return backend.ID{}, err
+	}
+	data = append(data, '\n')
+
+	// check if tree has been saved before
+	id := backend.Hash(data)
+	if repo.Index().Has(id) {
+		return id, nil
+	}
+
+	return repo.SaveJSON(pack.Tree, item)
+}
+
+// ArchiveReader reads from the reader and archives the data. Returned is the
+// resulting snapshot and its ID.
+func ArchiveReader(repo *repository.Repository, p *Progress, rd io.Reader, name string) (*Snapshot, backend.ID, error) {
+	debug.Log("ArchiveReader", "start archiving %s", name)
+	sn, err := NewSnapshot([]string{name})
+	if err != nil {
+		return nil, backend.ID{}, err
+	}
+
+	p.Start()
+	defer p.Done()
+
+	chnker := chunker.New(rd, repo.Config.ChunkerPolynomial)
+
+	var ids backend.IDs
+
+	for {
+		chunk, err := chnker.Next(getBuf())
+		if err == io.EOF {
+			break
+		}
+
+		if err != nil {
+			return nil, backend.ID{}, err
+		}
+
+		id := backend.Hash(chunk.Data)
+
+		if !repo.Index().Has(id) {
+			_, err := repo.SaveAndEncrypt(pack.Data, chunk.Data, nil)
+			if err != nil {
+				return nil, backend.ID{}, err
+			}
+			debug.Log("ArchiveReader", "saved blob %v (%d bytes)\n", id.Str(), chunk.Length)
+		} else {
+			debug.Log("ArchiveReader", "blob %v already saved in the repo\n", id.Str())
+		}
+
+		freeBuf(chunk.Data)
+
+		ids = append(ids, id)
+
+		p.Report(Stat{Bytes: uint64(chunk.Length)})
+	}
+
+	tree := &Tree{
+		Nodes: []*Node{
+			&Node{
+				Name:       name,
+				AccessTime: time.Now(),
+				ModTime:    time.Now(),
+				Type:       "file",
+				Mode:       0644,
+				UID:        sn.UID,
+				GID:        sn.GID,
+				User:       sn.Username,
+				Content:    ids,
+			},
+		},
+	}
+
+	treeID, err := saveTreeJSON(repo, tree)
+	if err != nil {
+		return nil, backend.ID{}, err
+	}
+	sn.Tree = &treeID
+	debug.Log("ArchiveReader", "tree saved as %v", treeID.Str())
+
+	id, err := repo.SaveJSONUnpacked(backend.Snapshot, sn)
+	if err != nil {
+		return nil, backend.ID{}, err
+	}
+
+	sn.id = &id
+	debug.Log("ArchiveReader", "snapshot saved as %v", id.Str())
+
+	err = repo.Flush()
+	if err != nil {
+		return nil, backend.ID{}, err
+	}
+
+	err = repo.SaveIndex()
+	if err != nil {
+		return nil, backend.ID{}, err
+	}
+
+	return sn, id, nil
+}