2
2
mirror of https://github.com/octoleo/restic.git synced 2024-11-02 11:46:36 +00:00
restic/cmd/restic/cmd_backup.go
Alexander Neumann 6caeff2408 Run goimports
2017-07-23 14:21:03 +02:00

503 lines
12 KiB
Go

package main
import (
"bufio"
"context"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"time"
"github.com/restic/restic/internal"
"github.com/spf13/cobra"
"github.com/restic/restic/internal/archiver"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/filter"
"github.com/restic/restic/internal/fs"
)
var cmdBackup = &cobra.Command{
Use: "backup [flags] FILE/DIR [FILE/DIR] ...",
Short: "create a new backup of files and/or directories",
Long: `
The "backup" command creates a new snapshot and saves the files and directories
given as the arguments.
`,
RunE: func(cmd *cobra.Command, args []string) error {
if backupOptions.Stdin && backupOptions.FilesFrom == "-" {
return errors.Fatal("cannot use both `--stdin` and `--files-from -`")
}
if backupOptions.Stdin {
return readBackupFromStdin(backupOptions, globalOptions, args)
}
return runBackup(backupOptions, globalOptions, args)
},
}
// BackupOptions bundles all options for the backup command.
type BackupOptions struct {
Parent string
Force bool
Excludes []string
ExcludeFiles []string
ExcludeOtherFS bool
Stdin bool
StdinFilename string
Tags []string
Hostname string
FilesFrom string
}
var backupOptions BackupOptions
func init() {
cmdRoot.AddCommand(cmdBackup)
hostname, err := os.Hostname()
if err != nil {
debug.Log("os.Hostname() returned err: %v", err)
hostname = ""
}
f := cmdBackup.Flags()
f.StringVar(&backupOptions.Parent, "parent", "", "use this parent snapshot (default: last snapshot in the repo that has the same target files/directories)")
f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the target files/directories (overrides the "parent" flag)`)
f.StringArrayVarP(&backupOptions.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
f.StringArrayVar(&backupOptions.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)")
f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems")
f.BoolVar(&backupOptions.Stdin, "stdin", false, "read backup from stdin")
f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "stdin", "file name to use when reading from stdin")
f.StringArrayVar(&backupOptions.Tags, "tag", nil, "add a `tag` for the new snapshot (can be specified multiple times)")
f.StringVar(&backupOptions.Hostname, "hostname", hostname, "set the `hostname` for the snapshot manually")
f.StringVar(&backupOptions.FilesFrom, "files-from", "", "read the files to backup from file (can be combined with file args)")
}
func newScanProgress(gopts GlobalOptions) *restic.Progress {
if gopts.Quiet {
return nil
}
p := restic.NewProgress()
p.OnUpdate = func(s restic.Stat, d time.Duration, ticker bool) {
if IsProcessBackground() {
return
}
PrintProgress("[%s] %d directories, %d files, %s", formatDuration(d), s.Dirs, s.Files, formatBytes(s.Bytes))
}
p.OnDone = func(s restic.Stat, d time.Duration, ticker bool) {
PrintProgress("scanned %d directories, %d files in %s\n", s.Dirs, s.Files, formatDuration(d))
}
return p
}
func newArchiveProgress(gopts GlobalOptions, todo restic.Stat) *restic.Progress {
if gopts.Quiet {
return nil
}
archiveProgress := restic.NewProgress()
var bps, eta uint64
itemsTodo := todo.Files + todo.Dirs
archiveProgress.OnUpdate = func(s restic.Stat, d time.Duration, ticker bool) {
if IsProcessBackground() {
return
}
sec := uint64(d / time.Second)
if todo.Bytes > 0 && sec > 0 && ticker {
bps = s.Bytes / sec
if s.Bytes >= todo.Bytes {
eta = 0
} else if bps > 0 {
eta = (todo.Bytes - s.Bytes) / bps
}
}
itemsDone := s.Files + s.Dirs
status1 := fmt.Sprintf("[%s] %s %s/s %s / %s %d / %d items %d errors ",
formatDuration(d),
formatPercent(s.Bytes, todo.Bytes),
formatBytes(bps),
formatBytes(s.Bytes), formatBytes(todo.Bytes),
itemsDone, itemsTodo,
s.Errors)
status2 := fmt.Sprintf("ETA %s ", formatSeconds(eta))
if w := stdoutTerminalWidth(); w > 0 {
maxlen := w - len(status2) - 1
if maxlen < 4 {
status1 = ""
} else if len(status1) > maxlen {
status1 = status1[:maxlen-4]
status1 += "... "
}
}
PrintProgress("%s%s", status1, status2)
}
archiveProgress.OnDone = func(s restic.Stat, d time.Duration, ticker bool) {
fmt.Printf("\nduration: %s, %s\n", formatDuration(d), formatRate(todo.Bytes, d))
}
return archiveProgress
}
func newArchiveStdinProgress(gopts GlobalOptions) *restic.Progress {
if gopts.Quiet {
return nil
}
archiveProgress := restic.NewProgress()
var bps uint64
archiveProgress.OnUpdate = func(s restic.Stat, d time.Duration, ticker bool) {
if IsProcessBackground() {
return
}
sec := uint64(d / time.Second)
if s.Bytes > 0 && sec > 0 && ticker {
bps = s.Bytes / sec
}
status1 := fmt.Sprintf("[%s] %s %s/s", formatDuration(d),
formatBytes(s.Bytes),
formatBytes(bps))
if w := stdoutTerminalWidth(); w > 0 {
maxlen := w - len(status1)
if maxlen < 4 {
status1 = ""
} else if len(status1) > maxlen {
status1 = status1[:maxlen-4]
status1 += "... "
}
}
PrintProgress("%s", status1)
}
archiveProgress.OnDone = func(s restic.Stat, d time.Duration, ticker bool) {
fmt.Printf("\nduration: %s, %s\n", formatDuration(d), formatRate(s.Bytes, d))
}
return archiveProgress
}
// filterExisting returns a slice of all existing items, or an error if no
// items exist at all.
func filterExisting(items []string) (result []string, err error) {
for _, item := range items {
_, err := fs.Lstat(item)
if err != nil && os.IsNotExist(errors.Cause(err)) {
continue
}
result = append(result, item)
}
if len(result) == 0 {
return nil, errors.Fatal("all target directories/files do not exist")
}
return
}
// gatherDevices returns the set of unique device ids of the files and/or
// directory paths listed in "items".
func gatherDevices(items []string) (deviceMap map[string]uint64, err error) {
deviceMap = make(map[string]uint64)
for _, item := range items {
fi, err := fs.Lstat(item)
if err != nil {
return nil, err
}
id, err := fs.DeviceID(fi)
if err != nil {
return nil, err
}
deviceMap[item] = id
}
if len(deviceMap) == 0 {
return nil, errors.New("zero allowed devices")
}
return deviceMap, nil
}
func readBackupFromStdin(opts BackupOptions, gopts GlobalOptions, args []string) error {
if len(args) != 0 {
return errors.Fatal("when reading from stdin, no additional files can be specified")
}
if opts.StdinFilename == "" {
return errors.Fatal("filename for backup from stdin must not be empty")
}
if gopts.password == "" && gopts.PasswordFile == "" {
return errors.Fatal("unable to read password from stdin when data is to be read from stdin, use --password-file or $RESTIC_PASSWORD")
}
repo, err := OpenRepository(gopts)
if err != nil {
return err
}
lock, err := lockRepo(repo)
defer unlockRepo(lock)
if err != nil {
return err
}
err = repo.LoadIndex(context.TODO())
if err != nil {
return err
}
r := &archiver.Reader{
Repository: repo,
Tags: opts.Tags,
Hostname: opts.Hostname,
}
_, id, err := r.Archive(context.TODO(), opts.StdinFilename, os.Stdin, newArchiveStdinProgress(gopts))
if err != nil {
return err
}
Verbosef("archived as %v\n", id.Str())
return nil
}
// readFromFile will read all lines from the given filename and write them to a
// string array, if filename is empty readFromFile returns and empty string
// array. If filename is a dash (-), readFromFile will read the lines from
// the standard input.
func readLinesFromFile(filename string) ([]string, error) {
if filename == "" {
return nil, nil
}
var r io.Reader = os.Stdin
if filename != "-" {
f, err := os.Open(filename)
if err != nil {
return nil, err
}
defer f.Close()
r = f
}
var lines []string
scanner := bufio.NewScanner(r)
for scanner.Scan() {
line := scanner.Text()
if line == "" {
continue
}
lines = append(lines, line)
}
if err := scanner.Err(); err != nil {
return nil, err
}
return lines, nil
}
func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
if opts.FilesFrom == "-" && gopts.password == "" && gopts.PasswordFile == "" {
return errors.Fatal("no password; either use `--password-file` option or put the password into the RESTIC_PASSWORD environment variable")
}
fromfile, err := readLinesFromFile(opts.FilesFrom)
if err != nil {
return err
}
// merge files from files-from into normal args so we can reuse the normal
// args checks and have the ability to use both files-from and args at the
// same time
args = append(args, fromfile...)
if len(args) == 0 {
return errors.Fatal("wrong number of parameters")
}
target := make([]string, 0, len(args))
for _, d := range args {
if a, err := filepath.Abs(d); err == nil {
d = a
}
target = append(target, d)
}
target, err = filterExisting(target)
if err != nil {
return err
}
// allowed devices
var allowedDevs map[string]uint64
if opts.ExcludeOtherFS {
allowedDevs, err = gatherDevices(target)
if err != nil {
return err
}
debug.Log("allowed devices: %v\n", allowedDevs)
}
repo, err := OpenRepository(gopts)
if err != nil {
return err
}
lock, err := lockRepo(repo)
defer unlockRepo(lock)
if err != nil {
return err
}
err = repo.LoadIndex(context.TODO())
if err != nil {
return err
}
var parentSnapshotID *restic.ID
// Force using a parent
if !opts.Force && opts.Parent != "" {
id, err := restic.FindSnapshot(repo, opts.Parent)
if err != nil {
return errors.Fatalf("invalid id %q: %v", opts.Parent, err)
}
parentSnapshotID = &id
}
// Find last snapshot to set it as parent, if not already set
if !opts.Force && parentSnapshotID == nil {
id, err := restic.FindLatestSnapshot(context.TODO(), repo, target, []restic.TagList{opts.Tags}, opts.Hostname)
if err == nil {
parentSnapshotID = &id
} else if err != restic.ErrNoSnapshotFound {
return err
}
}
if parentSnapshotID != nil {
Verbosef("using parent snapshot %v\n", parentSnapshotID.Str())
}
Verbosef("scan %v\n", target)
// add patterns from file
if len(opts.ExcludeFiles) > 0 {
for _, filename := range opts.ExcludeFiles {
file, err := fs.Open(filename)
if err != nil {
Warnf("error reading exclude patterns: %v", err)
return nil
}
scanner := bufio.NewScanner(file)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
// ignore empty lines
if line == "" {
continue
}
// strip comments
if strings.HasPrefix(line, "#") {
continue
}
line = os.ExpandEnv(line)
opts.Excludes = append(opts.Excludes, line)
}
}
}
selectFilter := func(item string, fi os.FileInfo) bool {
matched, err := filter.List(opts.Excludes, item)
if err != nil {
Warnf("error for exclude pattern: %v", err)
}
if matched {
debug.Log("path %q excluded by a filter", item)
return false
}
if !opts.ExcludeOtherFS || fi == nil {
return true
}
id, err := fs.DeviceID(fi)
if err != nil {
// This should never happen because gatherDevices() would have
// errored out earlier. If it still does that's a reason to panic.
panic(err)
}
for dir := item; dir != ""; dir = filepath.Dir(dir) {
debug.Log("item %v, test dir %v", item, dir)
allowedID, ok := allowedDevs[dir]
if !ok {
continue
}
if allowedID != id {
debug.Log("path %q on disallowed device %d", item, id)
return false
}
return true
}
panic(fmt.Sprintf("item %v, device id %v not found, allowedDevs: %v", item, id, allowedDevs))
}
stat, err := archiver.Scan(target, selectFilter, newScanProgress(gopts))
if err != nil {
return err
}
arch := archiver.New(repo)
arch.Excludes = opts.Excludes
arch.SelectFilter = selectFilter
arch.Warn = func(dir string, fi os.FileInfo, err error) {
// TODO: make ignoring errors configurable
Warnf("%s\rwarning for %s: %v\n", ClearLine(), dir, err)
}
_, id, err := arch.Snapshot(context.TODO(), newArchiveProgress(gopts, stat), target, opts.Tags, opts.Hostname, parentSnapshotID)
if err != nil {
return err
}
Verbosef("snapshot %s saved\n", id.Str())
return nil
}