From a8657bde689e36ea32a2e11a11f90b169908f06d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Ho=C3=9F?= Date: Thu, 16 Mar 2023 07:41:58 +0100 Subject: [PATCH 01/17] Add --stdin-from-command option MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This new flag is added to the backup subcommand in order to allow restic to control the execution of a command and determine whether to save a snapshot if the given command succeeds. Signed-off-by: Sebastian Hoß --- cmd/restic/cmd_backup.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index 9499701aa..d07b89a5d 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -97,6 +97,7 @@ type BackupOptions struct { ExcludeLargerThan string Stdin bool StdinFilename string + StdinCommand bool Tags restic.TagLists Host string FilesFrom []string @@ -134,6 +135,7 @@ func init() { f.StringVar(&backupOptions.ExcludeLargerThan, "exclude-larger-than", "", "max `size` of the files to be backed up (allowed suffixes: k/K, m/M, g/G, t/T)") f.BoolVar(&backupOptions.Stdin, "stdin", false, "read backup from stdin") f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "stdin", "`filename` to use when reading from stdin") + f.BoolVar(&backupOptions.StdinCommand, "stdin-from-command", false, "execute command and store its stdout") f.Var(&backupOptions.Tags, "tag", "add `tags` for the new snapshot in the format `tag[,tag,...]` (can be specified multiple times)") f.UintVar(&backupOptions.ReadConcurrency, "read-concurrency", 0, "read `n` files concurrently (default: $RESTIC_READ_CONCURRENCY or 2)") f.StringVarP(&backupOptions.Host, "host", "H", "", "set the `hostname` for the snapshot manually. To prevent an expensive rescan use the \"parent\" flag") From 333fe1c3cfec73f313affede4bc81c67fdd99791 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Ho=C3=9F?= Date: Thu, 16 Mar 2023 07:45:07 +0100 Subject: [PATCH 02/17] Align Stdin and StdinCommand in conditionals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In order to run with --stdin-from-command we need to short-circuit some functions similar to how it is handled for the --stdin flag. The only difference here is that --stdin-from-command actually expects that len(args) should be greater 0 whereas --stdin does not expect any args at all. Signed-off-by: Sebastian Hoß --- cmd/restic/cmd_backup.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index d07b89a5d..56e3c7939 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -289,7 +289,7 @@ func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error { } } - if opts.Stdin { + if opts.Stdin || opts.StdinCommand { if len(opts.FilesFrom) > 0 { return errors.Fatal("--stdin and --files-from cannot be used together") } @@ -300,7 +300,7 @@ func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error { return errors.Fatal("--stdin and --files-from-raw cannot be used together") } - if len(args) > 0 { + if len(args) > 0 && opts.StdinCommand == false { return errors.Fatal("--stdin was specified and files/dirs were listed as arguments") } } @@ -368,7 +368,7 @@ func collectRejectFuncs(opts BackupOptions, targets []string) (fs []RejectFunc, // collectTargets returns a list of target files/dirs from several sources. func collectTargets(opts BackupOptions, args []string) (targets []string, err error) { - if opts.Stdin { + if opts.Stdin || opts.StdinCommand { return nil, nil } From a2b76ff34f93867281fa2c9df327c0ae2af3b774 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Ho=C3=9F?= Date: Thu, 16 Mar 2023 08:11:49 +0100 Subject: [PATCH 03/17] Start command from --stdin-from-command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It acts similar to --stdin but reads its data from the stdout of the given command instead of os.Stdin. Signed-off-by: Sebastian Hoß --- cmd/restic/cmd_backup.go | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index 56e3c7939..3f494a8cf 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "os" + "os/exec" "path" "path/filepath" "runtime" @@ -594,16 +595,37 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter defer localVss.DeleteSnapshots() targetFS = localVss } - if opts.Stdin { + + var command *exec.Cmd + var stderr io.ReadCloser + if opts.Stdin || opts.StdinCommand { if !gopts.JSON { progressPrinter.V("read data from stdin") } filename := path.Join("/", opts.StdinFilename) + var closer io.ReadCloser + if opts.StdinCommand { + command = exec.CommandContext(ctx, args[0], args[1:]...) + stdout, err := command.StdoutPipe() + if err != nil { + return err + } + stderr, err = command.StderrPipe() + if err != nil { + return err + } + if err := command.Start(); err != nil { + return err + } + closer = stdout + } else { + closer = os.Stdin + } targetFS = &fs.Reader{ ModTime: timeStamp, Name: filename, Mode: 0644, - ReadCloser: os.Stdin, + ReadCloser: closer, } targets = []string{filename} } From 25350a9c55fab726106bb8fbc4adbabe399d6ee7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Ho=C3=9F?= Date: Thu, 16 Mar 2023 07:47:31 +0100 Subject: [PATCH 04/17] Extend SnapshotOptions w/ command data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In order to determine whether to save a snapshot, we need to capture the exit code returned by a command. In order to provide a nice error message, we supply stderr as well. Signed-off-by: Sebastian Hoß --- cmd/restic/cmd_backup.go | 2 ++ internal/archiver/archiver.go | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index 3f494a8cf..798d5609c 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -676,6 +676,8 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter Hostname: opts.Host, ParentSnapshot: parentSnapshot, ProgramVersion: "restic " + version, + Command: command, + CommandStderr: stderr, } if !gopts.JSON { diff --git a/internal/archiver/archiver.go b/internal/archiver/archiver.go index 98819d797..ed1eb62bd 100644 --- a/internal/archiver/archiver.go +++ b/internal/archiver/archiver.go @@ -2,7 +2,9 @@ package archiver import ( "context" + "io" "os" + "os/exec" "path" "runtime" "sort" @@ -681,6 +683,8 @@ type SnapshotOptions struct { Time time.Time ParentSnapshot *restic.Snapshot ProgramVersion string + Command *exec.Cmd + CommandStderr io.ReadCloser } // loadParentTree loads a tree referenced by snapshot id. If id is null, nil is returned. From c133065a9fa418a2fcb8cb21149f8bd6308ce508 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Ho=C3=9F?= Date: Thu, 16 Mar 2023 08:13:15 +0100 Subject: [PATCH 05/17] Check command result before snapshotting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Return with an error containing the stderr of the given command in case it fails. No new snapshot will be created and future prune operations on the repository will remove the unreferenced data. Signed-off-by: Sebastian Hoß --- internal/archiver/archiver.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/internal/archiver/archiver.go b/internal/archiver/archiver.go index ed1eb62bd..91f991e7d 100644 --- a/internal/archiver/archiver.go +++ b/internal/archiver/archiver.go @@ -796,6 +796,15 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps return nil, restic.ID{}, err } + if opts.Command != nil { + errBytes, _ := io.ReadAll(opts.CommandStderr) + cmdErr := opts.Command.Wait() + if cmdErr != nil { + debug.Log("error while executing command: %v", cmdErr) + return nil, restic.ID{}, errors.New(string(errBytes)) + } + } + sn, err := restic.NewSnapshot(targets, opts.Tags, opts.Hostname, opts.Time) if err != nil { return nil, restic.ID{}, err From 4e5caab11415cf332a42beae4d5c7e84fb46c4de Mon Sep 17 00:00:00 2001 From: Enrico204 Date: Sun, 27 Aug 2023 09:57:02 +0200 Subject: [PATCH 06/17] stdin-from-command: implemented suggestions in #4254 The code has been refactored so that the archiver is back to the original code, and the stderr is handled using a go routine to avoid deadlock. --- cmd/restic/cmd_backup.go | 50 ++++++++++++++++++++------------ internal/archiver/archiver.go | 13 --------- internal/fs/fs_reader_command.go | 23 +++++++++++++++ 3 files changed, 55 insertions(+), 31 deletions(-) create mode 100644 internal/fs/fs_reader_command.go diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index 798d5609c..af736d13b 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -301,7 +301,7 @@ func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error { return errors.Fatal("--stdin and --files-from-raw cannot be used together") } - if len(args) > 0 && opts.StdinCommand == false { + if len(args) > 0 && !opts.StdinCommand { return errors.Fatal("--stdin was specified and files/dirs were listed as arguments") } } @@ -596,30 +596,17 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter targetFS = localVss } - var command *exec.Cmd - var stderr io.ReadCloser if opts.Stdin || opts.StdinCommand { if !gopts.JSON { progressPrinter.V("read data from stdin") } filename := path.Join("/", opts.StdinFilename) - var closer io.ReadCloser + var closer io.ReadCloser = os.Stdin if opts.StdinCommand { - command = exec.CommandContext(ctx, args[0], args[1:]...) - stdout, err := command.StdoutPipe() + closer, err = prepareStdinCommand(ctx, args) if err != nil { return err } - stderr, err = command.StderrPipe() - if err != nil { - return err - } - if err := command.Start(); err != nil { - return err - } - closer = stdout - } else { - closer = os.Stdin } targetFS = &fs.Reader{ ModTime: timeStamp, @@ -676,8 +663,6 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter Hostname: opts.Host, ParentSnapshot: parentSnapshot, ProgramVersion: "restic " + version, - Command: command, - CommandStderr: stderr, } if !gopts.JSON { @@ -708,3 +693,32 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter // Return error if any return werr } + +func prepareStdinCommand(ctx context.Context, args []string) (io.ReadCloser, error) { + // Prepare command and stdout. These variables will be assigned to the + // io.ReadCloser that is used by the archiver to read data, so that the + // Close() function waits for the program to finish. See + // fs.ReadCloserCommand. + command := exec.CommandContext(ctx, args[0], args[1:]...) + stdout, err := command.StdoutPipe() + if err != nil { + return nil, errors.Wrap(err, "command.StdoutPipe") + } + + // Use a Go routine to handle the stderr to avoid deadlocks + stderr, err := command.StderrPipe() + if err != nil { + return nil, errors.Wrap(err, "command.StderrPipe") + } + go func() { + sc := bufio.NewScanner(stderr) + for sc.Scan() { + _, _ = fmt.Fprintf(os.Stderr, "subprocess %v: %v\n", command.Args[0], sc.Text()) + } + }() + + if err := command.Start(); err != nil { + return nil, errors.Wrap(err, "command.Start") + } + return &fs.ReadCloserCommand{Cmd: command, Stdout: stdout}, nil +} diff --git a/internal/archiver/archiver.go b/internal/archiver/archiver.go index 91f991e7d..98819d797 100644 --- a/internal/archiver/archiver.go +++ b/internal/archiver/archiver.go @@ -2,9 +2,7 @@ package archiver import ( "context" - "io" "os" - "os/exec" "path" "runtime" "sort" @@ -683,8 +681,6 @@ type SnapshotOptions struct { Time time.Time ParentSnapshot *restic.Snapshot ProgramVersion string - Command *exec.Cmd - CommandStderr io.ReadCloser } // loadParentTree loads a tree referenced by snapshot id. If id is null, nil is returned. @@ -796,15 +792,6 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps return nil, restic.ID{}, err } - if opts.Command != nil { - errBytes, _ := io.ReadAll(opts.CommandStderr) - cmdErr := opts.Command.Wait() - if cmdErr != nil { - debug.Log("error while executing command: %v", cmdErr) - return nil, restic.ID{}, errors.New(string(errBytes)) - } - } - sn, err := restic.NewSnapshot(targets, opts.Tags, opts.Hostname, opts.Time) if err != nil { return nil, restic.ID{}, err diff --git a/internal/fs/fs_reader_command.go b/internal/fs/fs_reader_command.go new file mode 100644 index 000000000..f35454c7f --- /dev/null +++ b/internal/fs/fs_reader_command.go @@ -0,0 +1,23 @@ +package fs + +import ( + "io" + "os/exec" +) + +// ReadCloserCommand wraps an exec.Cmd and its standard output to provide an +// io.ReadCloser that waits for the command to terminate on Close(), reporting +// any error in the command.Wait() function back to the Close() caller. +type ReadCloserCommand struct { + Cmd *exec.Cmd + Stdout io.ReadCloser +} + +func (fp *ReadCloserCommand) Read(p []byte) (n int, err error) { + return fp.Stdout.Read(p) +} + +func (fp *ReadCloserCommand) Close() error { + // No need to close fp.Stdout as Wait() closes all pipes. + return fp.Cmd.Wait() +} From 072b227544be19e13a81139e2e86a2f82f24a42d Mon Sep 17 00:00:00 2001 From: Enrico204 Date: Sun, 27 Aug 2023 10:33:46 +0200 Subject: [PATCH 07/17] stdin-from-command: add documentation in backup sub-command --- doc/040_backup.rst | 67 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 51 insertions(+), 16 deletions(-) diff --git a/doc/040_backup.rst b/doc/040_backup.rst index c917c3c29..f1372842e 100644 --- a/doc/040_backup.rst +++ b/doc/040_backup.rst @@ -489,35 +489,70 @@ particular note are:: - file ownership and ACLs on Windows - the "hidden" flag on Windows + +Reading data from a command standard output +*********************** + +Sometimes, it can be nice to directly save the output of a program, e.g., +``mysqldump`` so that the SQL can later be restored. Restic supports this mode +of operation; just supply the option ``--stdin-from-command`` when using the +``backup`` action, and write the command in place of the files/directories: + +.. code-block:: console + + $ restic -r /srv/restic-repo backup --stdin-from-command mysqldump [...] + +This command creates a new snapshot of the standard output of ``mysqldump``. +You can then use, e.g., the fuse mounting option (see below) to mount the +repository and read the file. + +By default, the command's standard output is saved in a file named ``stdin``. +A different name can be specified with ``--stdin-filename``, e.g.: + +.. code-block:: console + + $ restic -r /srv/restic-repo backup --stdin-filename production.sql --stdin-from-command mysqldump [...] + +Restic uses the command exit code to determine whether the backup succeeded. A +non-zero exit code from the command makes Restic cancel the backup. + + Reading data from stdin *********************** -Sometimes it can be nice to directly save the output of a program, e.g. -``mysqldump`` so that the SQL can later be restored. Restic supports -this mode of operation, just supply the option ``--stdin`` to the +If the ``--stdin-from-command`` option is insufficient, Restic supports reading +arbitrary data from the standard input. Use the option ``--stdin`` to the ``backup`` command like this: .. code-block:: console - $ set -o pipefail - $ mysqldump [...] | restic -r /srv/restic-repo backup --stdin + $ restic -r /srv/restic-repo backup --stdin < bigfile.dat -This creates a new snapshot of the output of ``mysqldump``. You can then -use e.g. the fuse mounting option (see below) to mount the repository -and read the file. +This creates a new snapshot of the content of ``bigfile.dat`` (note that, in +this example, you can trivially use the standard ``backup`` command by +specifying the file path). -By default, the file name ``stdin`` is used, a different name can be -specified with ``--stdin-filename``, e.g. like this: +As for ``--stdin-from-command``, the default file name is ``stdin``; a +different name can be specified with ``--stdin-filename``. + +**Important**: while it is possible to pipe a command output in Restic using +``--stdin``, doing so is highly discouraged as it will mask errors from the +command, leading to corrupted backups. For example, in the following code +block, if ``mysqldump`` has an error connecting to the MySQL database, Restic +backup will succeed in creating an empty backup: .. code-block:: console - $ mysqldump [...] | restic -r /srv/restic-repo backup --stdin --stdin-filename production.sql + $ # Don't do this, read the warning above + $ mysqldump [...] | restic -r /srv/restic-repo backup --stdin -The option ``pipefail`` is highly recommended so that a non-zero exit code from -one of the programs in the pipe (e.g. ``mysqldump`` here) makes the whole chain -return a non-zero exit code. Refer to the `Use the Unofficial Bash Strict Mode -`__ for more -details on this. +A simple solution is to use ``--stdin-from-command`` (see above). Shall you +still need to use the ``--stdin`` flag, you must use the option ``pipefail`` +(so that a non-zero exit code from one of the programs in the pipe makes the +whole chain return a non-zero exit code) and you must check the exit code of +the pipe and act accordingly (e.g., remove the last backup). Refer to the +`Use the Unofficial Bash Strict Mode `__ +for more details on this. Tags for backup From 6990b0122ec4e67edfabb72cf730e42c3b90f46f Mon Sep 17 00:00:00 2001 From: Enrico204 Date: Sun, 27 Aug 2023 10:40:57 +0200 Subject: [PATCH 08/17] Add issue-4251 (stdin-from-command) in the changelog --- changelog/unreleased/issue-4251 | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 changelog/unreleased/issue-4251 diff --git a/changelog/unreleased/issue-4251 b/changelog/unreleased/issue-4251 new file mode 100644 index 000000000..b2c39c290 --- /dev/null +++ b/changelog/unreleased/issue-4251 @@ -0,0 +1,10 @@ +Enhancement: Add flag to source the backup from a program's standard output + +The `backup` command now supports sourcing the backup content from the standard +output of an arbitrary command, ensuring that the exit code is zero for a +successful backup. + +Example: `restic backup --stdin-from-command mysqldump [...]` + +https://github.com/restic/restic/issues/4251 +https://github.com/restic/restic/pull/4410 From 81f8d473df96995795cb7c160b08f44c7f940b00 Mon Sep 17 00:00:00 2001 From: Enrico204 Date: Mon, 28 Aug 2023 07:53:17 +0200 Subject: [PATCH 09/17] restic-from-command: abort snapshot on non-zero exit codes --- cmd/restic/cmd_backup.go | 10 ++++++++- internal/fs/fs_reader_command.go | 37 +++++++++++++++++++++++++++++--- 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index af736d13b..69a68ccab 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -634,6 +634,8 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter wg.Go(func() error { return sc.Scan(cancelCtx, targets) }) } + snapshotCtx, cancelSnapshot := context.WithCancel(ctx) + arch := archiver.New(repo, targetFS, archiver.Options{ReadConcurrency: backupOptions.ReadConcurrency}) arch.SelectByName = selectByNameFilter arch.Select = selectFilter @@ -641,6 +643,11 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter success := true arch.Error = func(item string, err error) error { success = false + // If we receive a fatal error during the execution of the snapshot, + // we abort the snapshot. + if errors.IsFatal(err) { + cancelSnapshot() + } return progressReporter.Error(item, err) } arch.CompleteItem = progressReporter.CompleteItem @@ -668,7 +675,8 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter if !gopts.JSON { progressPrinter.V("start backup on %v", targets) } - _, id, err := arch.Snapshot(ctx, targets, snapshotOpts) + _, id, err := arch.Snapshot(snapshotCtx, targets, snapshotOpts) + cancelSnapshot() // cleanly shutdown all running goroutines cancel() diff --git a/internal/fs/fs_reader_command.go b/internal/fs/fs_reader_command.go index f35454c7f..e456ee965 100644 --- a/internal/fs/fs_reader_command.go +++ b/internal/fs/fs_reader_command.go @@ -1,6 +1,7 @@ package fs import ( + "github.com/restic/restic/internal/errors" "io" "os/exec" ) @@ -11,13 +12,43 @@ import ( type ReadCloserCommand struct { Cmd *exec.Cmd Stdout io.ReadCloser + + bytesRead bool } -func (fp *ReadCloserCommand) Read(p []byte) (n int, err error) { - return fp.Stdout.Read(p) +// Read populate the array with data from the process stdout. +func (fp *ReadCloserCommand) Read(p []byte) (int, error) { + // We may encounter two different error conditions here: + // - EOF with no bytes read: the program terminated prematurely, so we send + // a fatal error to cancel the snapshot; + // - an error that is not EOF: something bad happened, we need to abort the + // snapshot. + b, err := fp.Stdout.Read(p) + if b == 0 && errors.Is(err, io.EOF) && !fp.bytesRead { + // The command terminated with no output at all. Raise a fatal error. + return 0, errors.Fatalf("command terminated with no output") + } else if err != nil && !errors.Is(err, io.EOF) { + // The command terminated with an error that is not EOF. Raise a fatal + // error. + return 0, errors.Fatal(err.Error()) + } else if b > 0 { + fp.bytesRead = true + } + return b, err } func (fp *ReadCloserCommand) Close() error { // No need to close fp.Stdout as Wait() closes all pipes. - return fp.Cmd.Wait() + err := fp.Cmd.Wait() + if err != nil { + // If we have information about the exit code, let's use it in the + // error message. Otherwise, send the error message along. + // In any case, use a fatal error to abort the snapshot. + var err2 *exec.ExitError + if errors.As(err, &err2) { + return errors.Fatalf("command terminated with exit code %d", err2.ExitCode()) + } + return errors.Fatal(err.Error()) + } + return nil } From c0ca54dc8afe28969c1464c95aef40eb5ae55d3d Mon Sep 17 00:00:00 2001 From: Enrico204 Date: Mon, 28 Aug 2023 07:53:23 +0200 Subject: [PATCH 10/17] restic-from-command: add tests --- cmd/restic/cmd_backup_integration_test.go | 52 +++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/cmd/restic/cmd_backup_integration_test.go b/cmd/restic/cmd_backup_integration_test.go index 742b6ff6c..550834972 100644 --- a/cmd/restic/cmd_backup_integration_test.go +++ b/cmd/restic/cmd_backup_integration_test.go @@ -568,3 +568,55 @@ func linkEqual(source, dest []string) bool { return true } + +func TestStdinFromCommand(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testSetupBackupData(t, env) + opts := BackupOptions{ + StdinCommand: true, + StdinFilename: "stdin", + } + + testRunBackup(t, filepath.Dir(env.testdata), []string{"ls"}, opts, env.gopts) + testListSnapshots(t, env.gopts, 1) + + testRunCheck(t, env.gopts) +} + +func TestStdinFromCommandFailNoOutput(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testSetupBackupData(t, env) + opts := BackupOptions{ + StdinCommand: true, + StdinFilename: "stdin", + } + + err := testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"python", "-c", "import sys; sys.exit(1)"}, opts, env.gopts) + rtest.Assert(t, err != nil, "Expected error while backing up") + + testListSnapshots(t, env.gopts, 0) + + testRunCheck(t, env.gopts) +} + +func TestStdinFromCommandFailExitCode(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testSetupBackupData(t, env) + opts := BackupOptions{ + StdinCommand: true, + StdinFilename: "stdin", + } + + err := testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"python", "-c", "import sys; print('test'); sys.exit(1)"}, opts, env.gopts) + rtest.Assert(t, err != nil, "Expected error while backing up") + + testListSnapshots(t, env.gopts, 0) + + testRunCheck(t, env.gopts) +} From 37a312e50532a84d33087f97695f925984642614 Mon Sep 17 00:00:00 2001 From: Enrico204 Date: Tue, 29 Aug 2023 09:19:17 +0200 Subject: [PATCH 11/17] restic-from-command: use standard behavior when no output and exit code 0 from command The behavior of the new option should reflect the behavior of normal backups: when the command exit code is zero and there is no output in the stdout, emit a warning but create the snapshot. This commit fixes the integration tests and the ReadCloserCommand struct. --- cmd/restic/cmd_backup_integration_test.go | 29 +++++++++--- internal/fs/fs_reader_command.go | 55 ++++++++++++++++------- 2 files changed, 62 insertions(+), 22 deletions(-) diff --git a/cmd/restic/cmd_backup_integration_test.go b/cmd/restic/cmd_backup_integration_test.go index 550834972..c60e9c543 100644 --- a/cmd/restic/cmd_backup_integration_test.go +++ b/cmd/restic/cmd_backup_integration_test.go @@ -579,13 +579,13 @@ func TestStdinFromCommand(t *testing.T) { StdinFilename: "stdin", } - testRunBackup(t, filepath.Dir(env.testdata), []string{"ls"}, opts, env.gopts) + testRunBackup(t, filepath.Dir(env.testdata), []string{"python", "-c", "import sys; print('something'); sys.exit(0)"}, opts, env.gopts) testListSnapshots(t, env.gopts, 1) testRunCheck(t, env.gopts) } -func TestStdinFromCommandFailNoOutput(t *testing.T) { +func TestStdinFromCommandNoOutput(t *testing.T) { env, cleanup := withTestEnvironment(t) defer cleanup() @@ -595,10 +595,9 @@ func TestStdinFromCommandFailNoOutput(t *testing.T) { StdinFilename: "stdin", } - err := testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"python", "-c", "import sys; sys.exit(1)"}, opts, env.gopts) - rtest.Assert(t, err != nil, "Expected error while backing up") - - testListSnapshots(t, env.gopts, 0) + err := testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"python", "-c", "import sys; sys.exit(0)"}, opts, env.gopts) + rtest.Assert(t, err != nil && err.Error() == "at least one source file could not be read", "No data error expected") + testListSnapshots(t, env.gopts, 1) testRunCheck(t, env.gopts) } @@ -620,3 +619,21 @@ func TestStdinFromCommandFailExitCode(t *testing.T) { testRunCheck(t, env.gopts) } + +func TestStdinFromCommandFailNoOutputAndExitCode(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testSetupBackupData(t, env) + opts := BackupOptions{ + StdinCommand: true, + StdinFilename: "stdin", + } + + err := testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"python", "-c", "import sys; sys.exit(1)"}, opts, env.gopts) + rtest.Assert(t, err != nil, "Expected error while backing up") + + testListSnapshots(t, env.gopts, 0) + + testRunCheck(t, env.gopts) +} diff --git a/internal/fs/fs_reader_command.go b/internal/fs/fs_reader_command.go index e456ee965..bc377ab96 100644 --- a/internal/fs/fs_reader_command.go +++ b/internal/fs/fs_reader_command.go @@ -13,31 +13,54 @@ type ReadCloserCommand struct { Cmd *exec.Cmd Stdout io.ReadCloser - bytesRead bool + // We should call exec.Wait() once. waitHandled is taking care of storing + // whether we already called that function in Read() to avoid calling it + // again in Close(). + waitHandled bool + + // alreadyClosedReadErr is the error that we should return if we try to + // read the pipe again after closing. This works around a Read() call that + // is issued after a previous Read() with `io.EOF` (but some bytes were + // read in the past). + alreadyClosedReadErr error } // Read populate the array with data from the process stdout. func (fp *ReadCloserCommand) Read(p []byte) (int, error) { - // We may encounter two different error conditions here: - // - EOF with no bytes read: the program terminated prematurely, so we send - // a fatal error to cancel the snapshot; - // - an error that is not EOF: something bad happened, we need to abort the - // snapshot. - b, err := fp.Stdout.Read(p) - if b == 0 && errors.Is(err, io.EOF) && !fp.bytesRead { - // The command terminated with no output at all. Raise a fatal error. - return 0, errors.Fatalf("command terminated with no output") - } else if err != nil && !errors.Is(err, io.EOF) { - // The command terminated with an error that is not EOF. Raise a fatal - // error. - return 0, errors.Fatal(err.Error()) - } else if b > 0 { - fp.bytesRead = true + if fp.alreadyClosedReadErr != nil { + return 0, fp.alreadyClosedReadErr } + b, err := fp.Stdout.Read(p) + + // If the error is io.EOF, the program terminated. We need to check the + // exit code here because, if the program terminated with no output, the + // error in `Close()` is ignored. + if errors.Is(err, io.EOF) { + // Check if the command terminated successfully. If not, return the + // error. + fp.waitHandled = true + errw := fp.Cmd.Wait() + if errw != nil { + // If we have information about the exit code, let's use it in the + // error message. Otherwise, send the error message along. + // In any case, use a fatal error to abort the snapshot. + var err2 *exec.ExitError + if errors.As(errw, &err2) { + err = errors.Fatalf("command terminated with exit code %d", err2.ExitCode()) + } else { + err = errors.Fatal(errw.Error()) + } + } + } + fp.alreadyClosedReadErr = err return b, err } func (fp *ReadCloserCommand) Close() error { + if fp.waitHandled { + return nil + } + // No need to close fp.Stdout as Wait() closes all pipes. err := fp.Cmd.Wait() if err != nil { From 7d879705ada5c80b665c41edbdc82d59a19d0307 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 1 Oct 2023 15:07:51 +0200 Subject: [PATCH 12/17] fs: cleanup CommandReader implementation --- cmd/restic/cmd_backup.go | 2 +- internal/fs/fs_reader_command.go | 55 ++++++++++++++++---------------- 2 files changed, 29 insertions(+), 28 deletions(-) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index 69a68ccab..8207e0bb4 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -728,5 +728,5 @@ func prepareStdinCommand(ctx context.Context, args []string) (io.ReadCloser, err if err := command.Start(); err != nil { return nil, errors.Wrap(err, "command.Start") } - return &fs.ReadCloserCommand{Cmd: command, Stdout: stdout}, nil + return fs.NewCommandReader(command, stdout), nil } diff --git a/internal/fs/fs_reader_command.go b/internal/fs/fs_reader_command.go index bc377ab96..b257deb72 100644 --- a/internal/fs/fs_reader_command.go +++ b/internal/fs/fs_reader_command.go @@ -1,17 +1,18 @@ package fs import ( - "github.com/restic/restic/internal/errors" "io" "os/exec" + + "github.com/restic/restic/internal/errors" ) -// ReadCloserCommand wraps an exec.Cmd and its standard output to provide an +// CommandReader wraps an exec.Cmd and its standard output to provide an // io.ReadCloser that waits for the command to terminate on Close(), reporting // any error in the command.Wait() function back to the Close() caller. -type ReadCloserCommand struct { - Cmd *exec.Cmd - Stdout io.ReadCloser +type CommandReader struct { + cmd *exec.Cmd + stdout io.ReadCloser // We should call exec.Wait() once. waitHandled is taking care of storing // whether we already called that function in Read() to avoid calling it @@ -25,44 +26,36 @@ type ReadCloserCommand struct { alreadyClosedReadErr error } +func NewCommandReader(cmd *exec.Cmd, stdout io.ReadCloser) *CommandReader { + return &CommandReader{ + cmd: cmd, + stdout: stdout, + } +} + // Read populate the array with data from the process stdout. -func (fp *ReadCloserCommand) Read(p []byte) (int, error) { +func (fp *CommandReader) Read(p []byte) (int, error) { if fp.alreadyClosedReadErr != nil { return 0, fp.alreadyClosedReadErr } - b, err := fp.Stdout.Read(p) + b, err := fp.stdout.Read(p) // If the error is io.EOF, the program terminated. We need to check the // exit code here because, if the program terminated with no output, the // error in `Close()` is ignored. if errors.Is(err, io.EOF) { - // Check if the command terminated successfully. If not, return the - // error. fp.waitHandled = true - errw := fp.Cmd.Wait() - if errw != nil { - // If we have information about the exit code, let's use it in the - // error message. Otherwise, send the error message along. - // In any case, use a fatal error to abort the snapshot. - var err2 *exec.ExitError - if errors.As(errw, &err2) { - err = errors.Fatalf("command terminated with exit code %d", err2.ExitCode()) - } else { - err = errors.Fatal(errw.Error()) - } + // check if the command terminated successfully, If not return the error. + if errw := fp.wait(); errw != nil { + err = errw } } fp.alreadyClosedReadErr = err return b, err } -func (fp *ReadCloserCommand) Close() error { - if fp.waitHandled { - return nil - } - - // No need to close fp.Stdout as Wait() closes all pipes. - err := fp.Cmd.Wait() +func (fp *CommandReader) wait() error { + err := fp.cmd.Wait() if err != nil { // If we have information about the exit code, let's use it in the // error message. Otherwise, send the error message along. @@ -75,3 +68,11 @@ func (fp *ReadCloserCommand) Close() error { } return nil } + +func (fp *CommandReader) Close() error { + if fp.waitHandled { + return nil + } + + return fp.wait() +} From 317144c1d64cf61f927ceee9a008f09ec1d88fde Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 1 Oct 2023 15:25:48 +0200 Subject: [PATCH 13/17] fs: merge command startup into CommandReader --- cmd/restic/cmd_backup.go | 36 ++-------------------- internal/fs/fs_reader_command.go | 53 ++++++++++++++++++++++---------- 2 files changed, 39 insertions(+), 50 deletions(-) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index 8207e0bb4..3e16bd801 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -7,7 +7,6 @@ import ( "fmt" "io" "os" - "os/exec" "path" "path/filepath" "runtime" @@ -601,9 +600,9 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter progressPrinter.V("read data from stdin") } filename := path.Join("/", opts.StdinFilename) - var closer io.ReadCloser = os.Stdin + var source io.ReadCloser = os.Stdin if opts.StdinCommand { - closer, err = prepareStdinCommand(ctx, args) + source, err = fs.NewCommandReader(ctx, args, globalOptions.stderr) if err != nil { return err } @@ -612,7 +611,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter ModTime: timeStamp, Name: filename, Mode: 0644, - ReadCloser: closer, + ReadCloser: source, } targets = []string{filename} } @@ -701,32 +700,3 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter // Return error if any return werr } - -func prepareStdinCommand(ctx context.Context, args []string) (io.ReadCloser, error) { - // Prepare command and stdout. These variables will be assigned to the - // io.ReadCloser that is used by the archiver to read data, so that the - // Close() function waits for the program to finish. See - // fs.ReadCloserCommand. - command := exec.CommandContext(ctx, args[0], args[1:]...) - stdout, err := command.StdoutPipe() - if err != nil { - return nil, errors.Wrap(err, "command.StdoutPipe") - } - - // Use a Go routine to handle the stderr to avoid deadlocks - stderr, err := command.StderrPipe() - if err != nil { - return nil, errors.Wrap(err, "command.StderrPipe") - } - go func() { - sc := bufio.NewScanner(stderr) - for sc.Scan() { - _, _ = fmt.Fprintf(os.Stderr, "subprocess %v: %v\n", command.Args[0], sc.Text()) - } - }() - - if err := command.Start(); err != nil { - return nil, errors.Wrap(err, "command.Start") - } - return fs.NewCommandReader(command, stdout), nil -} diff --git a/internal/fs/fs_reader_command.go b/internal/fs/fs_reader_command.go index b257deb72..20d65a1ca 100644 --- a/internal/fs/fs_reader_command.go +++ b/internal/fs/fs_reader_command.go @@ -1,22 +1,24 @@ package fs import ( + "bufio" + "context" + "fmt" "io" "os/exec" "github.com/restic/restic/internal/errors" ) -// CommandReader wraps an exec.Cmd and its standard output to provide an -// io.ReadCloser that waits for the command to terminate on Close(), reporting -// any error in the command.Wait() function back to the Close() caller. +// CommandReader wrap a command such that its standard output can be read using +// a io.ReadCloser. Close() waits for the command to terminate, reporting +// any error back to the caller. type CommandReader struct { cmd *exec.Cmd stdout io.ReadCloser - // We should call exec.Wait() once. waitHandled is taking care of storing - // whether we already called that function in Read() to avoid calling it - // again in Close(). + // cmd.Wait() must only be called once. Prevent duplicate executions in + // Read() and Close(). waitHandled bool // alreadyClosedReadErr is the error that we should return if we try to @@ -26,11 +28,34 @@ type CommandReader struct { alreadyClosedReadErr error } -func NewCommandReader(cmd *exec.Cmd, stdout io.ReadCloser) *CommandReader { - return &CommandReader{ - cmd: cmd, - stdout: stdout, +func NewCommandReader(ctx context.Context, args []string, logOutput io.Writer) (*CommandReader, error) { + // Prepare command and stdout + command := exec.CommandContext(ctx, args[0], args[1:]...) + stdout, err := command.StdoutPipe() + if err != nil { + return nil, fmt.Errorf("failed to setup stdout pipe: %w", err) } + + // Use a Go routine to handle the stderr to avoid deadlocks + stderr, err := command.StderrPipe() + if err != nil { + return nil, fmt.Errorf("failed to setup stderr pipe: %w", err) + } + go func() { + sc := bufio.NewScanner(stderr) + for sc.Scan() { + _, _ = fmt.Fprintf(logOutput, "subprocess %v: %v\n", command.Args[0], sc.Text()) + } + }() + + if err := command.Start(); err != nil { + return nil, fmt.Errorf("failed to start command: %w", err) + } + + return &CommandReader{ + cmd: command, + stdout: stdout, + }, nil } // Read populate the array with data from the process stdout. @@ -57,13 +82,7 @@ func (fp *CommandReader) Read(p []byte) (int, error) { func (fp *CommandReader) wait() error { err := fp.cmd.Wait() if err != nil { - // If we have information about the exit code, let's use it in the - // error message. Otherwise, send the error message along. - // In any case, use a fatal error to abort the snapshot. - var err2 *exec.ExitError - if errors.As(err, &err2) { - return errors.Fatalf("command terminated with exit code %d", err2.ExitCode()) - } + // Use a fatal error to abort the snapshot. return errors.Fatal(err.Error()) } return nil From 8bceb8e3592d315436739ccc3facce42c48e6ae5 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 1 Oct 2023 15:45:08 +0200 Subject: [PATCH 14/17] fs: add tests for CommandReader --- internal/fs/fs_reader_command_test.go | 48 +++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 internal/fs/fs_reader_command_test.go diff --git a/internal/fs/fs_reader_command_test.go b/internal/fs/fs_reader_command_test.go new file mode 100644 index 000000000..a9028544c --- /dev/null +++ b/internal/fs/fs_reader_command_test.go @@ -0,0 +1,48 @@ +package fs_test + +import ( + "bytes" + "context" + "io" + "strings" + "testing" + + "github.com/restic/restic/internal/fs" + "github.com/restic/restic/internal/test" +) + +func TestCommandReaderSuccess(t *testing.T) { + reader, err := fs.NewCommandReader(context.TODO(), []string{"true"}, io.Discard) + test.OK(t, err) + + _, err = io.Copy(io.Discard, reader) + test.OK(t, err) + + test.OK(t, reader.Close()) +} + +func TestCommandReaderFail(t *testing.T) { + reader, err := fs.NewCommandReader(context.TODO(), []string{"false"}, io.Discard) + test.OK(t, err) + + _, err = io.Copy(io.Discard, reader) + test.Assert(t, err != nil, "missing error") +} + +func TestCommandReaderInvalid(t *testing.T) { + _, err := fs.NewCommandReader(context.TODO(), []string{"w54fy098hj7fy5twijouytfrj098y645wr"}, io.Discard) + test.Assert(t, err != nil, "missing error") +} + +func TestCommandReaderOutput(t *testing.T) { + reader, err := fs.NewCommandReader(context.TODO(), []string{"echo", "hello world"}, io.Discard) + test.OK(t, err) + + var buf bytes.Buffer + + _, err = io.Copy(&buf, reader) + test.OK(t, err) + test.OK(t, reader.Close()) + + test.Equals(t, "hello world", strings.TrimSpace(buf.String())) +} From ee305e6041d88086cad2f025fe27f8bb2543c1ec Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 1 Oct 2023 16:20:45 +0200 Subject: [PATCH 15/17] backup: rework error reporting for subcommand --- cmd/restic/cmd_backup.go | 12 +++++------- internal/archiver/file_saver.go | 3 ++- internal/fs/fs_reader_command.go | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index 3e16bd801..a2b81a759 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -633,8 +633,6 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter wg.Go(func() error { return sc.Scan(cancelCtx, targets) }) } - snapshotCtx, cancelSnapshot := context.WithCancel(ctx) - arch := archiver.New(repo, targetFS, archiver.Options{ReadConcurrency: backupOptions.ReadConcurrency}) arch.SelectByName = selectByNameFilter arch.Select = selectFilter @@ -642,12 +640,13 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter success := true arch.Error = func(item string, err error) error { success = false + reterr := progressReporter.Error(item, err) // If we receive a fatal error during the execution of the snapshot, // we abort the snapshot. - if errors.IsFatal(err) { - cancelSnapshot() + if reterr == nil && errors.IsFatal(err) { + reterr = err } - return progressReporter.Error(item, err) + return reterr } arch.CompleteItem = progressReporter.CompleteItem arch.StartFile = progressReporter.StartFile @@ -674,8 +673,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter if !gopts.JSON { progressPrinter.V("start backup on %v", targets) } - _, id, err := arch.Snapshot(snapshotCtx, targets, snapshotOpts) - cancelSnapshot() + _, id, err := arch.Snapshot(ctx, targets, snapshotOpts) // cleanly shutdown all running goroutines cancel() diff --git a/internal/archiver/file_saver.go b/internal/archiver/file_saver.go index 0742c8b57..724f5e620 100644 --- a/internal/archiver/file_saver.go +++ b/internal/archiver/file_saver.go @@ -2,6 +2,7 @@ package archiver import ( "context" + "fmt" "io" "os" "sync" @@ -146,7 +147,7 @@ func (s *FileSaver) saveFile(ctx context.Context, chnker *chunker.Chunker, snPat panic("completed twice") } isCompleted = true - fnr.err = err + fnr.err = fmt.Errorf("failed to save %v: %w", target, err) fnr.node = nil fnr.stats = ItemStats{} finish(fnr) diff --git a/internal/fs/fs_reader_command.go b/internal/fs/fs_reader_command.go index 20d65a1ca..3830e5811 100644 --- a/internal/fs/fs_reader_command.go +++ b/internal/fs/fs_reader_command.go @@ -83,7 +83,7 @@ func (fp *CommandReader) wait() error { err := fp.cmd.Wait() if err != nil { // Use a fatal error to abort the snapshot. - return errors.Fatal(err.Error()) + return errors.Fatal(fmt.Errorf("command failed: %w", err).Error()) } return nil } From 5d152c77203cc9a5dcbefcc5ff8e8b2944e331d6 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 1 Oct 2023 16:34:30 +0200 Subject: [PATCH 16/17] extend changelog for --stdin-from-command --- changelog/unreleased/issue-4251 | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/changelog/unreleased/issue-4251 b/changelog/unreleased/issue-4251 index b2c39c290..31be52401 100644 --- a/changelog/unreleased/issue-4251 +++ b/changelog/unreleased/issue-4251 @@ -1,8 +1,12 @@ -Enhancement: Add flag to source the backup from a program's standard output +Enhancement: Support reading backup from a program's standard output -The `backup` command now supports sourcing the backup content from the standard -output of an arbitrary command, ensuring that the exit code is zero for a -successful backup. +When reading data from stdin, the `backup` command could not verify whether the +corresponding command completed successfully. + +The `backup` command now supports starting an arbitrary command and sourcing +the backup content from its standard output. This enables restic to verify that +the command completes with exit code zero. A non-zero exit code causes the +backup to fail. Example: `restic backup --stdin-from-command mysqldump [...]` From be28a026262d30305023da769b4ef19aaae2d9d2 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 1 Oct 2023 16:54:38 +0200 Subject: [PATCH 17/17] doc: tweak description for --stdin-from-command --- doc/040_backup.rst | 51 +++++++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/doc/040_backup.rst b/doc/040_backup.rst index f1372842e..acafe2694 100644 --- a/doc/040_backup.rst +++ b/doc/040_backup.rst @@ -490,10 +490,10 @@ particular note are:: - the "hidden" flag on Windows -Reading data from a command standard output -*********************** +Reading data from a command +*************************** -Sometimes, it can be nice to directly save the output of a program, e.g., +Sometimes, it can be useful to directly save the output of a program, for example, ``mysqldump`` so that the SQL can later be restored. Restic supports this mode of operation; just supply the option ``--stdin-from-command`` when using the ``backup`` action, and write the command in place of the files/directories: @@ -502,52 +502,53 @@ of operation; just supply the option ``--stdin-from-command`` when using the $ restic -r /srv/restic-repo backup --stdin-from-command mysqldump [...] -This command creates a new snapshot of the standard output of ``mysqldump``. -You can then use, e.g., the fuse mounting option (see below) to mount the -repository and read the file. - +This command creates a new snapshot based on the standard output of ``mysqldump``. By default, the command's standard output is saved in a file named ``stdin``. -A different name can be specified with ``--stdin-filename``, e.g.: +A different name can be specified with ``--stdin-filename``: .. code-block:: console $ restic -r /srv/restic-repo backup --stdin-filename production.sql --stdin-from-command mysqldump [...] -Restic uses the command exit code to determine whether the backup succeeded. A -non-zero exit code from the command makes Restic cancel the backup. +Restic uses the command exit code to determine whether the command succeeded. A +non-zero exit code from the command causes restic to cancel the backup. This causes +restic to fail with exit code 1. No snapshot will be created in this case. Reading data from stdin *********************** -If the ``--stdin-from-command`` option is insufficient, Restic supports reading -arbitrary data from the standard input. Use the option ``--stdin`` to the -``backup`` command like this: +.. warning:: + + Restic cannot detect if data read from stdin is complete or not. As explained + below, this can cause incomplete backup unless additional checks (outside of + restic) are configured. If possible, use ``--stdin-from-command`` instead. + +Alternatively, restic supports reading arbitrary data directly from the standard +input. Use the option ``--stdin`` of the ``backup`` command as follows: .. code-block:: console - $ restic -r /srv/restic-repo backup --stdin < bigfile.dat - -This creates a new snapshot of the content of ``bigfile.dat`` (note that, in -this example, you can trivially use the standard ``backup`` command by -specifying the file path). + # Will not notice failures, see the warning below + $ gzip bigfile.dat | restic -r /srv/restic-repo backup --stdin +This creates a new snapshot of the content of ``bigfile.dat``. As for ``--stdin-from-command``, the default file name is ``stdin``; a different name can be specified with ``--stdin-filename``. -**Important**: while it is possible to pipe a command output in Restic using -``--stdin``, doing so is highly discouraged as it will mask errors from the +**Important**: while it is possible to pipe a command output to restic using +``--stdin``, doing so is discouraged as it will mask errors from the command, leading to corrupted backups. For example, in the following code -block, if ``mysqldump`` has an error connecting to the MySQL database, Restic -backup will succeed in creating an empty backup: +block, if ``mysqldump`` fails to connect to the MySQL database, the restic +backup will nevertheless succeed in creating an _empty_ backup: .. code-block:: console - $ # Don't do this, read the warning above + # Will not notice failures, read the warning above $ mysqldump [...] | restic -r /srv/restic-repo backup --stdin -A simple solution is to use ``--stdin-from-command`` (see above). Shall you -still need to use the ``--stdin`` flag, you must use the option ``pipefail`` +A simple solution is to use ``--stdin-from-command`` (see above). If you +still need to use the ``--stdin`` flag, you must use the shell option ``set -o pipefail`` (so that a non-zero exit code from one of the programs in the pipe makes the whole chain return a non-zero exit code) and you must check the exit code of the pipe and act accordingly (e.g., remove the last backup). Refer to the