Merge pull request #4334 from MichaelEischer/snapshot-subtree-syntax

Add support for snapshot:path syntax

changelog/unreleased/issue-3871

@@ -0,0 +1,22 @@
+Enhancement: Support `<snapshot>:<subfolder>` syntax to select subfolders
+
+Commands like `diff` or `restore` always worked with the full snapshot. This
+did not allow comparing only a specific subfolder or only restoring that folder
+(`restore --include subfolder` limits the restored files, but still creates the
+directories included in `subfolder`).
+
+The commands `diff`, `dump`, `ls`, `restore` now support the
+`<snapshot>:<subfolder>` syntax, where `snapshot` is the ID of a snapshot  (or
+the string `latest`) and `subfolder` is a path within the snapshot. The
+commands will then only work with the specified path of the snapshot. The
+`subfolder` must be a path to a folder as returned by `ls`. 
+
+`restic restore -t target latest:/some/path`
+`restic diff 12345678:/some/path 90abcef:/some/path`
+
+For debugging purposes, the `cat` command now supports `cat tree
+<snapshot>:<subfolder>` to return the directory metadata for the given
+subfolder.
+
+https://github.com/restic/restic/issues/3871
+https://github.com/restic/restic/pull/4334

cmd/restic/cmd_backup.go

@@ -453,7 +453,7 @@ func findParentSnapshot(ctx context.Context, repo restic.Repository, opts Backup
 		f.Tags = []restic.TagList{opts.Tags.Flatten()}
 	}
 
-	sn, err := f.FindLatest(ctx, repo.Backend(), repo, snName)
+	sn, _, err := f.FindLatest(ctx, repo.Backend(), repo, snName)
 	// Snapshot not found is ok if no explicit parent was set
 	if opts.Parent == "" && errors.Is(err, restic.ErrNoSnapshotFound) {
 		err = nil

cmd/restic/cmd_cat.go

@@ -13,7 +13,7 @@ import (
 )
 
 var cmdCat = &cobra.Command{
-	Use:   "cat [flags] [pack|blob|snapshot|index|key|masterkey|config|lock] ID",
+	Use:   "cat [flags] [masterkey|config|pack ID|blob ID|snapshot ID|index ID|key ID|lock ID|tree snapshot:subfolder]",
 	Short: "Print internal objects to stdout",
 	Long: `
 The "cat" command is used to print internal objects to stdout.
@@ -55,7 +55,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
 	tpe := args[0]
 
 	var id restic.ID
-	if tpe != "masterkey" && tpe != "config" && tpe != "snapshot" {
+	if tpe != "masterkey" && tpe != "config" && tpe != "snapshot" && tpe != "tree" {
 		id, err = restic.ParseID(args[1])
 		if err != nil {
 			return errors.Fatalf("unable to parse ID: %v\n", err)
@@ -80,7 +80,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
 		Println(string(buf))
 		return nil
 	case "snapshot":
-		sn, err := restic.FindSnapshot(ctx, repo.Backend(), repo, args[1])
+		sn, _, err := restic.FindSnapshot(ctx, repo.Backend(), repo, args[1])
 		if err != nil {
 			return errors.Fatalf("could not find snapshot: %v\n", err)
 		}
@@ -165,6 +165,29 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
 
 		return errors.Fatal("blob not found")
 
+	case "tree":
+		sn, subfolder, err := restic.FindSnapshot(ctx, repo.Backend(), repo, args[1])
+		if err != nil {
+			return errors.Fatalf("could not find snapshot: %v\n", err)
+		}
+
+		err = repo.LoadIndex(ctx)
+		if err != nil {
+			return err
+		}
+
+		sn.Tree, err = restic.FindTreeDirectory(ctx, repo, sn.Tree, subfolder)
+		if err != nil {
+			return err
+		}
+
+		buf, err := repo.LoadBlob(ctx, restic.TreeBlob, *sn.Tree, nil)
+		if err != nil {
+			return err
+		}
+		_, err = globalOptions.stdout.Write(buf)
+		return err
+
 	default:
 		return errors.Fatal("invalid type")
 	}

cmd/restic/cmd_diff.go

@@ -54,12 +54,12 @@ func init() {
 	f.BoolVar(&diffOptions.ShowMetadata, "metadata", false, "print changes in metadata")
 }
 
-func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.Repository, desc string) (*restic.Snapshot, error) {
+func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.Repository, desc string) (*restic.Snapshot, string, error) {
-	sn, err := restic.FindSnapshot(ctx, be, repo, desc)
+	sn, subfolder, err := restic.FindSnapshot(ctx, be, repo, desc)
 	if err != nil {
-		return nil, errors.Fatal(err.Error())
+		return nil, "", errors.Fatal(err.Error())
 	}
-	return sn, err
+	return sn, subfolder, err
 }
 
 // Comparer collects all things needed to compare two snapshots.
@@ -346,12 +346,12 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []
 	if err != nil {
 		return err
 	}
-	sn1, err := loadSnapshot(ctx, be, repo, args[0])
+	sn1, subfolder1, err := loadSnapshot(ctx, be, repo, args[0])
 	if err != nil {
 		return err
 	}
 
-	sn2, err := loadSnapshot(ctx, be, repo, args[1])
+	sn2, subfolder2, err := loadSnapshot(ctx, be, repo, args[1])
 	if err != nil {
 		return err
 	}
@@ -372,6 +372,16 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []
 		return errors.Errorf("snapshot %v has nil tree", sn2.ID().Str())
 	}
 
+	sn1.Tree, err = restic.FindTreeDirectory(ctx, repo, sn1.Tree, subfolder1)
+	if err != nil {
+		return err
+	}
+
+	sn2.Tree, err = restic.FindTreeDirectory(ctx, repo, sn2.Tree, subfolder2)
+	if err != nil {
+		return err
+	}
+
 	c := &Comparer{
 		repo: repo,
 		opts: diffOptions,

cmd/restic/cmd_dump.go

@@ -139,7 +139,7 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args []
 		}
 	}
 
-	sn, err := (&restic.SnapshotFilter{
+	sn, subfolder, err := (&restic.SnapshotFilter{
 		Hosts: opts.Hosts,
 		Paths: opts.Paths,
 		Tags:  opts.Tags,
@@ -153,6 +153,11 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args []
 		return err
 	}
 
+	sn.Tree, err = restic.FindTreeDirectory(ctx, repo, sn.Tree, subfolder)
+	if err != nil {
+		return err
+	}
+
 	tree, err := restic.LoadTree(ctx, repo, *sn.Tree)
 	if err != nil {
 		return errors.Fatalf("loading tree for snapshot %q failed: %v", snapshotIDString, err)

cmd/restic/cmd_ls.go

@@ -212,7 +212,7 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
 		}
 	}
 
-	sn, err := (&restic.SnapshotFilter{
+	sn, subfolder, err := (&restic.SnapshotFilter{
 		Hosts: opts.Hosts,
 		Paths: opts.Paths,
 		Tags:  opts.Tags,
@@ -221,6 +221,11 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
 		return err
 	}
 
+	sn.Tree, err = restic.FindTreeDirectory(ctx, repo, sn.Tree, subfolder)
+	if err != nil {
+		return err
+	}
+
 	printSnapshot(sn)
 
 	err = walker.Walk(ctx, repo, *sn.Tree, nil, func(_ restic.ID, nodepath string, node *restic.Node, err error) (bool, error) {

cmd/restic/cmd_restore.go

@@ -161,7 +161,7 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
 		}
 	}
 
-	sn, err := (&restic.SnapshotFilter{
+	sn, subfolder, err := (&restic.SnapshotFilter{
 		Hosts: opts.Hosts,
 		Paths: opts.Paths,
 		Tags:  opts.Tags,
@@ -175,6 +175,11 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
 		return err
 	}
 
+	sn.Tree, err = restic.FindTreeDirectory(ctx, repo, sn.Tree, subfolder)
+	if err != nil {
+		return err
+	}
+
 	msg := ui.NewMessage(term, gopts.verbosity)
 	var printer restoreui.ProgressPrinter
 	if gopts.JSON {

doc/040_backup.rst

@@ -451,6 +451,15 @@ and displays a small statistic, just pass the command two snapshot IDs:
       Added:   16.403 MiB
       Removed: 16.402 MiB
 
+To only compare files in specific subfolders, you can use the ``<snapshot>:<subfolder>``
+syntax, where ``snapshot`` is the ID of a snapshot (or the string ``latest``) and ``subfolder``
+is a path within the snapshot. For example, to only compare files in the ``/restic``
+folder, you could use the following command:
+
+.. code-block:: console
+
+    $ restic -r /srv/restic-repo diff 5845b002:/restic 2ab627a6:/restic
+
 
 Backing up special items and metadata
 *************************************

doc/050_restore.rst

@@ -48,6 +48,18 @@ files in the snapshot. For example, to restore a single file:
 
 This will restore the file ``foo`` to ``/tmp/restore-work/work/foo``.
 
+To only restore a specific subfolder, you can use the ``<snapshot>:<subfolder>``
+syntax, where ``snapshot`` is the ID of a snapshot (or the string ``latest``)
+and ``subfolder`` is a path within the snapshot.
+
+.. code-block:: console
+
+    $ restic -r /srv/restic-repo restore 79766175:/work --target /tmp/restore-work --include /foo
+    enter password for repository:
+    restoring <Snapshot of [/home/user/work] at 2015-05-08 21:40:19.884408621 +0200 CEST> to /tmp/restore-work
+
+This will restore the file ``foo`` to ``/tmp/restore-work/foo``.
+
 You can use the command ``restic ls latest`` or ``restic find foo`` to find the
 path to the file within the snapshot. This path you can then pass to
 ``--include`` in verbatim to only restore the single file or directory.
@@ -156,3 +168,9 @@ output the contents in the tar (default) or zip format:
 
     $ restic -r /srv/restic-repo dump -a zip latest /home/other/work > restore.zip
 
+The folder content is then contained at ``/home/other/work`` within the archive.
+To include the folder content at the root of the archive, you can use the ``<snapshot>:<subfolder>`` syntax:
+
+.. code-block:: console
+
+    $ restic -r /srv/restic-repo dump latest:/home/other/work / > restore.tar

internal/fuse/fuse_test.go

@@ -73,7 +73,7 @@ func TestFuseFile(t *testing.T) {
 
 	timestamp, err := time.Parse(time.RFC3339, "2017-01-24T10:42:56+01:00")
 	rtest.OK(t, err)
-	restic.TestCreateSnapshot(t, repo, timestamp, 2, 0.1)
+	restic.TestCreateSnapshot(t, repo, timestamp, 2)
 
 	sn := loadFirstSnapshot(t, repo)
 	tree := loadTree(t, repo, *sn.Tree)
@@ -180,7 +180,7 @@ func TestFuseDir(t *testing.T) {
 // Test top-level directories for their UID and GID.
 func TestTopUIDGID(t *testing.T) {
 	repo := repository.TestRepository(t)
-	restic.TestCreateSnapshot(t, repo, time.Unix(1460289341, 207401672), 0, 0)
+	restic.TestCreateSnapshot(t, repo, time.Unix(1460289341, 207401672), 0)
 
 	testTopUIDGID(t, Config{}, repo, uint32(os.Getuid()), uint32(os.Getgid()))
 	testTopUIDGID(t, Config{OwnerIsRoot: true}, repo, 0, 0)

internal/index/master_index_test.go

@@ -340,11 +340,11 @@ var (
 	depth        = 3
 )
 
-func createFilledRepo(t testing.TB, snapshots int, dup float32, version uint) restic.Repository {
+func createFilledRepo(t testing.TB, snapshots int, version uint) restic.Repository {
 	repo := repository.TestRepositoryWithVersion(t, version)
 
 	for i := 0; i < snapshots; i++ {
-		restic.TestCreateSnapshot(t, repo, snapshotTime.Add(time.Duration(i)*time.Second), depth, dup)
+		restic.TestCreateSnapshot(t, repo, snapshotTime.Add(time.Duration(i)*time.Second), depth)
 	}
 	return repo
 }
@@ -354,7 +354,7 @@ func TestIndexSave(t *testing.T) {
 }
 
 func testIndexSave(t *testing.T, version uint) {
-	repo := createFilledRepo(t, 3, 0, version)
+	repo := createFilledRepo(t, 3, version)
 
 	err := repo.LoadIndex(context.TODO())
 	if err != nil {

internal/restic/find_test.go

@@ -88,7 +88,7 @@ func TestFindUsedBlobs(t *testing.T) {
 
 	var snapshots []*restic.Snapshot
 	for i := 0; i < findTestSnapshots; i++ {
-		sn := restic.TestCreateSnapshot(t, repo, findTestTime.Add(time.Duration(i)*time.Second), findTestDepth, 0)
+		sn := restic.TestCreateSnapshot(t, repo, findTestTime.Add(time.Duration(i)*time.Second), findTestDepth)
 		t.Logf("snapshot %v saved, tree %v", sn.ID().Str(), sn.Tree.Str())
 		snapshots = append(snapshots, sn)
 	}
@@ -131,7 +131,7 @@ func TestMultiFindUsedBlobs(t *testing.T) {
 
 	var snapshotTrees restic.IDs
 	for i := 0; i < findTestSnapshots; i++ {
-		sn := restic.TestCreateSnapshot(t, repo, findTestTime.Add(time.Duration(i)*time.Second), findTestDepth, 0)
+		sn := restic.TestCreateSnapshot(t, repo, findTestTime.Add(time.Duration(i)*time.Second), findTestDepth)
 		t.Logf("snapshot %v saved, tree %v", sn.ID().Str(), sn.Tree.Str())
 		snapshotTrees = append(snapshotTrees, *sn.Tree)
 	}
@@ -177,7 +177,7 @@ func (r ForbiddenRepo) Connections() uint {
 func TestFindUsedBlobsSkipsSeenBlobs(t *testing.T) {
 	repo := repository.TestRepository(t)
 
-	snapshot := restic.TestCreateSnapshot(t, repo, findTestTime, findTestDepth, 0)
+	snapshot := restic.TestCreateSnapshot(t, repo, findTestTime, findTestDepth)
 	t.Logf("snapshot %v saved, tree %v", snapshot.ID().Str(), snapshot.Tree.Str())
 
 	usedBlobs := restic.NewBlobSet()
@@ -195,7 +195,7 @@ func TestFindUsedBlobsSkipsSeenBlobs(t *testing.T) {
 func BenchmarkFindUsedBlobs(b *testing.B) {
 	repo := repository.TestRepository(b)
 
-	sn := restic.TestCreateSnapshot(b, repo, findTestTime, findTestDepth, 0)
+	sn := restic.TestCreateSnapshot(b, repo, findTestTime, findTestDepth)
 
 	b.ResetTimer()
 

internal/restic/snapshot_find.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/restic/restic/internal/errors"
@@ -82,37 +83,48 @@ func (f *SnapshotFilter) findLatest(ctx context.Context, be Lister, loader Loade
 	return latest, nil
 }
 
+func splitSnapshotID(s string) (id, subfolder string) {
+	id, subfolder, _ = strings.Cut(s, ":")
+	return
+}
+
 // FindSnapshot takes a string and tries to find a snapshot whose ID matches
 // the string as closely as possible.
-func FindSnapshot(ctx context.Context, be Lister, loader LoaderUnpacked, s string) (*Snapshot, error) {
+func FindSnapshot(ctx context.Context, be Lister, loader LoaderUnpacked, s string) (*Snapshot, string, error) {
+	s, subfolder := splitSnapshotID(s)
+
 	// no need to list snapshots if `s` is already a full id
 	id, err := ParseID(s)
 	if err != nil {
 		// find snapshot id with prefix
 		id, err = Find(ctx, be, SnapshotFile, s)
 		if err != nil {
-			return nil, err
+			return nil, "", err
 		}
 	}
-	return LoadSnapshot(ctx, loader, id)
+	sn, err := LoadSnapshot(ctx, loader, id)
+	return sn, subfolder, err
 }
 
 // FindLatest returns either the latest of a filtered list of all snapshots
 // or a snapshot specified by `snapshotID`.
-func (f *SnapshotFilter) FindLatest(ctx context.Context, be Lister, loader LoaderUnpacked, snapshotID string) (*Snapshot, error) {
+func (f *SnapshotFilter) FindLatest(ctx context.Context, be Lister, loader LoaderUnpacked, snapshotID string) (*Snapshot, string, error) {
-	if snapshotID == "latest" {
+	id, subfolder := splitSnapshotID(snapshotID)
+	if id == "latest" {
 		sn, err := f.findLatest(ctx, be, loader)
 		if err == ErrNoSnapshotFound {
 			err = fmt.Errorf("snapshot filter (Paths:%v Tags:%v Hosts:%v): %w",
 				f.Paths, f.Tags, f.Hosts, err)
 		}
-		return sn, err
+		return sn, subfolder, err
 	}
 	return FindSnapshot(ctx, be, loader, snapshotID)
 }
 
 type SnapshotFindCb func(string, *Snapshot, error) error
 
+var ErrInvalidSnapshotSyntax = errors.New("<snapshot>:<subfolder> syntax not allowed")
+
 // FindAll yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots.
 func (f *SnapshotFilter) FindAll(ctx context.Context, be Lister, loader LoaderUnpacked, snapshotIDs []string, fn SnapshotFindCb) error {
 	if len(snapshotIDs) != 0 {
@@ -138,9 +150,14 @@ func (f *SnapshotFilter) FindAll(ctx context.Context, be Lister, loader LoaderUn
 				if sn != nil {
 					ids.Insert(*sn.ID())
 				}
+			} else if strings.HasPrefix(s, "latest:") {
+				err = ErrInvalidSnapshotSyntax
 			} else {
-				sn, err = FindSnapshot(ctx, be, loader, s)
+				var subfolder string
-				if err == nil {
+				sn, subfolder, err = FindSnapshot(ctx, be, loader, s)
+				if err == nil && subfolder != "" {
+					err = ErrInvalidSnapshotSyntax
+				} else if err == nil {
 					if ids.Has(*sn.ID()) {
 						continue
 					}

internal/restic/snapshot_find_test.go

@@ -6,16 +6,17 @@ import (
 
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/test"
 )
 
 func TestFindLatestSnapshot(t *testing.T) {
 	repo := repository.TestRepository(t)
-	restic.TestCreateSnapshot(t, repo, parseTimeUTC("2015-05-05 05:05:05"), 1, 0)
+	restic.TestCreateSnapshot(t, repo, parseTimeUTC("2015-05-05 05:05:05"), 1)
-	restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1, 0)
+	restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1)
-	latestSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2019-09-09 09:09:09"), 1, 0)
+	latestSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2019-09-09 09:09:09"), 1)
 
 	f := restic.SnapshotFilter{Hosts: []string{"foo"}}
-	sn, err := f.FindLatest(context.TODO(), repo.Backend(), repo, "latest")
+	sn, _, err := f.FindLatest(context.TODO(), repo.Backend(), repo, "latest")
 	if err != nil {
 		t.Fatalf("FindLatest returned error: %v", err)
 	}
@@ -27,11 +28,11 @@ func TestFindLatestSnapshot(t *testing.T) {
 
 func TestFindLatestSnapshotWithMaxTimestamp(t *testing.T) {
 	repo := repository.TestRepository(t)
-	restic.TestCreateSnapshot(t, repo, parseTimeUTC("2015-05-05 05:05:05"), 1, 0)
+	restic.TestCreateSnapshot(t, repo, parseTimeUTC("2015-05-05 05:05:05"), 1)
-	desiredSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1, 0)
+	desiredSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1)
-	restic.TestCreateSnapshot(t, repo, parseTimeUTC("2019-09-09 09:09:09"), 1, 0)
+	restic.TestCreateSnapshot(t, repo, parseTimeUTC("2019-09-09 09:09:09"), 1)
 
-	sn, err := (&restic.SnapshotFilter{
+	sn, _, err := (&restic.SnapshotFilter{
 		Hosts:          []string{"foo"},
 		TimestampLimit: parseTimeUTC("2018-08-08 08:08:08"),
 	}).FindLatest(context.TODO(), repo.Backend(), repo, "latest")
@@ -43,3 +44,48 @@ func TestFindLatestSnapshotWithMaxTimestamp(t *testing.T) {
 		t.Errorf("FindLatest returned wrong snapshot ID: %v", *sn.ID())
 	}
 }
+
+func TestFindLatestWithSubpath(t *testing.T) {
+	repo := repository.TestRepository(t)
+	restic.TestCreateSnapshot(t, repo, parseTimeUTC("2015-05-05 05:05:05"), 1)
+	desiredSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1)
+
+	for _, exp := range []struct {
+		query     string
+		subfolder string
+	}{
+		{"latest", ""},
+		{"latest:subfolder", "subfolder"},
+		{desiredSnapshot.ID().Str(), ""},
+		{desiredSnapshot.ID().Str() + ":subfolder", "subfolder"},
+		{desiredSnapshot.ID().String(), ""},
+		{desiredSnapshot.ID().String() + ":subfolder", "subfolder"},
+	} {
+		t.Run("", func(t *testing.T) {
+			sn, subfolder, err := (&restic.SnapshotFilter{}).FindLatest(context.TODO(), repo.Backend(), repo, exp.query)
+			if err != nil {
+				t.Fatalf("FindLatest returned error: %v", err)
+			}
+
+			test.Assert(t, *sn.ID() == *desiredSnapshot.ID(), "FindLatest returned wrong snapshot ID: %v", *sn.ID())
+			test.Assert(t, subfolder == exp.subfolder, "FindLatest returned wrong path in snapshot: %v", subfolder)
+		})
+	}
+}
+
+func TestFindAllSubpathError(t *testing.T) {
+	repo := repository.TestRepository(t)
+	desiredSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1)
+
+	count := 0
+	test.OK(t, (&restic.SnapshotFilter{}).FindAll(context.TODO(), repo.Backend(), repo,
+		[]string{"latest:subfolder", desiredSnapshot.ID().Str() + ":subfolder"},
+		func(id string, sn *restic.Snapshot, err error) error {
+			if err == restic.ErrInvalidSnapshotSyntax {
+				count++
+				return nil
+			}
+			return err
+		}))
+	test.Assert(t, count == 2, "unexpected number of subfolder errors: %v, wanted %v", count, 2)
+}

internal/restic/testdata/used_blobs_snapshot0

@@ -1,7 +1,8 @@
 {"ID":"05bddd650a800f83f7c0d844cecb1e02f99ce962df5652a53842be50386078e1","Type":"data"}
 {"ID":"087040b12f129e89e4eab2b86aa14467404366a17a6082efb0d11fa7e2f9f58e","Type":"data"}
+{"ID":"08a650e4d7575177ddeabf6a96896b76fa7e621aa3dd75e77293f22ce6c0c420","Type":"tree"}
 {"ID":"1e0f0e5799b9d711e07883050366c7eee6b7481c0d884694093149f6c4e9789a","Type":"data"}
-{"ID":"229eac8e4e6c2e8d7b1d9f9627ab5d1a59cb17c5744c1e3634215116e7a92e7d","Type":"tree"}
+{"ID":"435b9207cd489b41a7d119e0d75eab2a861e2b3c8d4d12ac51873ff76be0cf73","Type":"tree"}
 {"ID":"4719f8a039f5b745e16cf90e5b84c9255c290d500da716f7dd25909cdabb85b6","Type":"data"}
 {"ID":"4e352975938a29711c3003c498185972235af261a6cf8cf700a8a6ee4f914b05","Type":"data"}
 {"ID":"606772eacb7fe1a79267088dcadd13431914854faf1d39d47fe99a26b9fecdcb","Type":"data"}
@@ -9,7 +10,6 @@
 {"ID":"72b6eb0fd0d87e00392f8b91efc1a4c3f7f5c0c76f861b38aea054bc9d43463b","Type":"data"}
 {"ID":"77ab53b52e0cf13b300d1b7f6dac89287c8d86769d85e8a273311006ce6359be","Type":"data"}
 {"ID":"99dab094430d3c1be22c801a6ad7364d490a8d2ce3f9dfa3d2677431446925f4","Type":"data"}
-{"ID":"9face1b278a49ef8819fbc1855ce573a85077453bbf6683488cad7767c3a38a7","Type":"tree"}
 {"ID":"a4c97189465344038584e76c965dd59100eaed051db1fa5ba0e143897e2c87f1","Type":"data"}
 {"ID":"a69c8621776ca8bb34c6c90e5ad811ddc8e2e5cfd6bb0cec5e75cca70e0b9ade","Type":"data"}
 {"ID":"b11f4dd9d2722b3325186f57cd13a71a3af7791118477f355b49d101104e4c22","Type":"data"}
@@ -19,5 +19,5 @@
 {"ID":"b9e634143719742fe77feed78b61f09573d59d2efa23d6d54afe6c159d220503","Type":"data"}
 {"ID":"ca896fc9ebf95fcffd7c768b07b92110b21e332a47fef7e382bf15363b0ece1a","Type":"data"}
 {"ID":"e6fe3512ea23a4ebf040d30958c669f7ffe724400f155a756467a9f3cafc27c5","Type":"data"}
-{"ID":"e96774ac5abfbb59940939f614d65a397fb7b5abba76c29bfe14479c6616eea0","Type":"tree"}
 {"ID":"ed00928ce97ac5acd27c862d9097e606536e9063af1c47481257811f66260f3a","Type":"data"}
+{"ID":"fb62dd9093c4958b019b90e591b2d36320ff381a24bdc9c5db3b8960ff94d174","Type":"tree"}

internal/restic/testdata/used_blobs_snapshot1

@@ -1,4 +1,3 @@
-{"ID":"04ff190aea26dae65ba4c782926cdfb700b484a8b802a5ffd58e3fadcf70b797","Type":"tree"}
 {"ID":"05bddd650a800f83f7c0d844cecb1e02f99ce962df5652a53842be50386078e1","Type":"data"}
 {"ID":"18dcaa1a676823c909aafabbb909652591915eebdde4f9a65cee955157583494","Type":"data"}
 {"ID":"4719f8a039f5b745e16cf90e5b84c9255c290d500da716f7dd25909cdabb85b6","Type":"data"}
@@ -8,8 +7,9 @@
 {"ID":"a69c8621776ca8bb34c6c90e5ad811ddc8e2e5cfd6bb0cec5e75cca70e0b9ade","Type":"data"}
 {"ID":"b1f2ae9d748035e5bd9a87f2579405166d150c6560d8919496f02855e1c36cf9","Type":"data"}
 {"ID":"b9e634143719742fe77feed78b61f09573d59d2efa23d6d54afe6c159d220503","Type":"data"}
-{"ID":"bdd5a029dd295e5998c518022547d185794e72d8f8c38709a638c5841284daef","Type":"tree"}
 {"ID":"ca896fc9ebf95fcffd7c768b07b92110b21e332a47fef7e382bf15363b0ece1a","Type":"data"}
 {"ID":"cc4cab5b20a3a88995f8cdb8b0698d67a32dbc5b54487f03cb612c30a626af39","Type":"data"}
 {"ID":"e6fe3512ea23a4ebf040d30958c669f7ffe724400f155a756467a9f3cafc27c5","Type":"data"}
+{"ID":"e9f3c4fe78e903cba60d310a9668c42232c8274b3f29b5ecebb6ff1aaeabd7e3","Type":"tree"}
 {"ID":"ed00928ce97ac5acd27c862d9097e606536e9063af1c47481257811f66260f3a","Type":"data"}
+{"ID":"ff58f76c2313e68aa9aaaece855183855ac4ff682910404c2ae33dc999ebaca2","Type":"tree"}

internal/restic/testdata/used_blobs_snapshot2

@@ -1,6 +1,7 @@
 {"ID":"05bddd650a800f83f7c0d844cecb1e02f99ce962df5652a53842be50386078e1","Type":"data"}
 {"ID":"087040b12f129e89e4eab2b86aa14467404366a17a6082efb0d11fa7e2f9f58e","Type":"data"}
 {"ID":"0b88f99abc5ac71c54b3e8263c52ecb7d8903462779afdb3c8176ec5c4bb04fb","Type":"data"}
+{"ID":"0e1a817fca83f569d1733b11eba14b6c9b176e41bca3644eed8b29cb907d84d3","Type":"tree"}
 {"ID":"1e0f0e5799b9d711e07883050366c7eee6b7481c0d884694093149f6c4e9789a","Type":"data"}
 {"ID":"27917462f89cecae77a4c8fb65a094b9b75a917f13794c628b1640b17f4c4981","Type":"data"}
 {"ID":"32745e4b26a5883ecec272c9fbfe7f3c9835c9ab41c9a2baa4d06f319697a0bd","Type":"data"}
@@ -10,15 +11,14 @@
 {"ID":"6b5fd3a9baf615489c82a99a71f9917bf9a2d82d5f640d7f47d175412c4b8d19","Type":"data"}
 {"ID":"95c97192efa810ccb1cee112238dca28673fbffce205d75ce8cc990a31005a51","Type":"data"}
 {"ID":"99dab094430d3c1be22c801a6ad7364d490a8d2ce3f9dfa3d2677431446925f4","Type":"data"}
-{"ID":"9face1b278a49ef8819fbc1855ce573a85077453bbf6683488cad7767c3a38a7","Type":"tree"}
 {"ID":"a4c97189465344038584e76c965dd59100eaed051db1fa5ba0e143897e2c87f1","Type":"data"}
-{"ID":"a5f2ffcd54e28e2ef3089c35b72aafda66161125e23dad581087ccd050c111c3","Type":"tree"}
 {"ID":"a69c8621776ca8bb34c6c90e5ad811ddc8e2e5cfd6bb0cec5e75cca70e0b9ade","Type":"data"}
-{"ID":"ab5205525de94e564e3a00f634fcf9ebc397debd567734c68da7b406e612aae4","Type":"tree"}
 {"ID":"b6a7e8d2aa717e0a6bd68abab512c6b566074b5a6ca2edf4cd446edc5857d732","Type":"data"}
-{"ID":"be2055b7125ccf824fcfa8faa4eb3985119012bac26643944eee46218e71306e","Type":"tree"}
+{"ID":"bad84ed273c5fbfb40aa839a171675b7f16f5e67f3eaf4448730caa0ee27297c","Type":"tree"}
 {"ID":"bfc2fdb527b0c9f66bbb8d4ff1c44023cc2414efcc7f0831c10debab06bb4388","Type":"tree"}
 {"ID":"ca896fc9ebf95fcffd7c768b07b92110b21e332a47fef7e382bf15363b0ece1a","Type":"data"}
+{"ID":"d1d3137eb08de6d8c5d9f44788c45a9fea9bb082e173bed29a0945b3347f2661","Type":"tree"}
 {"ID":"e6fe3512ea23a4ebf040d30958c669f7ffe724400f155a756467a9f3cafc27c5","Type":"data"}
 {"ID":"ed00928ce97ac5acd27c862d9097e606536e9063af1c47481257811f66260f3a","Type":"data"}
 {"ID":"f3cd67d9c14d2a81663d63522ab914e465b021a3b65e2f1ea6caf7478f2ec139","Type":"data"}
+{"ID":"fb62dd9093c4958b019b90e591b2d36320ff381a24bdc9c5db3b8960ff94d174","Type":"tree"}

internal/restic/testing.go

@@ -2,7 +2,6 @@ package restic
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"io"
 	"math/rand"
@@ -19,12 +18,11 @@ func fakeFile(seed, size int64) io.Reader {
 }
 
 type fakeFileSystem struct {
-	t           testing.TB
+	t       testing.TB
-	repo        Repository
+	repo    Repository
-	duplication float32
+	buf     []byte
-	buf         []byte
+	chunker *chunker.Chunker
-	chunker     *chunker.Chunker
+	rand    *rand.Rand
-	rand        *rand.Rand
 }
 
 // saveFile reads from rd and saves the blobs in the repository. The list of
@@ -51,13 +49,9 @@ func (fs *fakeFileSystem) saveFile(ctx context.Context, rd io.Reader) (blobs IDs
 			fs.t.Fatalf("unable to save chunk in repo: %v", err)
 		}
 
-		id := Hash(chunk.Data)
+		id, _, _, err := fs.repo.SaveBlob(ctx, DataBlob, chunk.Data, ID{}, false)
-		if !fs.blobIsKnown(BlobHandle{ID: id, Type: DataBlob}) {
+		if err != nil {
-			_, _, _, err := fs.repo.SaveBlob(ctx, DataBlob, chunk.Data, id, true)
+			fs.t.Fatalf("error saving chunk: %v", err)
-			if err != nil {
-				fs.t.Fatalf("error saving chunk: %v", err)
-			}
-
 		}
 
 		blobs = append(blobs, id)
@@ -72,30 +66,6 @@ const (
 	maxNodes    = 15
 )
 
-func (fs *fakeFileSystem) treeIsKnown(tree *Tree) (bool, []byte, ID) {
-	data, err := json.Marshal(tree)
-	if err != nil {
-		fs.t.Fatalf("json.Marshal(tree) returned error: %v", err)
-		return false, nil, ID{}
-	}
-	data = append(data, '\n')
-
-	id := Hash(data)
-	return fs.blobIsKnown(BlobHandle{ID: id, Type: TreeBlob}), data, id
-}
-
-func (fs *fakeFileSystem) blobIsKnown(bh BlobHandle) bool {
-	if fs.rand.Float32() < fs.duplication {
-		return false
-	}
-
-	if fs.repo.Index().Has(bh) {
-		return true
-	}
-
-	return false
-}
-
 // saveTree saves a tree of fake files in the repo and returns the ID.
 func (fs *fakeFileSystem) saveTree(ctx context.Context, seed int64, depth int) ID {
 	rnd := rand.NewSource(seed)
@@ -134,16 +104,12 @@ func (fs *fakeFileSystem) saveTree(ctx context.Context, seed int64, depth int) I
 		tree.Nodes = append(tree.Nodes, node)
 	}
 
-	known, buf, id := fs.treeIsKnown(&tree)
+	tree.Sort()
-	if known {
-		return id
-	}
 
-	_, _, _, err := fs.repo.SaveBlob(ctx, TreeBlob, buf, id, false)
+	id, err := SaveTree(ctx, fs.repo, &tree)
 	if err != nil {
-		fs.t.Fatal(err)
+		fs.t.Fatalf("SaveTree returned error: %v", err)
 	}
-
 	return id
 }
 
@@ -152,22 +118,20 @@ func (fs *fakeFileSystem) saveTree(ctx context.Context, seed int64, depth int) I
 // also used as the snapshot's timestamp. The tree's depth can be specified
 // with the parameter depth. The parameter duplication is a probability that
 // the same blob will saved again.
-func TestCreateSnapshot(t testing.TB, repo Repository, at time.Time, depth int, duplication float32) *Snapshot {
+func TestCreateSnapshot(t testing.TB, repo Repository, at time.Time, depth int) *Snapshot {
 	seed := at.Unix()
 	t.Logf("create fake snapshot at %s with seed %d", at, seed)
 
 	fakedir := fmt.Sprintf("fakedir-at-%v", at.Format("2006-01-02 15:04:05"))
-	snapshot, err := NewSnapshot([]string{fakedir}, []string{"test"}, "foo", time.Now())
+	snapshot, err := NewSnapshot([]string{fakedir}, []string{"test"}, "foo", at)
 	if err != nil {
 		t.Fatal(err)
 	}
-	snapshot.Time = at
 
 	fs := fakeFileSystem{
-		t:           t,
+		t:    t,
-		repo:        repo,
+		repo: repo,
-		duplication: duplication,
+		rand: rand.New(rand.NewSource(seed)),
-		rand:        rand.New(rand.NewSource(seed)),
 	}
 
 	var wg errgroup.Group

internal/restic/testing_test.go

@@ -39,7 +39,7 @@ func loadAllSnapshots(ctx context.Context, repo restic.Repository, excludeIDs re
 func TestCreateSnapshot(t *testing.T) {
 	repo := repository.TestRepository(t)
 	for i := 0; i < testCreateSnapshots; i++ {
-		restic.TestCreateSnapshot(t, repo, testSnapshotTime.Add(time.Duration(i)*time.Second), testDepth, 0)
+		restic.TestCreateSnapshot(t, repo, testSnapshotTime.Add(time.Duration(i)*time.Second), testDepth)
 	}
 
 	snapshots, err := loadAllSnapshots(context.TODO(), repo, restic.NewIDSet())
@@ -73,6 +73,6 @@ func BenchmarkTestCreateSnapshot(t *testing.B) {
 	t.ResetTimer()
 
 	for i := 0; i < t.N; i++ {
-		restic.TestCreateSnapshot(t, repo, testSnapshotTime.Add(time.Duration(i)*time.Second), testDepth, 0)
+		restic.TestCreateSnapshot(t, repo, testSnapshotTime.Add(time.Duration(i)*time.Second), testDepth)
 	}
 }

internal/restic/tree.go

@@ -5,7 +5,9 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"path"
 	"sort"
+	"strings"
 
 	"github.com/restic/restic/internal/errors"
 
@@ -184,3 +186,32 @@ func (builder *TreeJSONBuilder) Finalize() ([]byte, error) {
 	builder.buf = bytes.Buffer{}
 	return buf, nil
 }
+
+func FindTreeDirectory(ctx context.Context, repo BlobLoader, id *ID, dir string) (*ID, error) {
+	if id == nil {
+		return nil, errors.New("tree id is null")
+	}
+
+	dirs := strings.Split(path.Clean(dir), "/")
+	subfolder := ""
+
+	for _, name := range dirs {
+		if name == "" || name == "." {
+			continue
+		}
+		subfolder = path.Join(subfolder, name)
+		tree, err := LoadTree(ctx, repo, *id)
+		if err != nil {
+			return nil, fmt.Errorf("path %s: %w", subfolder, err)
+		}
+		node := tree.Find(name)
+		if node == nil {
+			return nil, fmt.Errorf("path %s: not found", subfolder)
+		}
+		if node.Type != "dir" || node.Subtree == nil {
+			return nil, fmt.Errorf("path %s: not a directory", subfolder)
+		}
+		id = node.Subtree
+	}
+	return id, nil
+}

internal/restic/tree_test.go

@@ -210,3 +210,37 @@ func benchmarkLoadTree(t *testing.B, version uint) {
 		rtest.OK(t, err)
 	}
 }
+
+func TestFindTreeDirectory(t *testing.T) {
+	repo := repository.TestRepository(t)
+	sn := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:08"), 3)
+
+	for _, exp := range []struct {
+		subfolder string
+		id        restic.ID
+		err       error
+	}{
+		{"", restic.TestParseID("c25199703a67455b34cc0c6e49a8ac8861b268a5dd09dc5b2e31e7380973fc97"), nil},
+		{"/", restic.TestParseID("c25199703a67455b34cc0c6e49a8ac8861b268a5dd09dc5b2e31e7380973fc97"), nil},
+		{".", restic.TestParseID("c25199703a67455b34cc0c6e49a8ac8861b268a5dd09dc5b2e31e7380973fc97"), nil},
+		{"..", restic.ID{}, errors.New("path ..: not found")},
+		{"file-1", restic.ID{}, errors.New("path file-1: not a directory")},
+		{"dir-21", restic.TestParseID("76172f9dec15d7e4cb98d2993032e99f06b73b2f02ffea3b7cfd9e6b4d762712"), nil},
+		{"/dir-21", restic.TestParseID("76172f9dec15d7e4cb98d2993032e99f06b73b2f02ffea3b7cfd9e6b4d762712"), nil},
+		{"dir-21/", restic.TestParseID("76172f9dec15d7e4cb98d2993032e99f06b73b2f02ffea3b7cfd9e6b4d762712"), nil},
+		{"dir-21/dir-24", restic.TestParseID("74626b3fb2bd4b3e572b81a4059b3e912bcf2a8f69fecd9c187613b7173f13b1"), nil},
+	} {
+		t.Run("", func(t *testing.T) {
+			id, err := restic.FindTreeDirectory(context.TODO(), repo, sn.Tree, exp.subfolder)
+			if exp.err == nil {
+				rtest.OK(t, err)
+				rtest.Assert(t, exp.id == *id, "unexpected id, expected %v, got %v", exp.id, id)
+			} else {
+				rtest.Assert(t, exp.err.Error() == err.Error(), "unexpected err, expected %v, got %v", exp.err, err)
+			}
+		})
+	}
+
+	_, err := restic.FindTreeDirectory(context.TODO(), repo, nil, "")
+	rtest.Assert(t, err != nil, "missing error on null tree id")
+}