diff --git a/changelog/unreleased/issue-3871 b/changelog/unreleased/issue-3871 new file mode 100644 index 000000000..786e902d4 --- /dev/null +++ b/changelog/unreleased/issue-3871 @@ -0,0 +1,22 @@ +Enhancement: Support `:` syntax to select subfolders + +Commands like `diff` or `restore` always worked with the full snapshot. This +did not allow comparing only a specific subfolder or only restoring that folder +(`restore --include subfolder` limits the restored files, but still creates the +directories included in `subfolder`). + +The commands `diff`, `dump`, `ls`, `restore` now support the +`:` syntax, where `snapshot` is the ID of a snapshot (or +the string `latest`) and `subfolder` is a path within the snapshot. The +commands will then only work with the specified path of the snapshot. The +`subfolder` must be a path to a folder as returned by `ls`. + +`restic restore -t target latest:/some/path` +`restic diff 12345678:/some/path 90abcef:/some/path` + +For debugging purposes, the `cat` command now supports `cat tree +:` to return the directory metadata for the given +subfolder. + +https://github.com/restic/restic/issues/3871 +https://github.com/restic/restic/pull/4334 diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index 0e678eb2f..6b5706855 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -453,7 +453,7 @@ func findParentSnapshot(ctx context.Context, repo restic.Repository, opts Backup f.Tags = []restic.TagList{opts.Tags.Flatten()} } - sn, err := f.FindLatest(ctx, repo.Backend(), repo, snName) + sn, _, err := f.FindLatest(ctx, repo.Backend(), repo, snName) // Snapshot not found is ok if no explicit parent was set if opts.Parent == "" && errors.Is(err, restic.ErrNoSnapshotFound) { err = nil diff --git a/cmd/restic/cmd_cat.go b/cmd/restic/cmd_cat.go index 771731a58..7c4373812 100644 --- a/cmd/restic/cmd_cat.go +++ b/cmd/restic/cmd_cat.go @@ -13,7 +13,7 @@ import ( ) var cmdCat = &cobra.Command{ - Use: "cat [flags] [pack|blob|snapshot|index|key|masterkey|config|lock] ID", + Use: "cat [flags] [masterkey|config|pack ID|blob ID|snapshot ID|index ID|key ID|lock ID|tree snapshot:subfolder]", Short: "Print internal objects to stdout", Long: ` The "cat" command is used to print internal objects to stdout. @@ -55,7 +55,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error { tpe := args[0] var id restic.ID - if tpe != "masterkey" && tpe != "config" && tpe != "snapshot" { + if tpe != "masterkey" && tpe != "config" && tpe != "snapshot" && tpe != "tree" { id, err = restic.ParseID(args[1]) if err != nil { return errors.Fatalf("unable to parse ID: %v\n", err) @@ -80,7 +80,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error { Println(string(buf)) return nil case "snapshot": - sn, err := restic.FindSnapshot(ctx, repo.Backend(), repo, args[1]) + sn, _, err := restic.FindSnapshot(ctx, repo.Backend(), repo, args[1]) if err != nil { return errors.Fatalf("could not find snapshot: %v\n", err) } @@ -165,6 +165,29 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error { return errors.Fatal("blob not found") + case "tree": + sn, subfolder, err := restic.FindSnapshot(ctx, repo.Backend(), repo, args[1]) + if err != nil { + return errors.Fatalf("could not find snapshot: %v\n", err) + } + + err = repo.LoadIndex(ctx) + if err != nil { + return err + } + + sn.Tree, err = restic.FindTreeDirectory(ctx, repo, sn.Tree, subfolder) + if err != nil { + return err + } + + buf, err := repo.LoadBlob(ctx, restic.TreeBlob, *sn.Tree, nil) + if err != nil { + return err + } + _, err = globalOptions.stdout.Write(buf) + return err + default: return errors.Fatal("invalid type") } diff --git a/cmd/restic/cmd_diff.go b/cmd/restic/cmd_diff.go index 3c59b9580..28e60f464 100644 --- a/cmd/restic/cmd_diff.go +++ b/cmd/restic/cmd_diff.go @@ -54,12 +54,12 @@ func init() { f.BoolVar(&diffOptions.ShowMetadata, "metadata", false, "print changes in metadata") } -func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.Repository, desc string) (*restic.Snapshot, error) { - sn, err := restic.FindSnapshot(ctx, be, repo, desc) +func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.Repository, desc string) (*restic.Snapshot, string, error) { + sn, subfolder, err := restic.FindSnapshot(ctx, be, repo, desc) if err != nil { - return nil, errors.Fatal(err.Error()) + return nil, "", errors.Fatal(err.Error()) } - return sn, err + return sn, subfolder, err } // Comparer collects all things needed to compare two snapshots. @@ -346,12 +346,12 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args [] if err != nil { return err } - sn1, err := loadSnapshot(ctx, be, repo, args[0]) + sn1, subfolder1, err := loadSnapshot(ctx, be, repo, args[0]) if err != nil { return err } - sn2, err := loadSnapshot(ctx, be, repo, args[1]) + sn2, subfolder2, err := loadSnapshot(ctx, be, repo, args[1]) if err != nil { return err } @@ -372,6 +372,16 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args [] return errors.Errorf("snapshot %v has nil tree", sn2.ID().Str()) } + sn1.Tree, err = restic.FindTreeDirectory(ctx, repo, sn1.Tree, subfolder1) + if err != nil { + return err + } + + sn2.Tree, err = restic.FindTreeDirectory(ctx, repo, sn2.Tree, subfolder2) + if err != nil { + return err + } + c := &Comparer{ repo: repo, opts: diffOptions, diff --git a/cmd/restic/cmd_dump.go b/cmd/restic/cmd_dump.go index 34313f582..9acae7ca8 100644 --- a/cmd/restic/cmd_dump.go +++ b/cmd/restic/cmd_dump.go @@ -139,7 +139,7 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args [] } } - sn, err := (&restic.SnapshotFilter{ + sn, subfolder, err := (&restic.SnapshotFilter{ Hosts: opts.Hosts, Paths: opts.Paths, Tags: opts.Tags, @@ -153,6 +153,11 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args [] return err } + sn.Tree, err = restic.FindTreeDirectory(ctx, repo, sn.Tree, subfolder) + if err != nil { + return err + } + tree, err := restic.LoadTree(ctx, repo, *sn.Tree) if err != nil { return errors.Fatalf("loading tree for snapshot %q failed: %v", snapshotIDString, err) diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go index 1cd549e7c..256c9e002 100644 --- a/cmd/restic/cmd_ls.go +++ b/cmd/restic/cmd_ls.go @@ -212,7 +212,7 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri } } - sn, err := (&restic.SnapshotFilter{ + sn, subfolder, err := (&restic.SnapshotFilter{ Hosts: opts.Hosts, Paths: opts.Paths, Tags: opts.Tags, @@ -221,6 +221,11 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri return err } + sn.Tree, err = restic.FindTreeDirectory(ctx, repo, sn.Tree, subfolder) + if err != nil { + return err + } + printSnapshot(sn) err = walker.Walk(ctx, repo, *sn.Tree, nil, func(_ restic.ID, nodepath string, node *restic.Node, err error) (bool, error) { diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index c59ac34de..6ef8c99db 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -161,7 +161,7 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, } } - sn, err := (&restic.SnapshotFilter{ + sn, subfolder, err := (&restic.SnapshotFilter{ Hosts: opts.Hosts, Paths: opts.Paths, Tags: opts.Tags, @@ -175,6 +175,11 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, return err } + sn.Tree, err = restic.FindTreeDirectory(ctx, repo, sn.Tree, subfolder) + if err != nil { + return err + } + msg := ui.NewMessage(term, gopts.verbosity) var printer restoreui.ProgressPrinter if gopts.JSON { diff --git a/doc/040_backup.rst b/doc/040_backup.rst index b01683929..1655e7eed 100644 --- a/doc/040_backup.rst +++ b/doc/040_backup.rst @@ -451,6 +451,15 @@ and displays a small statistic, just pass the command two snapshot IDs: Added: 16.403 MiB Removed: 16.402 MiB +To only compare files in specific subfolders, you can use the ``:`` +syntax, where ``snapshot`` is the ID of a snapshot (or the string ``latest``) and ``subfolder`` +is a path within the snapshot. For example, to only compare files in the ``/restic`` +folder, you could use the following command: + +.. code-block:: console + + $ restic -r /srv/restic-repo diff 5845b002:/restic 2ab627a6:/restic + Backing up special items and metadata ************************************* diff --git a/doc/050_restore.rst b/doc/050_restore.rst index 47a1be003..ed2ddfd40 100644 --- a/doc/050_restore.rst +++ b/doc/050_restore.rst @@ -48,6 +48,18 @@ files in the snapshot. For example, to restore a single file: This will restore the file ``foo`` to ``/tmp/restore-work/work/foo``. +To only restore a specific subfolder, you can use the ``:`` +syntax, where ``snapshot`` is the ID of a snapshot (or the string ``latest``) +and ``subfolder`` is a path within the snapshot. + +.. code-block:: console + + $ restic -r /srv/restic-repo restore 79766175:/work --target /tmp/restore-work --include /foo + enter password for repository: + restoring to /tmp/restore-work + +This will restore the file ``foo`` to ``/tmp/restore-work/foo``. + You can use the command ``restic ls latest`` or ``restic find foo`` to find the path to the file within the snapshot. This path you can then pass to ``--include`` in verbatim to only restore the single file or directory. @@ -151,8 +163,14 @@ output the contents in the tar (default) or zip format: .. code-block:: console $ restic -r /srv/restic-repo dump latest /home/other/work > restore.tar - + .. code-block:: console $ restic -r /srv/restic-repo dump -a zip latest /home/other/work > restore.zip +The folder content is then contained at ``/home/other/work`` within the archive. +To include the folder content at the root of the archive, you can use the ``:`` syntax: + +.. code-block:: console + + $ restic -r /srv/restic-repo dump latest:/home/other/work / > restore.tar diff --git a/internal/fuse/fuse_test.go b/internal/fuse/fuse_test.go index ccdd2f774..0a121b986 100644 --- a/internal/fuse/fuse_test.go +++ b/internal/fuse/fuse_test.go @@ -73,7 +73,7 @@ func TestFuseFile(t *testing.T) { timestamp, err := time.Parse(time.RFC3339, "2017-01-24T10:42:56+01:00") rtest.OK(t, err) - restic.TestCreateSnapshot(t, repo, timestamp, 2, 0.1) + restic.TestCreateSnapshot(t, repo, timestamp, 2) sn := loadFirstSnapshot(t, repo) tree := loadTree(t, repo, *sn.Tree) @@ -180,7 +180,7 @@ func TestFuseDir(t *testing.T) { // Test top-level directories for their UID and GID. func TestTopUIDGID(t *testing.T) { repo := repository.TestRepository(t) - restic.TestCreateSnapshot(t, repo, time.Unix(1460289341, 207401672), 0, 0) + restic.TestCreateSnapshot(t, repo, time.Unix(1460289341, 207401672), 0) testTopUIDGID(t, Config{}, repo, uint32(os.Getuid()), uint32(os.Getgid())) testTopUIDGID(t, Config{OwnerIsRoot: true}, repo, 0, 0) diff --git a/internal/index/master_index_test.go b/internal/index/master_index_test.go index 5d12956bd..45286e89c 100644 --- a/internal/index/master_index_test.go +++ b/internal/index/master_index_test.go @@ -340,11 +340,11 @@ var ( depth = 3 ) -func createFilledRepo(t testing.TB, snapshots int, dup float32, version uint) restic.Repository { +func createFilledRepo(t testing.TB, snapshots int, version uint) restic.Repository { repo := repository.TestRepositoryWithVersion(t, version) for i := 0; i < snapshots; i++ { - restic.TestCreateSnapshot(t, repo, snapshotTime.Add(time.Duration(i)*time.Second), depth, dup) + restic.TestCreateSnapshot(t, repo, snapshotTime.Add(time.Duration(i)*time.Second), depth) } return repo } @@ -354,7 +354,7 @@ func TestIndexSave(t *testing.T) { } func testIndexSave(t *testing.T, version uint) { - repo := createFilledRepo(t, 3, 0, version) + repo := createFilledRepo(t, 3, version) err := repo.LoadIndex(context.TODO()) if err != nil { diff --git a/internal/restic/find_test.go b/internal/restic/find_test.go index 234561b6d..1ae30ded9 100644 --- a/internal/restic/find_test.go +++ b/internal/restic/find_test.go @@ -88,7 +88,7 @@ func TestFindUsedBlobs(t *testing.T) { var snapshots []*restic.Snapshot for i := 0; i < findTestSnapshots; i++ { - sn := restic.TestCreateSnapshot(t, repo, findTestTime.Add(time.Duration(i)*time.Second), findTestDepth, 0) + sn := restic.TestCreateSnapshot(t, repo, findTestTime.Add(time.Duration(i)*time.Second), findTestDepth) t.Logf("snapshot %v saved, tree %v", sn.ID().Str(), sn.Tree.Str()) snapshots = append(snapshots, sn) } @@ -131,7 +131,7 @@ func TestMultiFindUsedBlobs(t *testing.T) { var snapshotTrees restic.IDs for i := 0; i < findTestSnapshots; i++ { - sn := restic.TestCreateSnapshot(t, repo, findTestTime.Add(time.Duration(i)*time.Second), findTestDepth, 0) + sn := restic.TestCreateSnapshot(t, repo, findTestTime.Add(time.Duration(i)*time.Second), findTestDepth) t.Logf("snapshot %v saved, tree %v", sn.ID().Str(), sn.Tree.Str()) snapshotTrees = append(snapshotTrees, *sn.Tree) } @@ -177,7 +177,7 @@ func (r ForbiddenRepo) Connections() uint { func TestFindUsedBlobsSkipsSeenBlobs(t *testing.T) { repo := repository.TestRepository(t) - snapshot := restic.TestCreateSnapshot(t, repo, findTestTime, findTestDepth, 0) + snapshot := restic.TestCreateSnapshot(t, repo, findTestTime, findTestDepth) t.Logf("snapshot %v saved, tree %v", snapshot.ID().Str(), snapshot.Tree.Str()) usedBlobs := restic.NewBlobSet() @@ -195,7 +195,7 @@ func TestFindUsedBlobsSkipsSeenBlobs(t *testing.T) { func BenchmarkFindUsedBlobs(b *testing.B) { repo := repository.TestRepository(b) - sn := restic.TestCreateSnapshot(b, repo, findTestTime, findTestDepth, 0) + sn := restic.TestCreateSnapshot(b, repo, findTestTime, findTestDepth) b.ResetTimer() diff --git a/internal/restic/snapshot_find.go b/internal/restic/snapshot_find.go index b577b0919..cb761aee3 100644 --- a/internal/restic/snapshot_find.go +++ b/internal/restic/snapshot_find.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "path/filepath" + "strings" "time" "github.com/restic/restic/internal/errors" @@ -82,37 +83,48 @@ func (f *SnapshotFilter) findLatest(ctx context.Context, be Lister, loader Loade return latest, nil } +func splitSnapshotID(s string) (id, subfolder string) { + id, subfolder, _ = strings.Cut(s, ":") + return +} + // FindSnapshot takes a string and tries to find a snapshot whose ID matches // the string as closely as possible. -func FindSnapshot(ctx context.Context, be Lister, loader LoaderUnpacked, s string) (*Snapshot, error) { +func FindSnapshot(ctx context.Context, be Lister, loader LoaderUnpacked, s string) (*Snapshot, string, error) { + s, subfolder := splitSnapshotID(s) + // no need to list snapshots if `s` is already a full id id, err := ParseID(s) if err != nil { // find snapshot id with prefix id, err = Find(ctx, be, SnapshotFile, s) if err != nil { - return nil, err + return nil, "", err } } - return LoadSnapshot(ctx, loader, id) + sn, err := LoadSnapshot(ctx, loader, id) + return sn, subfolder, err } // FindLatest returns either the latest of a filtered list of all snapshots // or a snapshot specified by `snapshotID`. -func (f *SnapshotFilter) FindLatest(ctx context.Context, be Lister, loader LoaderUnpacked, snapshotID string) (*Snapshot, error) { - if snapshotID == "latest" { +func (f *SnapshotFilter) FindLatest(ctx context.Context, be Lister, loader LoaderUnpacked, snapshotID string) (*Snapshot, string, error) { + id, subfolder := splitSnapshotID(snapshotID) + if id == "latest" { sn, err := f.findLatest(ctx, be, loader) if err == ErrNoSnapshotFound { err = fmt.Errorf("snapshot filter (Paths:%v Tags:%v Hosts:%v): %w", f.Paths, f.Tags, f.Hosts, err) } - return sn, err + return sn, subfolder, err } return FindSnapshot(ctx, be, loader, snapshotID) } type SnapshotFindCb func(string, *Snapshot, error) error +var ErrInvalidSnapshotSyntax = errors.New(": syntax not allowed") + // FindAll yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots. func (f *SnapshotFilter) FindAll(ctx context.Context, be Lister, loader LoaderUnpacked, snapshotIDs []string, fn SnapshotFindCb) error { if len(snapshotIDs) != 0 { @@ -138,9 +150,14 @@ func (f *SnapshotFilter) FindAll(ctx context.Context, be Lister, loader LoaderUn if sn != nil { ids.Insert(*sn.ID()) } + } else if strings.HasPrefix(s, "latest:") { + err = ErrInvalidSnapshotSyntax } else { - sn, err = FindSnapshot(ctx, be, loader, s) - if err == nil { + var subfolder string + sn, subfolder, err = FindSnapshot(ctx, be, loader, s) + if err == nil && subfolder != "" { + err = ErrInvalidSnapshotSyntax + } else if err == nil { if ids.Has(*sn.ID()) { continue } diff --git a/internal/restic/snapshot_find_test.go b/internal/restic/snapshot_find_test.go index d098b5224..2f16dcb2f 100644 --- a/internal/restic/snapshot_find_test.go +++ b/internal/restic/snapshot_find_test.go @@ -6,16 +6,17 @@ import ( "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/test" ) func TestFindLatestSnapshot(t *testing.T) { repo := repository.TestRepository(t) - restic.TestCreateSnapshot(t, repo, parseTimeUTC("2015-05-05 05:05:05"), 1, 0) - restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1, 0) - latestSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2019-09-09 09:09:09"), 1, 0) + restic.TestCreateSnapshot(t, repo, parseTimeUTC("2015-05-05 05:05:05"), 1) + restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1) + latestSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2019-09-09 09:09:09"), 1) f := restic.SnapshotFilter{Hosts: []string{"foo"}} - sn, err := f.FindLatest(context.TODO(), repo.Backend(), repo, "latest") + sn, _, err := f.FindLatest(context.TODO(), repo.Backend(), repo, "latest") if err != nil { t.Fatalf("FindLatest returned error: %v", err) } @@ -27,11 +28,11 @@ func TestFindLatestSnapshot(t *testing.T) { func TestFindLatestSnapshotWithMaxTimestamp(t *testing.T) { repo := repository.TestRepository(t) - restic.TestCreateSnapshot(t, repo, parseTimeUTC("2015-05-05 05:05:05"), 1, 0) - desiredSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1, 0) - restic.TestCreateSnapshot(t, repo, parseTimeUTC("2019-09-09 09:09:09"), 1, 0) + restic.TestCreateSnapshot(t, repo, parseTimeUTC("2015-05-05 05:05:05"), 1) + desiredSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1) + restic.TestCreateSnapshot(t, repo, parseTimeUTC("2019-09-09 09:09:09"), 1) - sn, err := (&restic.SnapshotFilter{ + sn, _, err := (&restic.SnapshotFilter{ Hosts: []string{"foo"}, TimestampLimit: parseTimeUTC("2018-08-08 08:08:08"), }).FindLatest(context.TODO(), repo.Backend(), repo, "latest") @@ -43,3 +44,48 @@ func TestFindLatestSnapshotWithMaxTimestamp(t *testing.T) { t.Errorf("FindLatest returned wrong snapshot ID: %v", *sn.ID()) } } + +func TestFindLatestWithSubpath(t *testing.T) { + repo := repository.TestRepository(t) + restic.TestCreateSnapshot(t, repo, parseTimeUTC("2015-05-05 05:05:05"), 1) + desiredSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1) + + for _, exp := range []struct { + query string + subfolder string + }{ + {"latest", ""}, + {"latest:subfolder", "subfolder"}, + {desiredSnapshot.ID().Str(), ""}, + {desiredSnapshot.ID().Str() + ":subfolder", "subfolder"}, + {desiredSnapshot.ID().String(), ""}, + {desiredSnapshot.ID().String() + ":subfolder", "subfolder"}, + } { + t.Run("", func(t *testing.T) { + sn, subfolder, err := (&restic.SnapshotFilter{}).FindLatest(context.TODO(), repo.Backend(), repo, exp.query) + if err != nil { + t.Fatalf("FindLatest returned error: %v", err) + } + + test.Assert(t, *sn.ID() == *desiredSnapshot.ID(), "FindLatest returned wrong snapshot ID: %v", *sn.ID()) + test.Assert(t, subfolder == exp.subfolder, "FindLatest returned wrong path in snapshot: %v", subfolder) + }) + } +} + +func TestFindAllSubpathError(t *testing.T) { + repo := repository.TestRepository(t) + desiredSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1) + + count := 0 + test.OK(t, (&restic.SnapshotFilter{}).FindAll(context.TODO(), repo.Backend(), repo, + []string{"latest:subfolder", desiredSnapshot.ID().Str() + ":subfolder"}, + func(id string, sn *restic.Snapshot, err error) error { + if err == restic.ErrInvalidSnapshotSyntax { + count++ + return nil + } + return err + })) + test.Assert(t, count == 2, "unexpected number of subfolder errors: %v, wanted %v", count, 2) +} diff --git a/internal/restic/testdata/used_blobs_snapshot0 b/internal/restic/testdata/used_blobs_snapshot0 index 667ad34db..cc789f043 100644 --- a/internal/restic/testdata/used_blobs_snapshot0 +++ b/internal/restic/testdata/used_blobs_snapshot0 @@ -1,7 +1,8 @@ {"ID":"05bddd650a800f83f7c0d844cecb1e02f99ce962df5652a53842be50386078e1","Type":"data"} {"ID":"087040b12f129e89e4eab2b86aa14467404366a17a6082efb0d11fa7e2f9f58e","Type":"data"} +{"ID":"08a650e4d7575177ddeabf6a96896b76fa7e621aa3dd75e77293f22ce6c0c420","Type":"tree"} {"ID":"1e0f0e5799b9d711e07883050366c7eee6b7481c0d884694093149f6c4e9789a","Type":"data"} -{"ID":"229eac8e4e6c2e8d7b1d9f9627ab5d1a59cb17c5744c1e3634215116e7a92e7d","Type":"tree"} +{"ID":"435b9207cd489b41a7d119e0d75eab2a861e2b3c8d4d12ac51873ff76be0cf73","Type":"tree"} {"ID":"4719f8a039f5b745e16cf90e5b84c9255c290d500da716f7dd25909cdabb85b6","Type":"data"} {"ID":"4e352975938a29711c3003c498185972235af261a6cf8cf700a8a6ee4f914b05","Type":"data"} {"ID":"606772eacb7fe1a79267088dcadd13431914854faf1d39d47fe99a26b9fecdcb","Type":"data"} @@ -9,7 +10,6 @@ {"ID":"72b6eb0fd0d87e00392f8b91efc1a4c3f7f5c0c76f861b38aea054bc9d43463b","Type":"data"} {"ID":"77ab53b52e0cf13b300d1b7f6dac89287c8d86769d85e8a273311006ce6359be","Type":"data"} {"ID":"99dab094430d3c1be22c801a6ad7364d490a8d2ce3f9dfa3d2677431446925f4","Type":"data"} -{"ID":"9face1b278a49ef8819fbc1855ce573a85077453bbf6683488cad7767c3a38a7","Type":"tree"} {"ID":"a4c97189465344038584e76c965dd59100eaed051db1fa5ba0e143897e2c87f1","Type":"data"} {"ID":"a69c8621776ca8bb34c6c90e5ad811ddc8e2e5cfd6bb0cec5e75cca70e0b9ade","Type":"data"} {"ID":"b11f4dd9d2722b3325186f57cd13a71a3af7791118477f355b49d101104e4c22","Type":"data"} @@ -19,5 +19,5 @@ {"ID":"b9e634143719742fe77feed78b61f09573d59d2efa23d6d54afe6c159d220503","Type":"data"} {"ID":"ca896fc9ebf95fcffd7c768b07b92110b21e332a47fef7e382bf15363b0ece1a","Type":"data"} {"ID":"e6fe3512ea23a4ebf040d30958c669f7ffe724400f155a756467a9f3cafc27c5","Type":"data"} -{"ID":"e96774ac5abfbb59940939f614d65a397fb7b5abba76c29bfe14479c6616eea0","Type":"tree"} {"ID":"ed00928ce97ac5acd27c862d9097e606536e9063af1c47481257811f66260f3a","Type":"data"} +{"ID":"fb62dd9093c4958b019b90e591b2d36320ff381a24bdc9c5db3b8960ff94d174","Type":"tree"} diff --git a/internal/restic/testdata/used_blobs_snapshot1 b/internal/restic/testdata/used_blobs_snapshot1 index a5e8caedf..aa840294a 100644 --- a/internal/restic/testdata/used_blobs_snapshot1 +++ b/internal/restic/testdata/used_blobs_snapshot1 @@ -1,4 +1,3 @@ -{"ID":"04ff190aea26dae65ba4c782926cdfb700b484a8b802a5ffd58e3fadcf70b797","Type":"tree"} {"ID":"05bddd650a800f83f7c0d844cecb1e02f99ce962df5652a53842be50386078e1","Type":"data"} {"ID":"18dcaa1a676823c909aafabbb909652591915eebdde4f9a65cee955157583494","Type":"data"} {"ID":"4719f8a039f5b745e16cf90e5b84c9255c290d500da716f7dd25909cdabb85b6","Type":"data"} @@ -8,8 +7,9 @@ {"ID":"a69c8621776ca8bb34c6c90e5ad811ddc8e2e5cfd6bb0cec5e75cca70e0b9ade","Type":"data"} {"ID":"b1f2ae9d748035e5bd9a87f2579405166d150c6560d8919496f02855e1c36cf9","Type":"data"} {"ID":"b9e634143719742fe77feed78b61f09573d59d2efa23d6d54afe6c159d220503","Type":"data"} -{"ID":"bdd5a029dd295e5998c518022547d185794e72d8f8c38709a638c5841284daef","Type":"tree"} {"ID":"ca896fc9ebf95fcffd7c768b07b92110b21e332a47fef7e382bf15363b0ece1a","Type":"data"} {"ID":"cc4cab5b20a3a88995f8cdb8b0698d67a32dbc5b54487f03cb612c30a626af39","Type":"data"} {"ID":"e6fe3512ea23a4ebf040d30958c669f7ffe724400f155a756467a9f3cafc27c5","Type":"data"} +{"ID":"e9f3c4fe78e903cba60d310a9668c42232c8274b3f29b5ecebb6ff1aaeabd7e3","Type":"tree"} {"ID":"ed00928ce97ac5acd27c862d9097e606536e9063af1c47481257811f66260f3a","Type":"data"} +{"ID":"ff58f76c2313e68aa9aaaece855183855ac4ff682910404c2ae33dc999ebaca2","Type":"tree"} diff --git a/internal/restic/testdata/used_blobs_snapshot2 b/internal/restic/testdata/used_blobs_snapshot2 index f6404737e..3ed193f53 100644 --- a/internal/restic/testdata/used_blobs_snapshot2 +++ b/internal/restic/testdata/used_blobs_snapshot2 @@ -1,6 +1,7 @@ {"ID":"05bddd650a800f83f7c0d844cecb1e02f99ce962df5652a53842be50386078e1","Type":"data"} {"ID":"087040b12f129e89e4eab2b86aa14467404366a17a6082efb0d11fa7e2f9f58e","Type":"data"} {"ID":"0b88f99abc5ac71c54b3e8263c52ecb7d8903462779afdb3c8176ec5c4bb04fb","Type":"data"} +{"ID":"0e1a817fca83f569d1733b11eba14b6c9b176e41bca3644eed8b29cb907d84d3","Type":"tree"} {"ID":"1e0f0e5799b9d711e07883050366c7eee6b7481c0d884694093149f6c4e9789a","Type":"data"} {"ID":"27917462f89cecae77a4c8fb65a094b9b75a917f13794c628b1640b17f4c4981","Type":"data"} {"ID":"32745e4b26a5883ecec272c9fbfe7f3c9835c9ab41c9a2baa4d06f319697a0bd","Type":"data"} @@ -10,15 +11,14 @@ {"ID":"6b5fd3a9baf615489c82a99a71f9917bf9a2d82d5f640d7f47d175412c4b8d19","Type":"data"} {"ID":"95c97192efa810ccb1cee112238dca28673fbffce205d75ce8cc990a31005a51","Type":"data"} {"ID":"99dab094430d3c1be22c801a6ad7364d490a8d2ce3f9dfa3d2677431446925f4","Type":"data"} -{"ID":"9face1b278a49ef8819fbc1855ce573a85077453bbf6683488cad7767c3a38a7","Type":"tree"} {"ID":"a4c97189465344038584e76c965dd59100eaed051db1fa5ba0e143897e2c87f1","Type":"data"} -{"ID":"a5f2ffcd54e28e2ef3089c35b72aafda66161125e23dad581087ccd050c111c3","Type":"tree"} {"ID":"a69c8621776ca8bb34c6c90e5ad811ddc8e2e5cfd6bb0cec5e75cca70e0b9ade","Type":"data"} -{"ID":"ab5205525de94e564e3a00f634fcf9ebc397debd567734c68da7b406e612aae4","Type":"tree"} {"ID":"b6a7e8d2aa717e0a6bd68abab512c6b566074b5a6ca2edf4cd446edc5857d732","Type":"data"} -{"ID":"be2055b7125ccf824fcfa8faa4eb3985119012bac26643944eee46218e71306e","Type":"tree"} +{"ID":"bad84ed273c5fbfb40aa839a171675b7f16f5e67f3eaf4448730caa0ee27297c","Type":"tree"} {"ID":"bfc2fdb527b0c9f66bbb8d4ff1c44023cc2414efcc7f0831c10debab06bb4388","Type":"tree"} {"ID":"ca896fc9ebf95fcffd7c768b07b92110b21e332a47fef7e382bf15363b0ece1a","Type":"data"} +{"ID":"d1d3137eb08de6d8c5d9f44788c45a9fea9bb082e173bed29a0945b3347f2661","Type":"tree"} {"ID":"e6fe3512ea23a4ebf040d30958c669f7ffe724400f155a756467a9f3cafc27c5","Type":"data"} {"ID":"ed00928ce97ac5acd27c862d9097e606536e9063af1c47481257811f66260f3a","Type":"data"} {"ID":"f3cd67d9c14d2a81663d63522ab914e465b021a3b65e2f1ea6caf7478f2ec139","Type":"data"} +{"ID":"fb62dd9093c4958b019b90e591b2d36320ff381a24bdc9c5db3b8960ff94d174","Type":"tree"} diff --git a/internal/restic/testing.go b/internal/restic/testing.go index 658935eb6..004df627c 100644 --- a/internal/restic/testing.go +++ b/internal/restic/testing.go @@ -2,7 +2,6 @@ package restic import ( "context" - "encoding/json" "fmt" "io" "math/rand" @@ -19,12 +18,11 @@ func fakeFile(seed, size int64) io.Reader { } type fakeFileSystem struct { - t testing.TB - repo Repository - duplication float32 - buf []byte - chunker *chunker.Chunker - rand *rand.Rand + t testing.TB + repo Repository + buf []byte + chunker *chunker.Chunker + rand *rand.Rand } // saveFile reads from rd and saves the blobs in the repository. The list of @@ -51,13 +49,9 @@ func (fs *fakeFileSystem) saveFile(ctx context.Context, rd io.Reader) (blobs IDs fs.t.Fatalf("unable to save chunk in repo: %v", err) } - id := Hash(chunk.Data) - if !fs.blobIsKnown(BlobHandle{ID: id, Type: DataBlob}) { - _, _, _, err := fs.repo.SaveBlob(ctx, DataBlob, chunk.Data, id, true) - if err != nil { - fs.t.Fatalf("error saving chunk: %v", err) - } - + id, _, _, err := fs.repo.SaveBlob(ctx, DataBlob, chunk.Data, ID{}, false) + if err != nil { + fs.t.Fatalf("error saving chunk: %v", err) } blobs = append(blobs, id) @@ -72,30 +66,6 @@ const ( maxNodes = 15 ) -func (fs *fakeFileSystem) treeIsKnown(tree *Tree) (bool, []byte, ID) { - data, err := json.Marshal(tree) - if err != nil { - fs.t.Fatalf("json.Marshal(tree) returned error: %v", err) - return false, nil, ID{} - } - data = append(data, '\n') - - id := Hash(data) - return fs.blobIsKnown(BlobHandle{ID: id, Type: TreeBlob}), data, id -} - -func (fs *fakeFileSystem) blobIsKnown(bh BlobHandle) bool { - if fs.rand.Float32() < fs.duplication { - return false - } - - if fs.repo.Index().Has(bh) { - return true - } - - return false -} - // saveTree saves a tree of fake files in the repo and returns the ID. func (fs *fakeFileSystem) saveTree(ctx context.Context, seed int64, depth int) ID { rnd := rand.NewSource(seed) @@ -134,16 +104,12 @@ func (fs *fakeFileSystem) saveTree(ctx context.Context, seed int64, depth int) I tree.Nodes = append(tree.Nodes, node) } - known, buf, id := fs.treeIsKnown(&tree) - if known { - return id - } + tree.Sort() - _, _, _, err := fs.repo.SaveBlob(ctx, TreeBlob, buf, id, false) + id, err := SaveTree(ctx, fs.repo, &tree) if err != nil { - fs.t.Fatal(err) + fs.t.Fatalf("SaveTree returned error: %v", err) } - return id } @@ -152,22 +118,20 @@ func (fs *fakeFileSystem) saveTree(ctx context.Context, seed int64, depth int) I // also used as the snapshot's timestamp. The tree's depth can be specified // with the parameter depth. The parameter duplication is a probability that // the same blob will saved again. -func TestCreateSnapshot(t testing.TB, repo Repository, at time.Time, depth int, duplication float32) *Snapshot { +func TestCreateSnapshot(t testing.TB, repo Repository, at time.Time, depth int) *Snapshot { seed := at.Unix() t.Logf("create fake snapshot at %s with seed %d", at, seed) fakedir := fmt.Sprintf("fakedir-at-%v", at.Format("2006-01-02 15:04:05")) - snapshot, err := NewSnapshot([]string{fakedir}, []string{"test"}, "foo", time.Now()) + snapshot, err := NewSnapshot([]string{fakedir}, []string{"test"}, "foo", at) if err != nil { t.Fatal(err) } - snapshot.Time = at fs := fakeFileSystem{ - t: t, - repo: repo, - duplication: duplication, - rand: rand.New(rand.NewSource(seed)), + t: t, + repo: repo, + rand: rand.New(rand.NewSource(seed)), } var wg errgroup.Group diff --git a/internal/restic/testing_test.go b/internal/restic/testing_test.go index 2af5c607e..760a53a52 100644 --- a/internal/restic/testing_test.go +++ b/internal/restic/testing_test.go @@ -39,7 +39,7 @@ func loadAllSnapshots(ctx context.Context, repo restic.Repository, excludeIDs re func TestCreateSnapshot(t *testing.T) { repo := repository.TestRepository(t) for i := 0; i < testCreateSnapshots; i++ { - restic.TestCreateSnapshot(t, repo, testSnapshotTime.Add(time.Duration(i)*time.Second), testDepth, 0) + restic.TestCreateSnapshot(t, repo, testSnapshotTime.Add(time.Duration(i)*time.Second), testDepth) } snapshots, err := loadAllSnapshots(context.TODO(), repo, restic.NewIDSet()) @@ -73,6 +73,6 @@ func BenchmarkTestCreateSnapshot(t *testing.B) { t.ResetTimer() for i := 0; i < t.N; i++ { - restic.TestCreateSnapshot(t, repo, testSnapshotTime.Add(time.Duration(i)*time.Second), testDepth, 0) + restic.TestCreateSnapshot(t, repo, testSnapshotTime.Add(time.Duration(i)*time.Second), testDepth) } } diff --git a/internal/restic/tree.go b/internal/restic/tree.go index 373b36746..3c3e3ab56 100644 --- a/internal/restic/tree.go +++ b/internal/restic/tree.go @@ -5,7 +5,9 @@ import ( "context" "encoding/json" "fmt" + "path" "sort" + "strings" "github.com/restic/restic/internal/errors" @@ -184,3 +186,32 @@ func (builder *TreeJSONBuilder) Finalize() ([]byte, error) { builder.buf = bytes.Buffer{} return buf, nil } + +func FindTreeDirectory(ctx context.Context, repo BlobLoader, id *ID, dir string) (*ID, error) { + if id == nil { + return nil, errors.New("tree id is null") + } + + dirs := strings.Split(path.Clean(dir), "/") + subfolder := "" + + for _, name := range dirs { + if name == "" || name == "." { + continue + } + subfolder = path.Join(subfolder, name) + tree, err := LoadTree(ctx, repo, *id) + if err != nil { + return nil, fmt.Errorf("path %s: %w", subfolder, err) + } + node := tree.Find(name) + if node == nil { + return nil, fmt.Errorf("path %s: not found", subfolder) + } + if node.Type != "dir" || node.Subtree == nil { + return nil, fmt.Errorf("path %s: not a directory", subfolder) + } + id = node.Subtree + } + return id, nil +} diff --git a/internal/restic/tree_test.go b/internal/restic/tree_test.go index fb25ca373..da674eb1c 100644 --- a/internal/restic/tree_test.go +++ b/internal/restic/tree_test.go @@ -210,3 +210,37 @@ func benchmarkLoadTree(t *testing.B, version uint) { rtest.OK(t, err) } } + +func TestFindTreeDirectory(t *testing.T) { + repo := repository.TestRepository(t) + sn := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:08"), 3) + + for _, exp := range []struct { + subfolder string + id restic.ID + err error + }{ + {"", restic.TestParseID("c25199703a67455b34cc0c6e49a8ac8861b268a5dd09dc5b2e31e7380973fc97"), nil}, + {"/", restic.TestParseID("c25199703a67455b34cc0c6e49a8ac8861b268a5dd09dc5b2e31e7380973fc97"), nil}, + {".", restic.TestParseID("c25199703a67455b34cc0c6e49a8ac8861b268a5dd09dc5b2e31e7380973fc97"), nil}, + {"..", restic.ID{}, errors.New("path ..: not found")}, + {"file-1", restic.ID{}, errors.New("path file-1: not a directory")}, + {"dir-21", restic.TestParseID("76172f9dec15d7e4cb98d2993032e99f06b73b2f02ffea3b7cfd9e6b4d762712"), nil}, + {"/dir-21", restic.TestParseID("76172f9dec15d7e4cb98d2993032e99f06b73b2f02ffea3b7cfd9e6b4d762712"), nil}, + {"dir-21/", restic.TestParseID("76172f9dec15d7e4cb98d2993032e99f06b73b2f02ffea3b7cfd9e6b4d762712"), nil}, + {"dir-21/dir-24", restic.TestParseID("74626b3fb2bd4b3e572b81a4059b3e912bcf2a8f69fecd9c187613b7173f13b1"), nil}, + } { + t.Run("", func(t *testing.T) { + id, err := restic.FindTreeDirectory(context.TODO(), repo, sn.Tree, exp.subfolder) + if exp.err == nil { + rtest.OK(t, err) + rtest.Assert(t, exp.id == *id, "unexpected id, expected %v, got %v", exp.id, id) + } else { + rtest.Assert(t, exp.err.Error() == err.Error(), "unexpected err, expected %v, got %v", exp.err, err) + } + }) + } + + _, err := restic.FindTreeDirectory(context.TODO(), repo, nil, "") + rtest.Assert(t, err != nil, "missing error on null tree id") +}