octoleo
/
restic
mirror of https://github.com/octoleo/restic.git
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #4334 from MichaelEischer/snapshot-subtree-syntax 

Add support for snapshot:path syntax
This commit is contained in:
Michael Eischer 2023-07-22 23:59:26 +02:00 committed by GitHub
parent bdaec8fdb8 55b440b520
commit 25ff9fa893
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 291 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
changelog/unreleased/issue-3871 Normal file
View File

@ -0,0 +1,22 @@
Enhancement: Support `<snapshot>:<subfolder>` syntax to select subfolders
Commands like `diff` or `restore` always worked with the full snapshot. This
did not allow comparing only a specific subfolder or only restoring that folder
(`restore --include subfolder` limits the restored files, but still creates the
directories included in `subfolder`).
The commands `diff`, `dump`, `ls`, `restore` now support the
`<snapshot>:<subfolder>` syntax, where `snapshot` is the ID of a snapshot (or
the string `latest`) and `subfolder` is a path within the snapshot. The
commands will then only work with the specified path of the snapshot. The
`subfolder` must be a path to a folder as returned by `ls`.
`restic restore -t target latest:/some/path`
`restic diff 12345678:/some/path 90abcef:/some/path`
For debugging purposes, the `cat` command now supports `cat tree
<snapshot>:<subfolder>` to return the directory metadata for the given
subfolder.
https://github.com/restic/restic/issues/3871
https://github.com/restic/restic/pull/4334

2
cmd/restic/cmd_backup.go
View File

@ -453,7 +453,7 @@ func findParentSnapshot(ctx context.Context, repo restic.Repository, opts Backup
f.Tags = []restic.TagList{opts.Tags.Flatten()}
}
sn, err := f.FindLatest(ctx, repo.Backend(), repo, snName)
sn, _, err := f.FindLatest(ctx, repo.Backend(), repo, snName)
// Snapshot not found is ok if no explicit parent was set
if opts.Parent == "" && errors.Is(err, restic.ErrNoSnapshotFound) {
err = nil

29
cmd/restic/cmd_cat.go
View File

@ -13,7 +13,7 @@ import (
)
var cmdCat = &cobra.Command{
Use: "cat [flags] [pack|blob|snapshot|index|key|masterkey|config|lock] ID",
Use: "cat [flags] [masterkey|config|pack ID|blob ID|snapshot ID|index ID|key ID|lock ID|tree snapshot:subfolder]",
Short: "Print internal objects to stdout",
Long: `
The "cat" command is used to print internal objects to stdout.
@ -55,7 +55,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
tpe := args[0]
var id restic.ID
if tpe != "masterkey" && tpe != "config" && tpe != "snapshot" {
if tpe != "masterkey" && tpe != "config" && tpe != "snapshot" && tpe != "tree" {
id, err = restic.ParseID(args[1])
if err != nil {
return errors.Fatalf("unable to parse ID: %v\n", err)
@ -80,7 +80,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
Println(string(buf))
return nil
case "snapshot":
sn, err := restic.FindSnapshot(ctx, repo.Backend(), repo, args[1])
sn, _, err := restic.FindSnapshot(ctx, repo.Backend(), repo, args[1])
if err != nil {
return errors.Fatalf("could not find snapshot: %v\n", err)
}
@ -165,6 +165,29 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
return errors.Fatal("blob not found")
case "tree":
sn, subfolder, err := restic.FindSnapshot(ctx, repo.Backend(), repo, args[1])
if err != nil {
return errors.Fatalf("could not find snapshot: %v\n", err)
}
err = repo.LoadIndex(ctx)
if err != nil {
return err
}
sn.Tree, err = restic.FindTreeDirectory(ctx, repo, sn.Tree, subfolder)
if err != nil {
return err
}
buf, err := repo.LoadBlob(ctx, restic.TreeBlob, *sn.Tree, nil)
if err != nil {
return err
}
_, err = globalOptions.stdout.Write(buf)
return err
default:
return errors.Fatal("invalid type")
}

22
cmd/restic/cmd_diff.go
View File

@ -54,12 +54,12 @@ func init() {
f.BoolVar(&diffOptions.ShowMetadata, "metadata", false, "print changes in metadata")
}
func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.Repository, desc string) (*restic.Snapshot, error) {
sn, err := restic.FindSnapshot(ctx, be, repo, desc)
func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.Repository, desc string) (*restic.Snapshot, string, error) {
sn, subfolder, err := restic.FindSnapshot(ctx, be, repo, desc)
if err != nil {
return nil, errors.Fatal(err.Error())
return nil, "", errors.Fatal(err.Error())
}
return sn, err
return sn, subfolder, err
}
// Comparer collects all things needed to compare two snapshots.
@ -346,12 +346,12 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []
if err != nil {
return err
}
sn1, err := loadSnapshot(ctx, be, repo, args[0])
sn1, subfolder1, err := loadSnapshot(ctx, be, repo, args[0])
if err != nil {
return err
}
sn2, err := loadSnapshot(ctx, be, repo, args[1])
sn2, subfolder2, err := loadSnapshot(ctx, be, repo, args[1])
if err != nil {
return err
}
@ -372,6 +372,16 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []
return errors.Errorf("snapshot %v has nil tree", sn2.ID().Str())
}
sn1.Tree, err = restic.FindTreeDirectory(ctx, repo, sn1.Tree, subfolder1)
if err != nil {
return err
}
sn2.Tree, err = restic.FindTreeDirectory(ctx, repo, sn2.Tree, subfolder2)
if err != nil {
return err
}
c := &Comparer{
repo: repo,
opts: diffOptions,

7
cmd/restic/cmd_dump.go
View File

@ -139,7 +139,7 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args []
}
}
sn, err := (&restic.SnapshotFilter{
sn, subfolder, err := (&restic.SnapshotFilter{
Hosts: opts.Hosts,
Paths: opts.Paths,
Tags: opts.Tags,
@ -153,6 +153,11 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args []
return err
}
sn.Tree, err = restic.FindTreeDirectory(ctx, repo, sn.Tree, subfolder)
if err != nil {
return err
}
tree, err := restic.LoadTree(ctx, repo, *sn.Tree)
if err != nil {
return errors.Fatalf("loading tree for snapshot %q failed: %v", snapshotIDString, err)

7
cmd/restic/cmd_ls.go
View File

@ -212,7 +212,7 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
}
}
sn, err := (&restic.SnapshotFilter{
sn, subfolder, err := (&restic.SnapshotFilter{
Hosts: opts.Hosts,
Paths: opts.Paths,
Tags: opts.Tags,
@ -221,6 +221,11 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
return err
}
sn.Tree, err = restic.FindTreeDirectory(ctx, repo, sn.Tree, subfolder)
if err != nil {
return err
}
printSnapshot(sn)
err = walker.Walk(ctx, repo, *sn.Tree, nil, func(_ restic.ID, nodepath string, node *restic.Node, err error) (bool, error) {

7
cmd/restic/cmd_restore.go
View File

@ -161,7 +161,7 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
}
}
sn, err := (&restic.SnapshotFilter{
sn, subfolder, err := (&restic.SnapshotFilter{
Hosts: opts.Hosts,
Paths: opts.Paths,
Tags: opts.Tags,
@ -175,6 +175,11 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
return err
}
sn.Tree, err = restic.FindTreeDirectory(ctx, repo, sn.Tree, subfolder)
if err != nil {
return err
}
msg := ui.NewMessage(term, gopts.verbosity)
var printer restoreui.ProgressPrinter
if gopts.JSON {

9
doc/040_backup.rst
View File

@ -451,6 +451,15 @@ and displays a small statistic, just pass the command two snapshot IDs:
Added: 16.403 MiB
Removed: 16.402 MiB
To only compare files in specific subfolders, you can use the ``<snapshot>:<subfolder>``
syntax, where ``snapshot`` is the ID of a snapshot (or the string ``latest``) and ``subfolder``
is a path within the snapshot. For example, to only compare files in the ``/restic``
folder, you could use the following command:
.. code-block:: console
$ restic -r /srv/restic-repo diff 5845b002:/restic 2ab627a6:/restic
Backing up special items and metadata
*************************************

20
doc/050_restore.rst
View File

@ -48,6 +48,18 @@ files in the snapshot. For example, to restore a single file:
This will restore the file ``foo`` to ``/tmp/restore-work/work/foo``.
To only restore a specific subfolder, you can use the ``<snapshot>:<subfolder>``
syntax, where ``snapshot`` is the ID of a snapshot (or the string ``latest``)
and ``subfolder`` is a path within the snapshot.
.. code-block:: console
$ restic -r /srv/restic-repo restore 79766175:/work --target /tmp/restore-work --include /foo
enter password for repository:
restoring <Snapshot of [/home/user/work] at 2015-05-08 21:40:19.884408621 +0200 CEST> to /tmp/restore-work
This will restore the file ``foo`` to ``/tmp/restore-work/foo``.
You can use the command ``restic ls latest`` or ``restic find foo`` to find the
path to the file within the snapshot. This path you can then pass to
``--include`` in verbatim to only restore the single file or directory.
@ -151,8 +163,14 @@ output the contents in the tar (default) or zip format:
.. code-block:: console
$ restic -r /srv/restic-repo dump latest /home/other/work > restore.tar
.. code-block:: console
$ restic -r /srv/restic-repo dump -a zip latest /home/other/work > restore.zip
The folder content is then contained at ``/home/other/work`` within the archive.
To include the folder content at the root of the archive, you can use the ``<snapshot>:<subfolder>`` syntax:
.. code-block:: console
$ restic -r /srv/restic-repo dump latest:/home/other/work / > restore.tar

4
internal/fuse/fuse_test.go
View File

@ -73,7 +73,7 @@ func TestFuseFile(t *testing.T) {
timestamp, err := time.Parse(time.RFC3339, "2017-01-24T10:42:56+01:00")
rtest.OK(t, err)
restic.TestCreateSnapshot(t, repo, timestamp, 2, 0.1)
restic.TestCreateSnapshot(t, repo, timestamp, 2)
sn := loadFirstSnapshot(t, repo)
tree := loadTree(t, repo, *sn.Tree)
@ -180,7 +180,7 @@ func TestFuseDir(t *testing.T) {
// Test top-level directories for their UID and GID.
func TestTopUIDGID(t *testing.T) {
repo := repository.TestRepository(t)
restic.TestCreateSnapshot(t, repo, time.Unix(1460289341, 207401672), 0, 0)
restic.TestCreateSnapshot(t, repo, time.Unix(1460289341, 207401672), 0)
testTopUIDGID(t, Config{}, repo, uint32(os.Getuid()), uint32(os.Getgid()))
testTopUIDGID(t, Config{OwnerIsRoot: true}, repo, 0, 0)

6
internal/index/master_index_test.go
View File

@ -340,11 +340,11 @@ var (
depth = 3
)
func createFilledRepo(t testing.TB, snapshots int, dup float32, version uint) restic.Repository {
func createFilledRepo(t testing.TB, snapshots int, version uint) restic.Repository {
repo := repository.TestRepositoryWithVersion(t, version)
for i := 0; i < snapshots; i++ {
restic.TestCreateSnapshot(t, repo, snapshotTime.Add(time.Duration(i)*time.Second), depth, dup)
restic.TestCreateSnapshot(t, repo, snapshotTime.Add(time.Duration(i)*time.Second), depth)
}
return repo
}
@ -354,7 +354,7 @@ func TestIndexSave(t *testing.T) {
}
func testIndexSave(t *testing.T, version uint) {
repo := createFilledRepo(t, 3, 0, version)
repo := createFilledRepo(t, 3, version)
err := repo.LoadIndex(context.TODO())
if err != nil {

8
internal/restic/find_test.go
View File

@ -88,7 +88,7 @@ func TestFindUsedBlobs(t *testing.T) {
var snapshots []*restic.Snapshot
for i := 0; i < findTestSnapshots; i++ {
sn := restic.TestCreateSnapshot(t, repo, findTestTime.Add(time.Duration(i)*time.Second), findTestDepth, 0)
sn := restic.TestCreateSnapshot(t, repo, findTestTime.Add(time.Duration(i)*time.Second), findTestDepth)
t.Logf("snapshot %v saved, tree %v", sn.ID().Str(), sn.Tree.Str())
snapshots = append(snapshots, sn)
}
@ -131,7 +131,7 @@ func TestMultiFindUsedBlobs(t *testing.T) {
var snapshotTrees restic.IDs
for i := 0; i < findTestSnapshots; i++ {
sn := restic.TestCreateSnapshot(t, repo, findTestTime.Add(time.Duration(i)*time.Second), findTestDepth, 0)
sn := restic.TestCreateSnapshot(t, repo, findTestTime.Add(time.Duration(i)*time.Second), findTestDepth)
t.Logf("snapshot %v saved, tree %v", sn.ID().Str(), sn.Tree.Str())
snapshotTrees = append(snapshotTrees, *sn.Tree)
}
@ -177,7 +177,7 @@ func (r ForbiddenRepo) Connections() uint {
func TestFindUsedBlobsSkipsSeenBlobs(t *testing.T) {
repo := repository.TestRepository(t)
snapshot := restic.TestCreateSnapshot(t, repo, findTestTime, findTestDepth, 0)
snapshot := restic.TestCreateSnapshot(t, repo, findTestTime, findTestDepth)
t.Logf("snapshot %v saved, tree %v", snapshot.ID().Str(), snapshot.Tree.Str())
usedBlobs := restic.NewBlobSet()
@ -195,7 +195,7 @@ func TestFindUsedBlobsSkipsSeenBlobs(t *testing.T) {
func BenchmarkFindUsedBlobs(b *testing.B) {
repo := repository.TestRepository(b)
sn := restic.TestCreateSnapshot(b, repo, findTestTime, findTestDepth, 0)
sn := restic.TestCreateSnapshot(b, repo, findTestTime, findTestDepth)
b.ResetTimer()

33
internal/restic/snapshot_find.go
View File

@ -4,6 +4,7 @@ import (
"context"
"fmt"
"path/filepath"
"strings"
"time"
"github.com/restic/restic/internal/errors"
@ -82,37 +83,48 @@ func (f *SnapshotFilter) findLatest(ctx context.Context, be Lister, loader Loade
return latest, nil
}
func splitSnapshotID(s string) (id, subfolder string) {
id, subfolder, _ = strings.Cut(s, ":")
return
}
// FindSnapshot takes a string and tries to find a snapshot whose ID matches
// the string as closely as possible.
func FindSnapshot(ctx context.Context, be Lister, loader LoaderUnpacked, s string) (*Snapshot, error) {
func FindSnapshot(ctx context.Context, be Lister, loader LoaderUnpacked, s string) (*Snapshot, string, error) {
s, subfolder := splitSnapshotID(s)
// no need to list snapshots if `s` is already a full id
id, err := ParseID(s)
if err != nil {
// find snapshot id with prefix
id, err = Find(ctx, be, SnapshotFile, s)
if err != nil {
return nil, err
return nil, "", err
}
}
return LoadSnapshot(ctx, loader, id)
sn, err := LoadSnapshot(ctx, loader, id)
return sn, subfolder, err
}
// FindLatest returns either the latest of a filtered list of all snapshots
// or a snapshot specified by `snapshotID`.
func (f *SnapshotFilter) FindLatest(ctx context.Context, be Lister, loader LoaderUnpacked, snapshotID string) (*Snapshot, error) {
if snapshotID == "latest" {
func (f *SnapshotFilter) FindLatest(ctx context.Context, be Lister, loader LoaderUnpacked, snapshotID string) (*Snapshot, string, error) {
id, subfolder := splitSnapshotID(snapshotID)
if id == "latest" {
sn, err := f.findLatest(ctx, be, loader)
if err == ErrNoSnapshotFound {
err = fmt.Errorf("snapshot filter (Paths:%v Tags:%v Hosts:%v): %w",
f.Paths, f.Tags, f.Hosts, err)
}
return sn, err
return sn, subfolder, err
}
return FindSnapshot(ctx, be, loader, snapshotID)
}
type SnapshotFindCb func(string, *Snapshot, error) error
var ErrInvalidSnapshotSyntax = errors.New("<snapshot>:<subfolder> syntax not allowed")
// FindAll yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots.
func (f *SnapshotFilter) FindAll(ctx context.Context, be Lister, loader LoaderUnpacked, snapshotIDs []string, fn SnapshotFindCb) error {
if len(snapshotIDs) != 0 {
@ -138,9 +150,14 @@ func (f *SnapshotFilter) FindAll(ctx context.Context, be Lister, loader LoaderUn
if sn != nil {
ids.Insert(*sn.ID())
}
} else if strings.HasPrefix(s, "latest:") {
err = ErrInvalidSnapshotSyntax
} else {
sn, err = FindSnapshot(ctx, be, loader, s)
if err == nil {
var subfolder string
sn, subfolder, err = FindSnapshot(ctx, be, loader, s)
if err == nil && subfolder != "" {
err = ErrInvalidSnapshotSyntax
} else if err == nil {
if ids.Has(*sn.ID()) {
continue
}

62
internal/restic/snapshot_find_test.go
View File

@ -6,16 +6,17 @@ import (
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test"
)
func TestFindLatestSnapshot(t *testing.T) {
repo := repository.TestRepository(t)
restic.TestCreateSnapshot(t, repo, parseTimeUTC("2015-05-05 05:05:05"), 1, 0)
restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1, 0)
latestSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2019-09-09 09:09:09"), 1, 0)
restic.TestCreateSnapshot(t, repo, parseTimeUTC("2015-05-05 05:05:05"), 1)
restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1)
latestSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2019-09-09 09:09:09"), 1)
f := restic.SnapshotFilter{Hosts: []string{"foo"}}
sn, err := f.FindLatest(context.TODO(), repo.Backend(), repo, "latest")
sn, _, err := f.FindLatest(context.TODO(), repo.Backend(), repo, "latest")
if err != nil {
t.Fatalf("FindLatest returned error: %v", err)
}
@ -27,11 +28,11 @@ func TestFindLatestSnapshot(t *testing.T) {
func TestFindLatestSnapshotWithMaxTimestamp(t *testing.T) {
repo := repository.TestRepository(t)
restic.TestCreateSnapshot(t, repo, parseTimeUTC("2015-05-05 05:05:05"), 1, 0)
desiredSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1, 0)
restic.TestCreateSnapshot(t, repo, parseTimeUTC("2019-09-09 09:09:09"), 1, 0)
restic.TestCreateSnapshot(t, repo, parseTimeUTC("2015-05-05 05:05:05"), 1)
desiredSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1)
restic.TestCreateSnapshot(t, repo, parseTimeUTC("2019-09-09 09:09:09"), 1)
sn, err := (&restic.SnapshotFilter{
sn, _, err := (&restic.SnapshotFilter{
Hosts: []string{"foo"},
TimestampLimit: parseTimeUTC("2018-08-08 08:08:08"),
}).FindLatest(context.TODO(), repo.Backend(), repo, "latest")
@ -43,3 +44,48 @@ func TestFindLatestSnapshotWithMaxTimestamp(t *testing.T) {
t.Errorf("FindLatest returned wrong snapshot ID: %v", *sn.ID())
}
}
func TestFindLatestWithSubpath(t *testing.T) {
repo := repository.TestRepository(t)
restic.TestCreateSnapshot(t, repo, parseTimeUTC("2015-05-05 05:05:05"), 1)
desiredSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1)
for _, exp := range []struct {
query string
subfolder string
}{
{"latest", ""},
{"latest:subfolder", "subfolder"},
{desiredSnapshot.ID().Str(), ""},
{desiredSnapshot.ID().Str() + ":subfolder", "subfolder"},
{desiredSnapshot.ID().String(), ""},
{desiredSnapshot.ID().String() + ":subfolder", "subfolder"},
} {
t.Run("", func(t *testing.T) {
sn, subfolder, err := (&restic.SnapshotFilter{}).FindLatest(context.TODO(), repo.Backend(), repo, exp.query)
if err != nil {
t.Fatalf("FindLatest returned error: %v", err)
}
test.Assert(t, *sn.ID() == *desiredSnapshot.ID(), "FindLatest returned wrong snapshot ID: %v", *sn.ID())
test.Assert(t, subfolder == exp.subfolder, "FindLatest returned wrong path in snapshot: %v", subfolder)
})
}
}
func TestFindAllSubpathError(t *testing.T) {
repo := repository.TestRepository(t)
desiredSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1)
count := 0
test.OK(t, (&restic.SnapshotFilter{}).FindAll(context.TODO(), repo.Backend(), repo,
[]string{"latest:subfolder", desiredSnapshot.ID().Str() + ":subfolder"},
func(id string, sn *restic.Snapshot, err error) error {
if err == restic.ErrInvalidSnapshotSyntax {
count++
return nil
}
return err
}))
test.Assert(t, count == 2, "unexpected number of subfolder errors: %v, wanted %v", count, 2)
}

6
internal/restic/testdata/used_blobs_snapshot0 vendored
View File

@ -1,7 +1,8 @@
{"ID":"05bddd650a800f83f7c0d844cecb1e02f99ce962df5652a53842be50386078e1","Type":"data"}
{"ID":"087040b12f129e89e4eab2b86aa14467404366a17a6082efb0d11fa7e2f9f58e","Type":"data"}
{"ID":"08a650e4d7575177ddeabf6a96896b76fa7e621aa3dd75e77293f22ce6c0c420","Type":"tree"}
{"ID":"1e0f0e5799b9d711e07883050366c7eee6b7481c0d884694093149f6c4e9789a","Type":"data"}
{"ID":"229eac8e4e6c2e8d7b1d9f9627ab5d1a59cb17c5744c1e3634215116e7a92e7d","Type":"tree"}
{"ID":"435b9207cd489b41a7d119e0d75eab2a861e2b3c8d4d12ac51873ff76be0cf73","Type":"tree"}
{"ID":"4719f8a039f5b745e16cf90e5b84c9255c290d500da716f7dd25909cdabb85b6","Type":"data"}
{"ID":"4e352975938a29711c3003c498185972235af261a6cf8cf700a8a6ee4f914b05","Type":"data"}
{"ID":"606772eacb7fe1a79267088dcadd13431914854faf1d39d47fe99a26b9fecdcb","Type":"data"}
@ -9,7 +10,6 @@
{"ID":"72b6eb0fd0d87e00392f8b91efc1a4c3f7f5c0c76f861b38aea054bc9d43463b","Type":"data"}
{"ID":"77ab53b52e0cf13b300d1b7f6dac89287c8d86769d85e8a273311006ce6359be","Type":"data"}
{"ID":"99dab094430d3c1be22c801a6ad7364d490a8d2ce3f9dfa3d2677431446925f4","Type":"data"}
{"ID":"9face1b278a49ef8819fbc1855ce573a85077453bbf6683488cad7767c3a38a7","Type":"tree"}
{"ID":"a4c97189465344038584e76c965dd59100eaed051db1fa5ba0e143897e2c87f1","Type":"data"}
{"ID":"a69c8621776ca8bb34c6c90e5ad811ddc8e2e5cfd6bb0cec5e75cca70e0b9ade","Type":"data"}
{"ID":"b11f4dd9d2722b3325186f57cd13a71a3af7791118477f355b49d101104e4c22","Type":"data"}
@ -19,5 +19,5 @@
{"ID":"b9e634143719742fe77feed78b61f09573d59d2efa23d6d54afe6c159d220503","Type":"data"}
{"ID":"ca896fc9ebf95fcffd7c768b07b92110b21e332a47fef7e382bf15363b0ece1a","Type":"data"}
{"ID":"e6fe3512ea23a4ebf040d30958c669f7ffe724400f155a756467a9f3cafc27c5","Type":"data"}
{"ID":"e96774ac5abfbb59940939f614d65a397fb7b5abba76c29bfe14479c6616eea0","Type":"tree"}
{"ID":"ed00928ce97ac5acd27c862d9097e606536e9063af1c47481257811f66260f3a","Type":"data"}
{"ID":"fb62dd9093c4958b019b90e591b2d36320ff381a24bdc9c5db3b8960ff94d174","Type":"tree"}

4
internal/restic/testdata/used_blobs_snapshot1 vendored
View File

@ -1,4 +1,3 @@
{"ID":"04ff190aea26dae65ba4c782926cdfb700b484a8b802a5ffd58e3fadcf70b797","Type":"tree"}
{"ID":"05bddd650a800f83f7c0d844cecb1e02f99ce962df5652a53842be50386078e1","Type":"data"}
{"ID":"18dcaa1a676823c909aafabbb909652591915eebdde4f9a65cee955157583494","Type":"data"}
{"ID":"4719f8a039f5b745e16cf90e5b84c9255c290d500da716f7dd25909cdabb85b6","Type":"data"}
@ -8,8 +7,9 @@
{"ID":"a69c8621776ca8bb34c6c90e5ad811ddc8e2e5cfd6bb0cec5e75cca70e0b9ade","Type":"data"}
{"ID":"b1f2ae9d748035e5bd9a87f2579405166d150c6560d8919496f02855e1c36cf9","Type":"data"}
{"ID":"b9e634143719742fe77feed78b61f09573d59d2efa23d6d54afe6c159d220503","Type":"data"}
{"ID":"bdd5a029dd295e5998c518022547d185794e72d8f8c38709a638c5841284daef","Type":"tree"}
{"ID":"ca896fc9ebf95fcffd7c768b07b92110b21e332a47fef7e382bf15363b0ece1a","Type":"data"}
{"ID":"cc4cab5b20a3a88995f8cdb8b0698d67a32dbc5b54487f03cb612c30a626af39","Type":"data"}
{"ID":"e6fe3512ea23a4ebf040d30958c669f7ffe724400f155a756467a9f3cafc27c5","Type":"data"}
{"ID":"e9f3c4fe78e903cba60d310a9668c42232c8274b3f29b5ecebb6ff1aaeabd7e3","Type":"tree"}
{"ID":"ed00928ce97ac5acd27c862d9097e606536e9063af1c47481257811f66260f3a","Type":"data"}
{"ID":"ff58f76c2313e68aa9aaaece855183855ac4ff682910404c2ae33dc999ebaca2","Type":"tree"}

8
internal/restic/testdata/used_blobs_snapshot2 vendored
View File

@ -1,6 +1,7 @@
{"ID":"05bddd650a800f83f7c0d844cecb1e02f99ce962df5652a53842be50386078e1","Type":"data"}
{"ID":"087040b12f129e89e4eab2b86aa14467404366a17a6082efb0d11fa7e2f9f58e","Type":"data"}
{"ID":"0b88f99abc5ac71c54b3e8263c52ecb7d8903462779afdb3c8176ec5c4bb04fb","Type":"data"}
{"ID":"0e1a817fca83f569d1733b11eba14b6c9b176e41bca3644eed8b29cb907d84d3","Type":"tree"}
{"ID":"1e0f0e5799b9d711e07883050366c7eee6b7481c0d884694093149f6c4e9789a","Type":"data"}
{"ID":"27917462f89cecae77a4c8fb65a094b9b75a917f13794c628b1640b17f4c4981","Type":"data"}
{"ID":"32745e4b26a5883ecec272c9fbfe7f3c9835c9ab41c9a2baa4d06f319697a0bd","Type":"data"}
@ -10,15 +11,14 @@
{"ID":"6b5fd3a9baf615489c82a99a71f9917bf9a2d82d5f640d7f47d175412c4b8d19","Type":"data"}
{"ID":"95c97192efa810ccb1cee112238dca28673fbffce205d75ce8cc990a31005a51","Type":"data"}
{"ID":"99dab094430d3c1be22c801a6ad7364d490a8d2ce3f9dfa3d2677431446925f4","Type":"data"}
{"ID":"9face1b278a49ef8819fbc1855ce573a85077453bbf6683488cad7767c3a38a7","Type":"tree"}
{"ID":"a4c97189465344038584e76c965dd59100eaed051db1fa5ba0e143897e2c87f1","Type":"data"}
{"ID":"a5f2ffcd54e28e2ef3089c35b72aafda66161125e23dad581087ccd050c111c3","Type":"tree"}
{"ID":"a69c8621776ca8bb34c6c90e5ad811ddc8e2e5cfd6bb0cec5e75cca70e0b9ade","Type":"data"}
{"ID":"ab5205525de94e564e3a00f634fcf9ebc397debd567734c68da7b406e612aae4","Type":"tree"}
{"ID":"b6a7e8d2aa717e0a6bd68abab512c6b566074b5a6ca2edf4cd446edc5857d732","Type":"data"}
{"ID":"be2055b7125ccf824fcfa8faa4eb3985119012bac26643944eee46218e71306e","Type":"tree"}
{"ID":"bad84ed273c5fbfb40aa839a171675b7f16f5e67f3eaf4448730caa0ee27297c","Type":"tree"}
{"ID":"bfc2fdb527b0c9f66bbb8d4ff1c44023cc2414efcc7f0831c10debab06bb4388","Type":"tree"}
{"ID":"ca896fc9ebf95fcffd7c768b07b92110b21e332a47fef7e382bf15363b0ece1a","Type":"data"}
{"ID":"d1d3137eb08de6d8c5d9f44788c45a9fea9bb082e173bed29a0945b3347f2661","Type":"tree"}
{"ID":"e6fe3512ea23a4ebf040d30958c669f7ffe724400f155a756467a9f3cafc27c5","Type":"data"}
{"ID":"ed00928ce97ac5acd27c862d9097e606536e9063af1c47481257811f66260f3a","Type":"data"}
{"ID":"f3cd67d9c14d2a81663d63522ab914e465b021a3b65e2f1ea6caf7478f2ec139","Type":"data"}
{"ID":"fb62dd9093c4958b019b90e591b2d36320ff381a24bdc9c5db3b8960ff94d174","Type":"tree"}

68
internal/restic/testing.go
View File

@ -2,7 +2,6 @@ package restic
import (
"context"
"encoding/json"
"fmt"
"io"
"math/rand"
@ -19,12 +18,11 @@ func fakeFile(seed, size int64) io.Reader {
}
type fakeFileSystem struct {
t testing.TB
repo Repository
duplication float32
buf []byte
chunker *chunker.Chunker
rand *rand.Rand
t testing.TB
repo Repository
buf []byte
chunker *chunker.Chunker
rand *rand.Rand
}
// saveFile reads from rd and saves the blobs in the repository. The list of
@ -51,13 +49,9 @@ func (fs *fakeFileSystem) saveFile(ctx context.Context, rd io.Reader) (blobs IDs
fs.t.Fatalf("unable to save chunk in repo: %v", err)
}
id := Hash(chunk.Data)
if !fs.blobIsKnown(BlobHandle{ID: id, Type: DataBlob}) {
_, _, _, err := fs.repo.SaveBlob(ctx, DataBlob, chunk.Data, id, true)
if err != nil {
fs.t.Fatalf("error saving chunk: %v", err)
}
id, _, _, err := fs.repo.SaveBlob(ctx, DataBlob, chunk.Data, ID{}, false)
if err != nil {
fs.t.Fatalf("error saving chunk: %v", err)
}
blobs = append(blobs, id)
@ -72,30 +66,6 @@ const (
maxNodes = 15
)
func (fs *fakeFileSystem) treeIsKnown(tree *Tree) (bool, []byte, ID) {
data, err := json.Marshal(tree)
if err != nil {
fs.t.Fatalf("json.Marshal(tree) returned error: %v", err)
return false, nil, ID{}
}
data = append(data, '\n')
id := Hash(data)
return fs.blobIsKnown(BlobHandle{ID: id, Type: TreeBlob}), data, id
}
func (fs *fakeFileSystem) blobIsKnown(bh BlobHandle) bool {
if fs.rand.Float32() < fs.duplication {
return false
}
if fs.repo.Index().Has(bh) {
return true
}
return false
}
// saveTree saves a tree of fake files in the repo and returns the ID.
func (fs *fakeFileSystem) saveTree(ctx context.Context, seed int64, depth int) ID {
rnd := rand.NewSource(seed)
@ -134,16 +104,12 @@ func (fs *fakeFileSystem) saveTree(ctx context.Context, seed int64, depth int) I
tree.Nodes = append(tree.Nodes, node)
}
known, buf, id := fs.treeIsKnown(&tree)
if known {
return id
}
tree.Sort()
_, _, _, err := fs.repo.SaveBlob(ctx, TreeBlob, buf, id, false)
id, err := SaveTree(ctx, fs.repo, &tree)
if err != nil {
fs.t.Fatal(err)
fs.t.Fatalf("SaveTree returned error: %v", err)
}
return id
}
@ -152,22 +118,20 @@ func (fs *fakeFileSystem) saveTree(ctx context.Context, seed int64, depth int) I
// also used as the snapshot's timestamp. The tree's depth can be specified
// with the parameter depth. The parameter duplication is a probability that
// the same blob will saved again.
func TestCreateSnapshot(t testing.TB, repo Repository, at time.Time, depth int, duplication float32) *Snapshot {
func TestCreateSnapshot(t testing.TB, repo Repository, at time.Time, depth int) *Snapshot {
seed := at.Unix()
t.Logf("create fake snapshot at %s with seed %d", at, seed)
fakedir := fmt.Sprintf("fakedir-at-%v", at.Format("2006-01-02 15:04:05"))
snapshot, err := NewSnapshot([]string{fakedir}, []string{"test"}, "foo", time.Now())
snapshot, err := NewSnapshot([]string{fakedir}, []string{"test"}, "foo", at)
if err != nil {
t.Fatal(err)
}
snapshot.Time = at
fs := fakeFileSystem{
t: t,
repo: repo,
duplication: duplication,
rand: rand.New(rand.NewSource(seed)),
t: t,
repo: repo,
rand: rand.New(rand.NewSource(seed)),
}
var wg errgroup.Group

4
internal/restic/testing_test.go
View File

@ -39,7 +39,7 @@ func loadAllSnapshots(ctx context.Context, repo restic.Repository, excludeIDs re
func TestCreateSnapshot(t *testing.T) {
repo := repository.TestRepository(t)
for i := 0; i < testCreateSnapshots; i++ {
restic.TestCreateSnapshot(t, repo, testSnapshotTime.Add(time.Duration(i)*time.Second), testDepth, 0)
restic.TestCreateSnapshot(t, repo, testSnapshotTime.Add(time.Duration(i)*time.Second), testDepth)
}
snapshots, err := loadAllSnapshots(context.TODO(), repo, restic.NewIDSet())
@ -73,6 +73,6 @@ func BenchmarkTestCreateSnapshot(t *testing.B) {
t.ResetTimer()
for i := 0; i < t.N; i++ {
restic.TestCreateSnapshot(t, repo, testSnapshotTime.Add(time.Duration(i)*time.Second), testDepth, 0)
restic.TestCreateSnapshot(t, repo, testSnapshotTime.Add(time.Duration(i)*time.Second), testDepth)
}
}

31
internal/restic/tree.go
View File

@ -5,7 +5,9 @@ import (
"context"
"encoding/json"
"fmt"
"path"
"sort"
"strings"
"github.com/restic/restic/internal/errors"
@ -184,3 +186,32 @@ func (builder *TreeJSONBuilder) Finalize() ([]byte, error) {
builder.buf = bytes.Buffer{}
return buf, nil
}
func FindTreeDirectory(ctx context.Context, repo BlobLoader, id *ID, dir string) (*ID, error) {
if id == nil {
return nil, errors.New("tree id is null")
}
dirs := strings.Split(path.Clean(dir), "/")
subfolder := ""
for _, name := range dirs {
if name == "" || name == "." {
continue
}
subfolder = path.Join(subfolder, name)
tree, err := LoadTree(ctx, repo, *id)
if err != nil {
return nil, fmt.Errorf("path %s: %w", subfolder, err)
}
node := tree.Find(name)
if node == nil {
return nil, fmt.Errorf("path %s: not found", subfolder)
}
if node.Type != "dir" || node.Subtree == nil {
return nil, fmt.Errorf("path %s: not a directory", subfolder)
}
id = node.Subtree
}
return id, nil
}

34
internal/restic/tree_test.go
View File

@ -210,3 +210,37 @@ func benchmarkLoadTree(t *testing.B, version uint) {
rtest.OK(t, err)
}
}
func TestFindTreeDirectory(t *testing.T) {
repo := repository.TestRepository(t)
sn := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:08"), 3)
for _, exp := range []struct {
subfolder string
id restic.ID
err error
}{
{"", restic.TestParseID("c25199703a67455b34cc0c6e49a8ac8861b268a5dd09dc5b2e31e7380973fc97"), nil},
{"/", restic.TestParseID("c25199703a67455b34cc0c6e49a8ac8861b268a5dd09dc5b2e31e7380973fc97"), nil},
{".", restic.TestParseID("c25199703a67455b34cc0c6e49a8ac8861b268a5dd09dc5b2e31e7380973fc97"), nil},
{"..", restic.ID{}, errors.New("path ..: not found")},
{"file-1", restic.ID{}, errors.New("path file-1: not a directory")},
{"dir-21", restic.TestParseID("76172f9dec15d7e4cb98d2993032e99f06b73b2f02ffea3b7cfd9e6b4d762712"), nil},
{"/dir-21", restic.TestParseID("76172f9dec15d7e4cb98d2993032e99f06b73b2f02ffea3b7cfd9e6b4d762712"), nil},
{"dir-21/", restic.TestParseID("76172f9dec15d7e4cb98d2993032e99f06b73b2f02ffea3b7cfd9e6b4d762712"), nil},
{"dir-21/dir-24", restic.TestParseID("74626b3fb2bd4b3e572b81a4059b3e912bcf2a8f69fecd9c187613b7173f13b1"), nil},
} {
t.Run("", func(t *testing.T) {
id, err := restic.FindTreeDirectory(context.TODO(), repo, sn.Tree, exp.subfolder)
if exp.err == nil {
rtest.OK(t, err)
rtest.Assert(t, exp.id == *id, "unexpected id, expected %v, got %v", exp.id, id)
} else {
rtest.Assert(t, exp.err.Error() == err.Error(), "unexpected err, expected %v, got %v", exp.err, err)
}
})
}
_, err := restic.FindTreeDirectory(context.TODO(), repo, nil, "")
rtest.Assert(t, err != nil, "missing error on null tree id")
}