mirror of
https://github.com/octoleo/restic.git
synced 2024-06-05 02:20:50 +00:00
2b88cd6eab
SaveTree did not use the TreeSaver but rather managed the tree collection and upload itself. This prevents using the parallelism offered by the TreeSaver and duplicates all related code. Using the TreeSaver can provide some speed-ups as all steps within the backup tree now rely on FutureNodes. This can be especially relevant for backups with large amounts of explicitly specified files. The main difference between SaveTree and SaveDir is, that only the former can save tree blobs in which nodes have a different name than the actual file on disk. This is the result of resolving name conflicts between multiple files with the same name. The filename that must be used within the snapshot is now passed directly to restic.NodeFromFileInfo. This ensures that a FutureNode already contains the correct filename.
131 lines
2.6 KiB
Go
131 lines
2.6 KiB
Go
package archiver
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"runtime"
|
|
"testing"
|
|
|
|
"github.com/restic/restic/internal/errors"
|
|
"github.com/restic/restic/internal/restic"
|
|
"golang.org/x/sync/errgroup"
|
|
)
|
|
|
|
func newFutureBlobWithResponse() FutureBlob {
|
|
ch := make(chan SaveBlobResponse, 1)
|
|
ch <- SaveBlobResponse{
|
|
id: restic.NewRandomID(),
|
|
known: false,
|
|
length: 123,
|
|
sizeInRepo: 123,
|
|
}
|
|
return FutureBlob{ch: ch}
|
|
}
|
|
|
|
func TestTreeSaver(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
|
|
wg, ctx := errgroup.WithContext(ctx)
|
|
|
|
saveFn := func(ctx context.Context, t restic.BlobType, buf *Buffer) FutureBlob {
|
|
return newFutureBlobWithResponse()
|
|
}
|
|
errFn := func(snPath string, err error) error {
|
|
return err
|
|
}
|
|
|
|
b := NewTreeSaver(ctx, wg, uint(runtime.NumCPU()), saveFn, errFn)
|
|
|
|
var results []FutureNode
|
|
|
|
for i := 0; i < 20; i++ {
|
|
node := &restic.Node{
|
|
Name: fmt.Sprintf("file-%d", i),
|
|
}
|
|
|
|
fb := b.Save(ctx, join("/", node.Name), node.Name, node, nil, nil)
|
|
results = append(results, fb)
|
|
}
|
|
|
|
for _, tree := range results {
|
|
tree.take(ctx)
|
|
}
|
|
|
|
b.TriggerShutdown()
|
|
|
|
err := wg.Wait()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestTreeSaverError(t *testing.T) {
|
|
var tests = []struct {
|
|
trees int
|
|
failAt int
|
|
}{
|
|
{1, 1},
|
|
{20, 2},
|
|
{20, 5},
|
|
{20, 15},
|
|
{200, 150},
|
|
}
|
|
|
|
errTest := errors.New("test error")
|
|
|
|
for _, test := range tests {
|
|
t.Run("", func(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
|
|
wg, ctx := errgroup.WithContext(ctx)
|
|
|
|
saveFn := func(ctx context.Context, tpe restic.BlobType, buf *Buffer) FutureBlob {
|
|
return newFutureBlobWithResponse()
|
|
}
|
|
errFn := func(snPath string, err error) error {
|
|
return err
|
|
}
|
|
|
|
b := NewTreeSaver(ctx, wg, uint(runtime.NumCPU()), saveFn, errFn)
|
|
|
|
var results []FutureNode
|
|
|
|
for i := 0; i < test.trees; i++ {
|
|
node := &restic.Node{
|
|
Name: fmt.Sprintf("file-%d", i),
|
|
}
|
|
nodes := []FutureNode{
|
|
newFutureNodeWithResult(futureNodeResult{node: &restic.Node{
|
|
Name: fmt.Sprintf("child-%d", i),
|
|
}}),
|
|
}
|
|
if (i + 1) == test.failAt {
|
|
nodes = append(nodes, newFutureNodeWithResult(futureNodeResult{
|
|
err: errTest,
|
|
}))
|
|
}
|
|
|
|
fb := b.Save(ctx, join("/", node.Name), node.Name, node, nodes, nil)
|
|
results = append(results, fb)
|
|
}
|
|
|
|
for _, tree := range results {
|
|
tree.take(ctx)
|
|
}
|
|
|
|
b.TriggerShutdown()
|
|
|
|
err := wg.Wait()
|
|
if err == nil {
|
|
t.Errorf("expected error not found")
|
|
}
|
|
|
|
if err != errTest {
|
|
t.Fatalf("unexpected error found: %v", err)
|
|
}
|
|
})
|
|
}
|
|
}
|