package cache import ( "bytes" "fmt" "io" "math/rand" "os" "runtime" "testing" "time" "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" "golang.org/x/sync/errgroup" ) func generateRandomFiles(t testing.TB, random *rand.Rand, tpe backend.FileType, c *Cache) restic.IDSet { ids := restic.NewIDSet() for i := 0; i < random.Intn(15)+10; i++ { buf := rtest.Random(random.Int(), 1<<19) id := restic.Hash(buf) h := backend.Handle{Type: tpe, Name: id.String()} if c.Has(h) { t.Errorf("index %v present before save", id) } err := c.save(h, bytes.NewReader(buf)) if err != nil { t.Fatal(err) } ids.Insert(id) } return ids } // randomID returns a random ID from s. func randomID(s restic.IDSet) restic.ID { for id := range s { return id } panic("set is empty") } func load(t testing.TB, c *Cache, h backend.Handle) []byte { rd, inCache, err := c.load(h, 0, 0) if err != nil { t.Fatal(err) } rtest.Equals(t, true, inCache, "expected inCache flag to be true") if rd == nil { t.Fatalf("load() returned nil reader") } buf, err := io.ReadAll(rd) if err != nil { t.Fatal(err) } if err = rd.Close(); err != nil { t.Fatal(err) } return buf } func listFiles(t testing.TB, c *Cache, tpe restic.FileType) restic.IDSet { list, err := c.list(tpe) if err != nil { t.Errorf("listing failed: %v", err) } return list } func clearFiles(t testing.TB, c *Cache, tpe restic.FileType, valid restic.IDSet) { if err := c.Clear(tpe, valid); err != nil { t.Error(err) } } func TestFiles(t *testing.T) { seed := time.Now().Unix() t.Logf("seed is %v", seed) random := rand.New(rand.NewSource(seed)) c := TestNewCache(t) var tests = []restic.FileType{ restic.SnapshotFile, restic.PackFile, restic.IndexFile, } for _, tpe := range tests { t.Run(tpe.String(), func(t *testing.T) { ids := generateRandomFiles(t, random, tpe, c) id := randomID(ids) h := backend.Handle{Type: tpe, Name: id.String()} id2 := restic.Hash(load(t, c, h)) if !id.Equal(id2) { t.Errorf("wrong data returned, want %v, got %v", id.Str(), id2.Str()) } if !c.Has(h) { t.Errorf("cache thinks index %v isn't present", id.Str()) } list := listFiles(t, c, tpe) if !ids.Equals(list) { t.Errorf("wrong list of index IDs returned, want:\n %v\ngot:\n %v", ids, list) } clearFiles(t, c, tpe, restic.NewIDSet(id)) list2 := listFiles(t, c, tpe) ids.Delete(id) want := restic.NewIDSet(id) if !list2.Equals(want) { t.Errorf("ClearIndexes removed indexes, want:\n %v\ngot:\n %v", list2, want) } clearFiles(t, c, tpe, restic.NewIDSet()) want = restic.NewIDSet() list3 := listFiles(t, c, tpe) if !list3.Equals(want) { t.Errorf("ClearIndexes returned a wrong list, want:\n %v\ngot:\n %v", want, list3) } }) } } func TestFileLoad(t *testing.T) { seed := time.Now().Unix() t.Logf("seed is %v", seed) random := rand.New(rand.NewSource(seed)) c := TestNewCache(t) // save about 5 MiB of data in the cache data := rtest.Random(random.Int(), 5234142) id := restic.ID{} copy(id[:], data) h := backend.Handle{ Type: restic.PackFile, Name: id.String(), } if err := c.save(h, bytes.NewReader(data)); err != nil { t.Fatalf("Save() returned error: %v", err) } var tests = []struct { offset int64 length int }{ {0, 0}, {5, 0}, {32*1024 + 5, 0}, {0, 123}, {0, 64*1024 + 234}, {100, 5234142 - 100}, } for _, test := range tests { t.Run(fmt.Sprintf("%v/%v", test.length, test.offset), func(t *testing.T) { rd, inCache, err := c.load(h, test.length, test.offset) if err != nil { t.Fatal(err) } rtest.Equals(t, true, inCache, "expected inCache flag to be true") buf, err := io.ReadAll(rd) if err != nil { t.Fatal(err) } if err = rd.Close(); err != nil { t.Fatal(err) } o := int(test.offset) l := test.length if test.length == 0 { l = len(data) - o } if l > len(data)-o { l = len(data) - o } if len(buf) != l { t.Fatalf("wrong number of bytes returned: want %d, got %d", l, len(buf)) } if !bytes.Equal(buf, data[o:o+l]) { t.Fatalf("wrong data returned, want:\n %02x\ngot:\n %02x", data[o:o+16], buf[:16]) } }) } } // Simulate multiple processes writing to a cache, using goroutines. // // The possibility of sharing a cache between multiple concurrent restic // processes isn't guaranteed in the docs and doesn't always work on Windows, hence the // check on GOOS. Cache sharing is considered a "nice to have" on POSIX, for now. // // The cache first creates a temporary file and then renames it to its final name. // On Windows renaming internally creates a file handle with a shareMode which // includes FILE_SHARE_DELETE. The Go runtime opens files without FILE_SHARE_DELETE, // thus Open(fn) will fail until the file handle used for renaming was closed. // See https://devblogs.microsoft.com/oldnewthing/20211022-00/?p=105822 // for hints on how to fix this properly. func TestFileSaveConcurrent(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("may not work due to FILE_SHARE_DELETE issue") } seed := time.Now().Unix() t.Logf("seed is %v", seed) random := rand.New(rand.NewSource(seed)) const nproc = 40 var ( c = TestNewCache(t) data = rtest.Random(1, 10000) g errgroup.Group id restic.ID ) random.Read(id[:]) h := backend.Handle{ Type: restic.PackFile, Name: id.String(), } for i := 0; i < nproc/2; i++ { g.Go(func() error { return c.save(h, bytes.NewReader(data)) }) // Can't use load because only the main goroutine may call t.Fatal. g.Go(func() error { // The timing is hard to get right, but the main thing we want to // ensure is ENOENT or nil error. time.Sleep(time.Duration(100+rand.Intn(200)) * time.Millisecond) f, _, err := c.load(h, 0, 0) t.Logf("Load error: %v", err) switch { case err == nil: case errors.Is(err, os.ErrNotExist): return nil default: return err } defer func() { _ = f.Close() }() read, err := io.ReadAll(f) if err == nil && !bytes.Equal(read, data) { err = errors.New("mismatch between Save and Load") } return err }) } rtest.OK(t, g.Wait()) saved := load(t, c, h) rtest.Equals(t, data, saved) } func TestFileSaveAfterDamage(t *testing.T) { c := TestNewCache(t) rtest.OK(t, fs.RemoveAll(c.path)) // save a few bytes of data in the cache data := rtest.Random(123456789, 42) id := restic.Hash(data) h := backend.Handle{ Type: restic.PackFile, Name: id.String(), } if err := c.save(h, bytes.NewReader(data)); err == nil { t.Fatal("Missing error when saving to deleted cache directory") } }