Add first version of FindUsedBlobs

src/restic/find.go

@@ -5,8 +5,35 @@ import (
 	"restic/repository"
 )
 
+//  FindUsedBlobs traverse the tree ID and adds all seen blobs to blobs.
+func findUsedBlobs(repo *repository.Repository, treeID backend.ID, blobs backend.IDSet) error {
+	blobs.Insert(treeID)
+
+	tree, err := LoadTree(repo, treeID)
+	if err != nil {
+		return err
+	}
+
+	for _, node := range tree.Nodes {
+		switch node.Type {
+		case "file":
+			for _, blob := range node.Content {
+				blobs.Insert(blob)
+			}
+		case "dir":
+			err := findUsedBlobs(repo, *node.Subtree, blobs)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
 // FindUsedBlobs traverses the tree ID and returns a set of all blobs
 // encountered.
-func FindUsedBlobs(repo *repository.Repository, treeID backend.ID) (backend.IDSet, error) {
-	return nil, nil
+func FindUsedBlobs(repo *repository.Repository, treeID backend.ID) (blobs backend.IDSet, err error) {
+	blobs = backend.NewIDSet()
+	return blobs, findUsedBlobs(repo, treeID, blobs)
 }

src/restic/find_test.go

@@ -1,12 +1,69 @@
 package restic
 
 import (
+	"bufio"
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
 	"testing"
 	"time"
 
+	"restic/backend"
 	"restic/repository"
 )
 
+func loadIDSet(t testing.TB, filename string) backend.IDSet {
+	f, err := os.Open(filename)
+	if err != nil {
+		t.Logf("unable to open golden file %v: %v", filename, err)
+		return backend.IDSet{}
+	}
+
+	sc := bufio.NewScanner(f)
+
+	ids := backend.NewIDSet()
+	for sc.Scan() {
+		id, err := backend.ParseID(sc.Text())
+		if err != nil {
+			t.Errorf("file %v contained invalid id: %v", filename, err)
+		}
+
+		ids.Insert(id)
+	}
+
+	if err = f.Close(); err != nil {
+		t.Errorf("closing file %v failed with error %v", filename, err)
+	}
+
+	return ids
+}
+
+func saveIDSet(t testing.TB, filename string, s backend.IDSet) {
+	f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE, 0644)
+	if err != nil {
+		t.Fatalf("unable to update golden file %v: %v", filename, err)
+		return
+	}
+
+	var ids backend.IDs
+	for id := range s {
+		ids = append(ids, id)
+	}
+
+	sort.Sort(ids)
+	for _, id := range ids {
+		fmt.Fprintf(f, "%s\n", id)
+	}
+
+	if err = f.Close(); err != nil {
+		t.Fatalf("close file %v returned error: %v", filename, err)
+	}
+}
+
+var updateGoldenFiles = flag.Bool("update", false, "update golden files in testdata/")
+
 const (
 	testSnapshots = 3
 	testDepth     = 2
@@ -25,7 +82,7 @@ func TestFindUsedBlobs(t *testing.T) {
 		snapshots = append(snapshots, sn)
 	}
 
-	for _, sn := range snapshots {
+	for i, sn := range snapshots {
 		usedBlobs, err := FindUsedBlobs(repo, *sn.Tree)
 		if err != nil {
 			t.Errorf("FindUsedBlobs returned error: %v", err)
@@ -37,6 +94,16 @@ func TestFindUsedBlobs(t *testing.T) {
 			continue
 		}
 
-		t.Logf("used blobs from snapshot %v (tree %v): %v", sn.ID().Str(), sn.Tree.Str(), usedBlobs)
+		goldenFilename := filepath.Join("testdata", fmt.Sprintf("used_blobs_snapshot%d", i))
+		want := loadIDSet(t, goldenFilename)
+
+		if !want.Equals(usedBlobs) {
+			t.Errorf("snapshot %d: wrong list of blobs returned:\n  missing blobs: %v\n  extra blobs: %v",
+				i, want.Sub(usedBlobs), usedBlobs.Sub(want))
+		}
+
+		if *updateGoldenFiles {
+			saveIDSet(t, goldenFilename, usedBlobs)
+		}
 	}
 }

src/restic/testdata/used_blobs_snapshot0

@@ -0,0 +1,37 @@
+087e8d5f45f93a78e52a938ac0b7864f92f8910091c0da69201a156242df3b78
+0bf505951741c44714527d252313b6959ce4f19d2e5512fca1c1b2da14424da3
+0c82d00e6ee78b48559cda2f9cc909beeb8769183b115dfda0a5767832accc8d
+2941bfd03b8933bb150b085a2252b69675495af64523bf8d38e67429e7cccb45
+378a9b6862c8fa5c6915f158d16e4416243159bb9da44c564896c065bc6c1cf4
+3ffcf5128fc404c2a363e3e8a8d4c8a7ae8c36fcacba7fdfe71ec9dabcadd567
+40f5ca234e5eed1dc967c83fa99076ef636619148082f300cf877676728ebf14
+42aad1ab6cc964043e53e5da13ed0f2b44a3bf6ae7702f60a805f13028377524
+42bc8f509dbd6b9881cab4c1684d5cf74207046336f654db1b884197f15cae7b
+47cf470c1c6de9af00b3b1ee963de8b94f51a2870b3338b3f33cfc565c0f8be4
+587045d0ec69e47a3cc91b13c959aa80add9118ecfac47232ea992650f25f0b9
+615e8851030f318751f3c8baf8fbfa9958e2dd7f25dc1a87dcf6d6f79d1f1a9f
+63ec5e835e11203bbeef69095523344dd975f1ab52bdbf4a1db7a53914d967ca
+714f9e16404b9ec83de56715e5387b2c4c2ed0af1889166a4e767822f971bf52
+80ba9a145bf46cae605e911c18165c02213e8d11d68dc5b7824f259d17b7b6d0
+86af714d79d18be1c9c0ae23cca9dbd7cef44530e253e80af5bd5c34eab09714
+8a445cf5b6313cbe3b5872a55adde52aa8d1ae188f41d56f176e40a3137ac058
+8e171f7367d1b68012ed1ceec8f54b7b9b8654ebaf63a760017c34d761b17878
+8e98f35e65fb42c85eb4a2ab4793e294148e3f318252cb850a896274d2aa90bc
+9d65ba6443863394a8c6582fef4a8aaab2fb46417eef41f1792cdbdb38ee0b4c
+9da502ea8e7a768ee0dbafdc613db3df4a7cd9c98af08328265c4d2e953e8efa
+9f2899688d2f23391cfd86e7b6d326a54f352bb294160878178639aab4aa378f
+a2f3ccf973b3600c06c42dc3b867b263a788c18aa57f4448fea2525b7cbfd784
+b2deaf9174086129ec3b9f79e05401fdb3baf8b75335addffac1950182d779df
+b81870ebe27b98f6b8746349e8ea444c96bf2eaac5dbd6236175150ce579f46b
+bd4dacd46031b2b837bc9bd06145b0571156fa496408ce728c003ae50b265aaf
+c0775cfc822f59524b4ed714d257607fd5f2c9f0dc9f65763a86ffc33aac325b
+c3596f717c495d20c33561e991d4295550b6d7544687f2363e999bdc0266224d
+c54c4899c4d7dcda8b9e597aebfbaf7d65c9c7a760527d77e7fc9894283d736e
+ca51ecf1633896f852929cb2d56ad1b5bed4ab6055bdcf370ced4011bed164aa
+ce8b656cead478c34060510962daf97cea52abde68bbef7934dd5c5513cf6f3b
+dafbb65569781083b627de833fb931cf98401299a62d747f03d8fc135ab57279
+e193d395410520580e76a5b89b8d23a1d162c0e28c52cb8194d409a74a120f7d
+e752efd93f9850ba0cafbbac01bb283c10095ac923cdb8ff027393001123d406
+f728e5576d4ab63248c310396d67d9afa3267dd2dea3cfba690dbd04efe181fb
+f75b6460b68d254f2195b08c606672fb55c05fb7bed7e16699b3231104b673ea
+fe19f084021bdac5a9a5d270042ff53ef36357dd0743318d0480dee1a43de266

src/restic/testdata/used_blobs_snapshot1

@@ -0,0 +1,34 @@
+011a951a9796979c2b515ef4209662013bd1f16a20a1b35d1d950d7408bdc8b4
+087e8d5f45f93a78e52a938ac0b7864f92f8910091c0da69201a156242df3b78
+0bad18b7f2d82d7c9cf8e405262ad2f3dbe57928aa242c1070b917042a99072d
+0bf505951741c44714527d252313b6959ce4f19d2e5512fca1c1b2da14424da3
+0c82d00e6ee78b48559cda2f9cc909beeb8769183b115dfda0a5767832accc8d
+2941bfd03b8933bb150b085a2252b69675495af64523bf8d38e67429e7cccb45
+3ffcf5128fc404c2a363e3e8a8d4c8a7ae8c36fcacba7fdfe71ec9dabcadd567
+40f5ca234e5eed1dc967c83fa99076ef636619148082f300cf877676728ebf14
+42bc8f509dbd6b9881cab4c1684d5cf74207046336f654db1b884197f15cae7b
+47cf470c1c6de9af00b3b1ee963de8b94f51a2870b3338b3f33cfc565c0f8be4
+4b2e91022c34c756b7bd8ece046a2bab6f0dcad89f46c52d1f84cd48e8da55df
+6416bc2321cdeb8758188af2b3925f2c82ffde014bf53b7a69c0f113a5c460fe
+714f9e16404b9ec83de56715e5387b2c4c2ed0af1889166a4e767822f971bf52
+80ba9a145bf46cae605e911c18165c02213e8d11d68dc5b7824f259d17b7b6d0
+83bf0196cf45bbca0be7e292688a3622af7888c0e9ec01bb78edaff302cced06
+8a445cf5b6313cbe3b5872a55adde52aa8d1ae188f41d56f176e40a3137ac058
+8e98f35e65fb42c85eb4a2ab4793e294148e3f318252cb850a896274d2aa90bc
+907acef01e05c3e0140858423e9284ddd3d64145ba8b0c3293371c5c7ab3d6b7
+9d65ba6443863394a8c6582fef4a8aaab2fb46417eef41f1792cdbdb38ee0b4c
+9da502ea8e7a768ee0dbafdc613db3df4a7cd9c98af08328265c4d2e953e8efa
+a2f3ccf973b3600c06c42dc3b867b263a788c18aa57f4448fea2525b7cbfd784
+b2deaf9174086129ec3b9f79e05401fdb3baf8b75335addffac1950182d779df
+b3915971171e049292e28d7bc61fe362e94f73aa49b578f4ca1322b47d7fc39c
+bd4dacd46031b2b837bc9bd06145b0571156fa496408ce728c003ae50b265aaf
+c3596f717c495d20c33561e991d4295550b6d7544687f2363e999bdc0266224d
+c54c4899c4d7dcda8b9e597aebfbaf7d65c9c7a760527d77e7fc9894283d736e
+ca51ecf1633896f852929cb2d56ad1b5bed4ab6055bdcf370ced4011bed164aa
+cb8001715217b4f6960aa24c1abb4b60a20c10f23abc1e5f69e0f5436bd788c8
+d39c4c264e01ec47b0386da3775c6b0cc337974627ff55792938cca4895ac6c4
+dafbb65569781083b627de833fb931cf98401299a62d747f03d8fc135ab57279
+e193d395410520580e76a5b89b8d23a1d162c0e28c52cb8194d409a74a120f7d
+e791912a7fad8954c764fae41d2958d2feeae2278e403429add9119ab43a36f5
+f728e5576d4ab63248c310396d67d9afa3267dd2dea3cfba690dbd04efe181fb
+fe19f084021bdac5a9a5d270042ff53ef36357dd0743318d0480dee1a43de266

src/restic/testdata/used_blobs_snapshot2

@@ -0,0 +1,9 @@
+35e13e123748cd27d1634c4e07e5ff2fc86901b09b215f3125331d1226c782be
+378a9b6862c8fa5c6915f158d16e4416243159bb9da44c564896c065bc6c1cf4
+42aad1ab6cc964043e53e5da13ed0f2b44a3bf6ae7702f60a805f13028377524
+47cf470c1c6de9af00b3b1ee963de8b94f51a2870b3338b3f33cfc565c0f8be4
+615e8851030f318751f3c8baf8fbfa9958e2dd7f25dc1a87dcf6d6f79d1f1a9f
+83bf0196cf45bbca0be7e292688a3622af7888c0e9ec01bb78edaff302cced06
+9d65ba6443863394a8c6582fef4a8aaab2fb46417eef41f1792cdbdb38ee0b4c
+b3915971171e049292e28d7bc61fe362e94f73aa49b578f4ca1322b47d7fc39c
+c0775cfc822f59524b4ed714d257607fd5f2c9f0dc9f65763a86ffc33aac325b