octoleo
/
restic
mirror of https://github.com/octoleo/restic.git
2
2
Fork 0
Code Issues Releases Wiki Activity

Use pack.BlobSet instead of backend.IDSet

This commit is contained in:
Alexander Neumann 2016-08-04 18:59:26 +02:00
parent 3cca831b2e
commit 6285f31604
6 changed files with 114 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/restic/find.go
View File

@ -2,12 +2,13 @@ package restic
import (
"restic/backend"
"restic/pack"
"restic/repository"
)
// findUsedBlobs traverse the tree ID and adds all seen blobs to blobs.
func findUsedBlobs(repo *repository.Repository, treeID backend.ID, blobs backend.IDSet, seen backend.IDSet) error {
blobs.Insert(treeID)
func findUsedBlobs(repo *repository.Repository, treeID backend.ID, blobs pack.BlobSet, seen pack.BlobSet) error {
blobs.Insert(pack.Handle{ID: treeID, Type: pack.Tree})
tree, err := LoadTree(repo, treeID)
if err != nil {
@ -18,15 +19,16 @@ func findUsedBlobs(repo *repository.Repository, treeID backend.ID, blobs backend
switch node.Type {
case "file":
for _, blob := range node.Content {
blobs.Insert(blob)
blobs.Insert(pack.Handle{ID: blob, Type: pack.Data})
}
case "dir":
subtreeID := *node.Subtree
if seen.Has(subtreeID) {
h := pack.Handle{ID: subtreeID, Type: pack.Tree}
if seen.Has(h) {
continue
}
seen.Insert(subtreeID)
seen.Insert(h)
err := findUsedBlobs(repo, subtreeID, blobs, seen)
if err != nil {
@ -39,6 +41,6 @@ func findUsedBlobs(repo *repository.Repository, treeID backend.ID, blobs backend
}
// FindUsedBlobs traverses the tree ID and adds all seen blobs (trees and data blobs) to the set blobs.
func FindUsedBlobs(repo *repository.Repository, treeID backend.ID, blobs backend.IDSet) error {
return findUsedBlobs(repo, treeID, blobs, backend.NewIDSet())
func FindUsedBlobs(repo *repository.Repository, treeID backend.ID, blobs pack.BlobSet) error {
return findUsedBlobs(repo, treeID, blobs, pack.NewBlobSet())
}

40
src/restic/find_test.go
View File

@ -2,6 +2,7 @@ package restic
import (
"bufio"
"encoding/json"
"flag"
"fmt"
"os"
@ -10,51 +11,58 @@ import (
"testing"
"time"
"restic/backend"
"restic/pack"
"restic/repository"
)
func loadIDSet(t testing.TB, filename string) backend.IDSet {
func loadIDSet(t testing.TB, filename string) pack.BlobSet {
f, err := os.Open(filename)
if err != nil {
t.Logf("unable to open golden file %v: %v", filename, err)
return backend.IDSet{}
return pack.NewBlobSet()
}
sc := bufio.NewScanner(f)
ids := backend.NewIDSet()
blobs := pack.NewBlobSet()
for sc.Scan() {
id, err := backend.ParseID(sc.Text())
var h pack.Handle
err := json.Unmarshal([]byte(sc.Text()), &h)
if err != nil {
t.Errorf("file %v contained invalid id: %v", filename, err)
t.Errorf("file %v contained invalid blob: %#v", filename, err)
continue
}
ids.Insert(id)
blobs.Insert(h)
}
if err = f.Close(); err != nil {
t.Errorf("closing file %v failed with error %v", filename, err)
}
return ids
return blobs
}
func saveIDSet(t testing.TB, filename string, s backend.IDSet) {
func saveIDSet(t testing.TB, filename string, s pack.BlobSet) {
f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE, 0644)
if err != nil {
t.Fatalf("unable to update golden file %v: %v", filename, err)
return
}
var ids backend.IDs
for id := range s {
ids = append(ids, id)
var hs pack.Handles
for h := range s {
hs = append(hs, h)
}
sort.Sort(ids)
for _, id := range ids {
fmt.Fprintf(f, "%s\n", id)
sort.Sort(hs)
enc := json.NewEncoder(f)
for _, h := range hs {
err = enc.Encode(h)
if err != nil {
t.Fatalf("Encode() returned error: %v", err)
}
}
if err = f.Close(); err != nil {
@ -83,7 +91,7 @@ func TestFindUsedBlobs(t *testing.T) {
}
for i, sn := range snapshots {
usedBlobs := backend.NewIDSet()
usedBlobs := pack.NewBlobSet()
err := FindUsedBlobs(repo, *sn.Tree, usedBlobs)
if err != nil {
t.Errorf("FindUsedBlobs returned error: %v", err)

6
src/restic/pack/handle.go
View File

@ -23,10 +23,6 @@ func (h Handles) Len() int {
}
func (h Handles) Less(i, j int) bool {
if h[i].Type != h[j].Type {
return h[i].Type < h[j].Type
}
for k, b := range h[i].ID {
if b == h[j].ID[k] {
continue
@ -39,7 +35,7 @@ func (h Handles) Less(i, j int) bool {
return false
}
return false
return h[i].Type < h[j].Type
}
func (h Handles) Swap(i, j int) {

74
src/restic/testdata/used_blobs_snapshot0 vendored
View File

@ -1,37 +1,37 @@
087e8d5f45f93a78e52a938ac0b7864f92f8910091c0da69201a156242df3b78
0bf505951741c44714527d252313b6959ce4f19d2e5512fca1c1b2da14424da3
0c82d00e6ee78b48559cda2f9cc909beeb8769183b115dfda0a5767832accc8d
2941bfd03b8933bb150b085a2252b69675495af64523bf8d38e67429e7cccb45
378a9b6862c8fa5c6915f158d16e4416243159bb9da44c564896c065bc6c1cf4
3ffcf5128fc404c2a363e3e8a8d4c8a7ae8c36fcacba7fdfe71ec9dabcadd567
40f5ca234e5eed1dc967c83fa99076ef636619148082f300cf877676728ebf14
42aad1ab6cc964043e53e5da13ed0f2b44a3bf6ae7702f60a805f13028377524
42bc8f509dbd6b9881cab4c1684d5cf74207046336f654db1b884197f15cae7b
47cf470c1c6de9af00b3b1ee963de8b94f51a2870b3338b3f33cfc565c0f8be4
587045d0ec69e47a3cc91b13c959aa80add9118ecfac47232ea992650f25f0b9
615e8851030f318751f3c8baf8fbfa9958e2dd7f25dc1a87dcf6d6f79d1f1a9f
63ec5e835e11203bbeef69095523344dd975f1ab52bdbf4a1db7a53914d967ca
714f9e16404b9ec83de56715e5387b2c4c2ed0af1889166a4e767822f971bf52
80ba9a145bf46cae605e911c18165c02213e8d11d68dc5b7824f259d17b7b6d0
86af714d79d18be1c9c0ae23cca9dbd7cef44530e253e80af5bd5c34eab09714
8a445cf5b6313cbe3b5872a55adde52aa8d1ae188f41d56f176e40a3137ac058
8e171f7367d1b68012ed1ceec8f54b7b9b8654ebaf63a760017c34d761b17878
8e98f35e65fb42c85eb4a2ab4793e294148e3f318252cb850a896274d2aa90bc
9d65ba6443863394a8c6582fef4a8aaab2fb46417eef41f1792cdbdb38ee0b4c
9da502ea8e7a768ee0dbafdc613db3df4a7cd9c98af08328265c4d2e953e8efa
9f2899688d2f23391cfd86e7b6d326a54f352bb294160878178639aab4aa378f
a2f3ccf973b3600c06c42dc3b867b263a788c18aa57f4448fea2525b7cbfd784
b2deaf9174086129ec3b9f79e05401fdb3baf8b75335addffac1950182d779df
b81870ebe27b98f6b8746349e8ea444c96bf2eaac5dbd6236175150ce579f46b
bd4dacd46031b2b837bc9bd06145b0571156fa496408ce728c003ae50b265aaf
c0775cfc822f59524b4ed714d257607fd5f2c9f0dc9f65763a86ffc33aac325b
c3596f717c495d20c33561e991d4295550b6d7544687f2363e999bdc0266224d
c54c4899c4d7dcda8b9e597aebfbaf7d65c9c7a760527d77e7fc9894283d736e
ca51ecf1633896f852929cb2d56ad1b5bed4ab6055bdcf370ced4011bed164aa
ce8b656cead478c34060510962daf97cea52abde68bbef7934dd5c5513cf6f3b
dafbb65569781083b627de833fb931cf98401299a62d747f03d8fc135ab57279
e193d395410520580e76a5b89b8d23a1d162c0e28c52cb8194d409a74a120f7d
e752efd93f9850ba0cafbbac01bb283c10095ac923cdb8ff027393001123d406
f728e5576d4ab63248c310396d67d9afa3267dd2dea3cfba690dbd04efe181fb
f75b6460b68d254f2195b08c606672fb55c05fb7bed7e16699b3231104b673ea
fe19f084021bdac5a9a5d270042ff53ef36357dd0743318d0480dee1a43de266
{"ID":"087e8d5f45f93a78e52a938ac0b7864f92f8910091c0da69201a156242df3b78","Type":"data"}
{"ID":"0bf505951741c44714527d252313b6959ce4f19d2e5512fca1c1b2da14424da3","Type":"data"}
{"ID":"0c82d00e6ee78b48559cda2f9cc909beeb8769183b115dfda0a5767832accc8d","Type":"data"}
{"ID":"2941bfd03b8933bb150b085a2252b69675495af64523bf8d38e67429e7cccb45","Type":"data"}
{"ID":"378a9b6862c8fa5c6915f158d16e4416243159bb9da44c564896c065bc6c1cf4","Type":"data"}
{"ID":"3ffcf5128fc404c2a363e3e8a8d4c8a7ae8c36fcacba7fdfe71ec9dabcadd567","Type":"data"}
{"ID":"40f5ca234e5eed1dc967c83fa99076ef636619148082f300cf877676728ebf14","Type":"data"}
{"ID":"42aad1ab6cc964043e53e5da13ed0f2b44a3bf6ae7702f60a805f13028377524","Type":"data"}
{"ID":"42bc8f509dbd6b9881cab4c1684d5cf74207046336f654db1b884197f15cae7b","Type":"data"}
{"ID":"47cf470c1c6de9af00b3b1ee963de8b94f51a2870b3338b3f33cfc565c0f8be4","Type":"data"}
{"ID":"587045d0ec69e47a3cc91b13c959aa80add9118ecfac47232ea992650f25f0b9","Type":"data"}
{"ID":"615e8851030f318751f3c8baf8fbfa9958e2dd7f25dc1a87dcf6d6f79d1f1a9f","Type":"data"}
{"ID":"63ec5e835e11203bbeef69095523344dd975f1ab52bdbf4a1db7a53914d967ca","Type":"tree"}
{"ID":"714f9e16404b9ec83de56715e5387b2c4c2ed0af1889166a4e767822f971bf52","Type":"data"}
{"ID":"80ba9a145bf46cae605e911c18165c02213e8d11d68dc5b7824f259d17b7b6d0","Type":"data"}
{"ID":"86af714d79d18be1c9c0ae23cca9dbd7cef44530e253e80af5bd5c34eab09714","Type":"data"}
{"ID":"8a445cf5b6313cbe3b5872a55adde52aa8d1ae188f41d56f176e40a3137ac058","Type":"data"}
{"ID":"8e171f7367d1b68012ed1ceec8f54b7b9b8654ebaf63a760017c34d761b17878","Type":"tree"}
{"ID":"8e98f35e65fb42c85eb4a2ab4793e294148e3f318252cb850a896274d2aa90bc","Type":"data"}
{"ID":"9d65ba6443863394a8c6582fef4a8aaab2fb46417eef41f1792cdbdb38ee0b4c","Type":"data"}
{"ID":"9da502ea8e7a768ee0dbafdc613db3df4a7cd9c98af08328265c4d2e953e8efa","Type":"data"}
{"ID":"9f2899688d2f23391cfd86e7b6d326a54f352bb294160878178639aab4aa378f","Type":"tree"}
{"ID":"a2f3ccf973b3600c06c42dc3b867b263a788c18aa57f4448fea2525b7cbfd784","Type":"data"}
{"ID":"b2deaf9174086129ec3b9f79e05401fdb3baf8b75335addffac1950182d779df","Type":"data"}
{"ID":"b81870ebe27b98f6b8746349e8ea444c96bf2eaac5dbd6236175150ce579f46b","Type":"tree"}
{"ID":"bd4dacd46031b2b837bc9bd06145b0571156fa496408ce728c003ae50b265aaf","Type":"data"}
{"ID":"c0775cfc822f59524b4ed714d257607fd5f2c9f0dc9f65763a86ffc33aac325b","Type":"data"}
{"ID":"c3596f717c495d20c33561e991d4295550b6d7544687f2363e999bdc0266224d","Type":"data"}
{"ID":"c54c4899c4d7dcda8b9e597aebfbaf7d65c9c7a760527d77e7fc9894283d736e","Type":"data"}
{"ID":"ca51ecf1633896f852929cb2d56ad1b5bed4ab6055bdcf370ced4011bed164aa","Type":"data"}
{"ID":"ce8b656cead478c34060510962daf97cea52abde68bbef7934dd5c5513cf6f3b","Type":"data"}
{"ID":"dafbb65569781083b627de833fb931cf98401299a62d747f03d8fc135ab57279","Type":"data"}
{"ID":"e193d395410520580e76a5b89b8d23a1d162c0e28c52cb8194d409a74a120f7d","Type":"data"}
{"ID":"e752efd93f9850ba0cafbbac01bb283c10095ac923cdb8ff027393001123d406","Type":"tree"}
{"ID":"f728e5576d4ab63248c310396d67d9afa3267dd2dea3cfba690dbd04efe181fb","Type":"data"}
{"ID":"f75b6460b68d254f2195b08c606672fb55c05fb7bed7e16699b3231104b673ea","Type":"tree"}
{"ID":"fe19f084021bdac5a9a5d270042ff53ef36357dd0743318d0480dee1a43de266","Type":"data"}

68
src/restic/testdata/used_blobs_snapshot1 vendored
View File

@ -1,34 +1,34 @@
011a951a9796979c2b515ef4209662013bd1f16a20a1b35d1d950d7408bdc8b4
087e8d5f45f93a78e52a938ac0b7864f92f8910091c0da69201a156242df3b78
0bad18b7f2d82d7c9cf8e405262ad2f3dbe57928aa242c1070b917042a99072d
0bf505951741c44714527d252313b6959ce4f19d2e5512fca1c1b2da14424da3
0c82d00e6ee78b48559cda2f9cc909beeb8769183b115dfda0a5767832accc8d
2941bfd03b8933bb150b085a2252b69675495af64523bf8d38e67429e7cccb45
3ffcf5128fc404c2a363e3e8a8d4c8a7ae8c36fcacba7fdfe71ec9dabcadd567
40f5ca234e5eed1dc967c83fa99076ef636619148082f300cf877676728ebf14
42bc8f509dbd6b9881cab4c1684d5cf74207046336f654db1b884197f15cae7b
47cf470c1c6de9af00b3b1ee963de8b94f51a2870b3338b3f33cfc565c0f8be4
4b2e91022c34c756b7bd8ece046a2bab6f0dcad89f46c52d1f84cd48e8da55df
6416bc2321cdeb8758188af2b3925f2c82ffde014bf53b7a69c0f113a5c460fe
714f9e16404b9ec83de56715e5387b2c4c2ed0af1889166a4e767822f971bf52
80ba9a145bf46cae605e911c18165c02213e8d11d68dc5b7824f259d17b7b6d0
83bf0196cf45bbca0be7e292688a3622af7888c0e9ec01bb78edaff302cced06
8a445cf5b6313cbe3b5872a55adde52aa8d1ae188f41d56f176e40a3137ac058
8e98f35e65fb42c85eb4a2ab4793e294148e3f318252cb850a896274d2aa90bc
907acef01e05c3e0140858423e9284ddd3d64145ba8b0c3293371c5c7ab3d6b7
9d65ba6443863394a8c6582fef4a8aaab2fb46417eef41f1792cdbdb38ee0b4c
9da502ea8e7a768ee0dbafdc613db3df4a7cd9c98af08328265c4d2e953e8efa
a2f3ccf973b3600c06c42dc3b867b263a788c18aa57f4448fea2525b7cbfd784
b2deaf9174086129ec3b9f79e05401fdb3baf8b75335addffac1950182d779df
b3915971171e049292e28d7bc61fe362e94f73aa49b578f4ca1322b47d7fc39c
bd4dacd46031b2b837bc9bd06145b0571156fa496408ce728c003ae50b265aaf
c3596f717c495d20c33561e991d4295550b6d7544687f2363e999bdc0266224d
c54c4899c4d7dcda8b9e597aebfbaf7d65c9c7a760527d77e7fc9894283d736e
ca51ecf1633896f852929cb2d56ad1b5bed4ab6055bdcf370ced4011bed164aa
cb8001715217b4f6960aa24c1abb4b60a20c10f23abc1e5f69e0f5436bd788c8
d39c4c264e01ec47b0386da3775c6b0cc337974627ff55792938cca4895ac6c4
dafbb65569781083b627de833fb931cf98401299a62d747f03d8fc135ab57279
e193d395410520580e76a5b89b8d23a1d162c0e28c52cb8194d409a74a120f7d
e791912a7fad8954c764fae41d2958d2feeae2278e403429add9119ab43a36f5
f728e5576d4ab63248c310396d67d9afa3267dd2dea3cfba690dbd04efe181fb
fe19f084021bdac5a9a5d270042ff53ef36357dd0743318d0480dee1a43de266
{"ID":"011a951a9796979c2b515ef4209662013bd1f16a20a1b35d1d950d7408bdc8b4","Type":"tree"}
{"ID":"087e8d5f45f93a78e52a938ac0b7864f92f8910091c0da69201a156242df3b78","Type":"data"}
{"ID":"0bad18b7f2d82d7c9cf8e405262ad2f3dbe57928aa242c1070b917042a99072d","Type":"data"}
{"ID":"0bf505951741c44714527d252313b6959ce4f19d2e5512fca1c1b2da14424da3","Type":"data"}
{"ID":"0c82d00e6ee78b48559cda2f9cc909beeb8769183b115dfda0a5767832accc8d","Type":"data"}
{"ID":"2941bfd03b8933bb150b085a2252b69675495af64523bf8d38e67429e7cccb45","Type":"data"}
{"ID":"3ffcf5128fc404c2a363e3e8a8d4c8a7ae8c36fcacba7fdfe71ec9dabcadd567","Type":"data"}
{"ID":"40f5ca234e5eed1dc967c83fa99076ef636619148082f300cf877676728ebf14","Type":"data"}
{"ID":"42bc8f509dbd6b9881cab4c1684d5cf74207046336f654db1b884197f15cae7b","Type":"data"}
{"ID":"47cf470c1c6de9af00b3b1ee963de8b94f51a2870b3338b3f33cfc565c0f8be4","Type":"data"}
{"ID":"4b2e91022c34c756b7bd8ece046a2bab6f0dcad89f46c52d1f84cd48e8da55df","Type":"tree"}
{"ID":"6416bc2321cdeb8758188af2b3925f2c82ffde014bf53b7a69c0f113a5c460fe","Type":"data"}
{"ID":"714f9e16404b9ec83de56715e5387b2c4c2ed0af1889166a4e767822f971bf52","Type":"data"}
{"ID":"80ba9a145bf46cae605e911c18165c02213e8d11d68dc5b7824f259d17b7b6d0","Type":"data"}
{"ID":"83bf0196cf45bbca0be7e292688a3622af7888c0e9ec01bb78edaff302cced06","Type":"data"}
{"ID":"8a445cf5b6313cbe3b5872a55adde52aa8d1ae188f41d56f176e40a3137ac058","Type":"data"}
{"ID":"8e98f35e65fb42c85eb4a2ab4793e294148e3f318252cb850a896274d2aa90bc","Type":"data"}
{"ID":"907acef01e05c3e0140858423e9284ddd3d64145ba8b0c3293371c5c7ab3d6b7","Type":"data"}
{"ID":"9d65ba6443863394a8c6582fef4a8aaab2fb46417eef41f1792cdbdb38ee0b4c","Type":"data"}
{"ID":"9da502ea8e7a768ee0dbafdc613db3df4a7cd9c98af08328265c4d2e953e8efa","Type":"data"}
{"ID":"a2f3ccf973b3600c06c42dc3b867b263a788c18aa57f4448fea2525b7cbfd784","Type":"data"}
{"ID":"b2deaf9174086129ec3b9f79e05401fdb3baf8b75335addffac1950182d779df","Type":"data"}
{"ID":"b3915971171e049292e28d7bc61fe362e94f73aa49b578f4ca1322b47d7fc39c","Type":"data"}
{"ID":"bd4dacd46031b2b837bc9bd06145b0571156fa496408ce728c003ae50b265aaf","Type":"data"}
{"ID":"c3596f717c495d20c33561e991d4295550b6d7544687f2363e999bdc0266224d","Type":"data"}
{"ID":"c54c4899c4d7dcda8b9e597aebfbaf7d65c9c7a760527d77e7fc9894283d736e","Type":"data"}
{"ID":"ca51ecf1633896f852929cb2d56ad1b5bed4ab6055bdcf370ced4011bed164aa","Type":"data"}
{"ID":"cb8001715217b4f6960aa24c1abb4b60a20c10f23abc1e5f69e0f5436bd788c8","Type":"data"}
{"ID":"d39c4c264e01ec47b0386da3775c6b0cc337974627ff55792938cca4895ac6c4","Type":"data"}
{"ID":"dafbb65569781083b627de833fb931cf98401299a62d747f03d8fc135ab57279","Type":"data"}
{"ID":"e193d395410520580e76a5b89b8d23a1d162c0e28c52cb8194d409a74a120f7d","Type":"data"}
{"ID":"e791912a7fad8954c764fae41d2958d2feeae2278e403429add9119ab43a36f5","Type":"tree"}
{"ID":"f728e5576d4ab63248c310396d67d9afa3267dd2dea3cfba690dbd04efe181fb","Type":"data"}
{"ID":"fe19f084021bdac5a9a5d270042ff53ef36357dd0743318d0480dee1a43de266","Type":"data"}

18
src/restic/testdata/used_blobs_snapshot2 vendored
View File

@ -1,9 +1,9 @@
35e13e123748cd27d1634c4e07e5ff2fc86901b09b215f3125331d1226c782be
378a9b6862c8fa5c6915f158d16e4416243159bb9da44c564896c065bc6c1cf4
42aad1ab6cc964043e53e5da13ed0f2b44a3bf6ae7702f60a805f13028377524
47cf470c1c6de9af00b3b1ee963de8b94f51a2870b3338b3f33cfc565c0f8be4
615e8851030f318751f3c8baf8fbfa9958e2dd7f25dc1a87dcf6d6f79d1f1a9f
83bf0196cf45bbca0be7e292688a3622af7888c0e9ec01bb78edaff302cced06
9d65ba6443863394a8c6582fef4a8aaab2fb46417eef41f1792cdbdb38ee0b4c
b3915971171e049292e28d7bc61fe362e94f73aa49b578f4ca1322b47d7fc39c
c0775cfc822f59524b4ed714d257607fd5f2c9f0dc9f65763a86ffc33aac325b
{"ID":"35e13e123748cd27d1634c4e07e5ff2fc86901b09b215f3125331d1226c782be","Type":"tree"}
{"ID":"378a9b6862c8fa5c6915f158d16e4416243159bb9da44c564896c065bc6c1cf4","Type":"data"}
{"ID":"42aad1ab6cc964043e53e5da13ed0f2b44a3bf6ae7702f60a805f13028377524","Type":"data"}
{"ID":"47cf470c1c6de9af00b3b1ee963de8b94f51a2870b3338b3f33cfc565c0f8be4","Type":"data"}
{"ID":"615e8851030f318751f3c8baf8fbfa9958e2dd7f25dc1a87dcf6d6f79d1f1a9f","Type":"data"}
{"ID":"83bf0196cf45bbca0be7e292688a3622af7888c0e9ec01bb78edaff302cced06","Type":"data"}
{"ID":"9d65ba6443863394a8c6582fef4a8aaab2fb46417eef41f1792cdbdb38ee0b4c","Type":"data"}
{"ID":"b3915971171e049292e28d7bc61fe362e94f73aa49b578f4ca1322b47d7fc39c","Type":"data"}
{"ID":"c0775cfc822f59524b4ed714d257607fd5f2c9f0dc9f65763a86ffc33aac325b","Type":"data"}