Merge pull request #392 from restic/fix-build

Allow saving duplicate blobs in the repacker

checker/repacker.go

@@ -136,7 +136,7 @@ func repackBlob(src, dst *repository.Repository, id backend.ID) error {
 		return errors.New("LoadBlob returned wrong data, len() doesn't match")
 	}
 
-	_, err = dst.SaveAndEncrypt(blob.Type, buf, &id)
+	_, err = dst.SaveAndEncrypt(blob.Type, buf, &id, true)
 	if err != nil {
 		return err
 	}

crypto/crypto_test.go

@@ -133,7 +133,6 @@ func TestLargeEncrypt(t *testing.T) {
 
 func BenchmarkEncryptWriter(b *testing.B) {
 	size := 8 << 20 // 8MiB
-	rd := RandomReader(23, size)
 
 	k := crypto.NewRandomKey()
 
@@ -141,7 +140,7 @@ func BenchmarkEncryptWriter(b *testing.B) {
 	b.SetBytes(int64(size))
 
 	for i := 0; i < b.N; i++ {
-		rd.Seek(0, 0)
+		rd := RandomReader(23, size)
 		wr := crypto.EncryptTo(k, ioutil.Discard)
 		n, err := io.Copy(wr, rd)
 		OK(b, err)
@@ -195,14 +194,13 @@ func BenchmarkEncryptDecryptReader(b *testing.B) {
 	k := crypto.NewRandomKey()
 
 	size := 8 << 20 // 8MiB
-	rd := RandomReader(23, size)
 
 	b.ResetTimer()
 	b.SetBytes(int64(size))
 
 	buf := bytes.NewBuffer(nil)
 	for i := 0; i < b.N; i++ {
-		rd.Seek(0, 0)
+		rd := RandomReader(23, size)
 		buf.Reset()
 		wr := crypto.EncryptTo(k, buf)
 		_, err := io.Copy(wr, rd)

repository/master_index.go

@@ -155,25 +155,32 @@ func (mi *MasterIndex) Current() *Index {
 }
 
 // AddInFlight add the given ID to the list of in-flight IDs. An error is
-// returned when the ID is already in the list.
+// returned when the ID is already in the list. Setting ignoreDuplicates to
-func (mi *MasterIndex) AddInFlight(id backend.ID) error {
+// true only checks the in flight list, otherwise the index itself is also
+// tested.
+func (mi *MasterIndex) AddInFlight(id backend.ID, ignoreDuplicates bool) error {
 	// The index + inFlight store must be searched for a matching id in one
 	// atomic operation. This requires locking the inFlight store and the
 	// index together!
 	mi.inFlight.Lock()
 	defer mi.inFlight.Unlock()
 
+	if !ignoreDuplicates {
 		// Note: mi.Has read locks the index again.
 		mi.idxMutex.RLock()
 		defer mi.idxMutex.RUnlock()
-
-    debug.Log("MasterIndex.AddInFlight", "adding %v", id)
-    if mi.inFlight.Has(id) {
-        return fmt.Errorf("%v is already in flight", id)
 	}
+
+	debug.Log("MasterIndex.AddInFlight", "adding %v", id.Str())
+	if mi.inFlight.Has(id) {
+		return fmt.Errorf("%v is already in flight", id.Str())
+	}
+
+	if !ignoreDuplicates {
 		if mi.Has(id) {
 			return fmt.Errorf("%v is already indexed (fully processed)", id)
 		}
+	}
 
 	mi.inFlight.Insert(id)
 	return nil
@@ -187,7 +194,6 @@ func (mi *MasterIndex) IsInFlight(id backend.ID) bool {
 	mi.inFlight.RLock()
 	defer mi.inFlight.RUnlock()
 
-    // Note: mi.Has read locks the index again.
 	mi.idxMutex.RLock()
 	defer mi.idxMutex.RUnlock()
 

repository/repository.go

@@ -219,8 +219,9 @@ func (r *Repository) LookupBlobSize(id backend.ID) (uint, error) {
 }
 
 // SaveAndEncrypt encrypts data and stores it to the backend as type t. If data is small
-// enough, it will be packed together with other small blobs.
+// enough, it will be packed together with other small blobs. When
-func (r *Repository) SaveAndEncrypt(t pack.BlobType, data []byte, id *backend.ID) (backend.ID, error) {
+// ignoreDuplicates is true, blobs already in the index will be saved again.
+func (r *Repository) SaveAndEncrypt(t pack.BlobType, data []byte, id *backend.ID, ignoreDuplicates bool) (backend.ID, error) {
 	if id == nil {
 		// compute plaintext hash
 		hashedID := backend.Hash(data)
@@ -241,7 +242,7 @@ func (r *Repository) SaveAndEncrypt(t pack.BlobType, data []byte, id *backend.ID
 
 	// add this id to the list of in-flight chunk ids.
 	debug.Log("Repo.Save", "add %v to list of in-flight IDs", id.Str())
-	err = r.idx.AddInFlight(*id)
+	err = r.idx.AddInFlight(*id, ignoreDuplicates)
 	if err != nil {
 		debug.Log("Repo.Save", "another goroutine is already working on %v (%v) does already exist", id.Str, t)
 		return *id, nil
@@ -284,7 +285,7 @@ func (r *Repository) SaveFrom(t pack.BlobType, id *backend.ID, length uint, rd i
 		return err
 	}
 
-	_, err = r.SaveAndEncrypt(t, buf, id)
+	_, err = r.SaveAndEncrypt(t, buf, id, false)
 	if err != nil {
 		return err
 	}
@@ -308,7 +309,7 @@ func (r *Repository) SaveJSON(t pack.BlobType, item interface{}) (backend.ID, er
 	}
 
 	buf = wr.Bytes()
-	return r.SaveAndEncrypt(t, buf, nil)
+	return r.SaveAndEncrypt(t, buf, nil, false)
 }
 
 // SaveJSONUnpacked serialises item as JSON and encrypts and saves it in the

repository/repository_test.go

@@ -83,7 +83,7 @@ func TestSave(t *testing.T) {
 		id := backend.Hash(data)
 
 		// save
-		sid, err := repo.SaveAndEncrypt(pack.Data, data, nil)
+		sid, err := repo.SaveAndEncrypt(pack.Data, data, nil, false)
 		OK(t, err)
 
 		Equals(t, id, sid)
@@ -253,7 +253,7 @@ func saveRandomDataBlobs(t testing.TB, repo *repository.Repository, num int, siz
 		_, err := io.ReadFull(rand.Reader, buf)
 		OK(t, err)
 
-		_, err = repo.SaveAndEncrypt(pack.Data, buf, nil)
+		_, err = repo.SaveAndEncrypt(pack.Data, buf, nil, false)
 		OK(t, err)
 	}
 }

test/helpers.go

@@ -90,7 +90,6 @@ type rndReader struct {
 }
 
 func (r *rndReader) Read(p []byte) (int, error) {
-	fmt.Printf("Read(%v)\n", len(p))
 	for i := range p {
 		p[i] = byte(r.src.Uint32())
 	}