repo: Split packers for tree and data

The code now bundles tree blobs and data blobs into different pack
files, so we'll end up with pack files that either only contain data or
trees. This is in preparation to adding a cache (#1040), because
tree-only pack files can easily be cached later on.

internal/repository/packer_manager.go

@@ -50,7 +50,7 @@ func newPackerManager(be Saver, key *crypto.Key) *packerManager {
 
 // findPacker returns a packer for a new blob of size bytes. Either a new one is
 // created or one is returned that already has some blobs.
-func (r *packerManager) findPacker(size uint) (packer *Packer, err error) {
+func (r *packerManager) findPacker() (packer *Packer, err error) {
 	r.pm.Lock()
 	defer r.pm.Unlock()
 
@@ -62,7 +62,7 @@ func (r *packerManager) findPacker(size uint) (packer *Packer, err error) {
 	}
 
 	// no suitable packer found, return new
-	debug.Log("create new pack for %d bytes", size)
+	debug.Log("create new pack")
 	tmpfile, err := fs.TempFile("", "restic-temp-pack-")
 	if err != nil {
 		return nil, errors.Wrap(err, "fs.TempFile")

internal/repository/packer_manager_test.go

@@ -72,7 +72,7 @@ func fillPacks(t testing.TB, rnd *randReader, be Saver, pm *packerManager, buf [
 		l := rnd.rand.Intn(1 << 20)
 		seed := rnd.rand.Int63()
 
-		packer, err := pm.findPacker(uint(l))
+		packer, err := pm.findPacker()
 		if err != nil {
 			t.Fatal(err)
 		}

internal/repository/repository.go

@@ -24,15 +24,17 @@ type Repository struct {
 	keyName string
 	idx     *MasterIndex
 
-	*packerManager
+	treePM *packerManager
+	dataPM *packerManager
 }
 
 // New returns a new repository with backend be.
 func New(be restic.Backend) *Repository {
 	repo := &Repository{
-		be:            be,
+		be:     be,
-		idx:           NewMasterIndex(),
+		idx:    NewMasterIndex(),
-		packerManager: newPackerManager(be, nil),
+		dataPM: newPackerManager(be, nil),
+		treePM: newPackerManager(be, nil),
 	}
 
 	return repo
@@ -180,7 +182,18 @@ func (r *Repository) SaveAndEncrypt(ctx context.Context, t restic.BlobType, data
 	}
 
 	// find suitable packer and add blob
-	packer, err := r.findPacker(uint(len(ciphertext)))
+	var pm *packerManager
+
+	switch t {
+	case restic.TreeBlob:
+		pm = r.treePM
+	case restic.DataBlob:
+		pm = r.dataPM
+	default:
+		panic(fmt.Sprintf("invalid type: %v", t))
+	}
+
+	packer, err := pm.findPacker()
 	if err != nil {
 		return restic.ID{}, err
 	}
@@ -194,7 +207,7 @@ func (r *Repository) SaveAndEncrypt(ctx context.Context, t restic.BlobType, data
 	// if the pack is not full enough, put back to the list
 	if packer.Size() < minPackSize {
 		debug.Log("pack is not full enough (%d bytes)", packer.Size())
-		r.insertPacker(packer)
+		pm.insertPacker(packer)
 		return *id, nil
 	}
 
@@ -238,18 +251,22 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, p []by
 
 // Flush saves all remaining packs.
 func (r *Repository) Flush() error {
-	r.pm.Lock()
+	for _, pm := range []*packerManager{r.dataPM, r.treePM} {
-	defer r.pm.Unlock()
+		pm.pm.Lock()
 
-	debug.Log("manually flushing %d packs", len(r.packerManager.packers))
+		debug.Log("manually flushing %d packs", len(pm.packers))
-
+		for _, p := range pm.packers {
-	for _, p := range r.packerManager.packers {
+			err := r.savePacker(p)
-		err := r.savePacker(p)
+			if err != nil {
-		if err != nil {
+				pm.pm.Unlock()
-			return err
+				return err
+			}
 		}
+		pm.packers = pm.packers[:0]
+
+		pm.pm.Unlock()
 	}
-	r.packerManager.packers = r.packerManager.packers[:0]
+
 	return nil
 }
 
@@ -371,7 +388,8 @@ func (r *Repository) SearchKey(ctx context.Context, password string, maxKeys int
 	}
 
 	r.key = key.master
-	r.packerManager.key = key.master
+	r.dataPM.key = key.master
+	r.treePM.key = key.master
 	r.keyName = key.Name()
 	r.cfg, err = restic.LoadConfig(ctx, r)
 	return err
@@ -405,7 +423,8 @@ func (r *Repository) init(ctx context.Context, password string, cfg restic.Confi
 	}
 
 	r.key = key.master
-	r.packerManager.key = key.master
+	r.dataPM.key = key.master
+	r.treePM.key = key.master
 	r.keyName = key.Name()
 	r.cfg = cfg
 	_, err = r.SaveJSONUnpacked(ctx, restic.ConfigFile, cfg)