Reduce memory usage for fuse mount

This changes `repository.LoadBlob()` so that a destination buffer must
be provided, which enables the fuse code to use a buffer from a
`sync.Pool`. In addition, release the buffers when the file is closed.
At the moment, the max memory usage is defined by the max file size that
is read in one go (e.g. with `cat`). It could be further optimized by
implementing a LRU caching scheme.

cmd/restic/cmd_cat.go

@@ -162,9 +162,18 @@ func (cmd CmdCat) Execute(args []string) error {
 		return err
 
 	case "blob":
-		data, err := repo.LoadBlob(pack.Data, id)
+		_, tpe, _, length, err := repo.Index().Lookup(id)
-		if err == nil {
+		if err != nil {
-			_, err = os.Stdout.Write(data)
+			return err
+		}
+
+		if tpe != pack.Data {
+			return errors.New("wrong type for blob")
+		}
+
+		buf := make([]byte, length)
+		data, err := repo.LoadBlob(pack.Data, id, buf)
+		if err != nil {
 			return err
 		}
 

cmd/restic/fuse/file.go

@@ -1,6 +1,8 @@
 package fuse
 
 import (
+	"sync"
+
 	"github.com/restic/restic"
 	"github.com/restic/restic/pack"
 	"github.com/restic/restic/repository"
@@ -12,6 +14,7 @@ import (
 
 // Statically ensure that *file implements the given interface
 var _ = fs.HandleReader(&file{})
+var _ = fs.HandleReleaser(&file{})
 
 type file struct {
 	repo *repository.Repository
@@ -21,6 +24,14 @@ type file struct {
 	blobs [][]byte
 }
 
+const defaultBlobSize = 128 * 1024
+
+var blobPool = sync.Pool{
+	New: func() interface{} {
+		return make([]byte, defaultBlobSize)
+	},
+}
+
 func newFile(repo *repository.Repository, node *restic.Node) (*file, error) {
 	sizes := make([]uint32, len(node.Content))
 	for i, blobID := range node.Content {
@@ -48,50 +59,69 @@ func (f *file) Attr(ctx context.Context, a *fuse.Attr) error {
 
 func (f *file) getBlobAt(i int) (blob []byte, err error) {
 	if f.blobs[i] != nil {
-		blob = f.blobs[i]
+		return f.blobs[i], nil
-	} else {
-		blob, err = f.repo.LoadBlob(pack.Data, f.node.Content[i])
-		if err != nil {
-			return nil, err
-		}
-		f.blobs[i] = blob
 	}
 
+	buf := blobPool.Get().([]byte)
+	buf = buf[:cap(buf)]
+
+	if uint32(len(buf)) < f.sizes[i] {
+		if len(buf) > defaultBlobSize {
+			blobPool.Put(buf)
+		}
+		buf = make([]byte, f.sizes[i])
+	}
+
+	blob, err = f.repo.LoadBlob(pack.Data, f.node.Content[i], buf)
+	if err != nil {
+		return nil, err
+	}
+	f.blobs[i] = blob
+
 	return blob, nil
 }
 
 func (f *file) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error {
-	off := req.Offset
+	offset := req.Offset
 
 	// Skip blobs before the offset
 	startContent := 0
-	for off > int64(f.sizes[startContent]) {
+	for offset > int64(f.sizes[startContent]) {
-		off -= int64(f.sizes[startContent])
+		offset -= int64(f.sizes[startContent])
 		startContent++
 	}
 
-	content := make([]byte, req.Size)
+	dst := resp.Data[0:req.Size]
-	allContent := content
+	readBytes := 0
-	for i := startContent; i < len(f.sizes); i++ {
+	remainingBytes := req.Size
+	for i := startContent; remainingBytes > 0 && i < len(f.sizes); i++ {
 		blob, err := f.getBlobAt(i)
 		if err != nil {
 			return err
 		}
 
-		blob = blob[off:]
+		if offset > 0 {
-		off = 0
+			blob = blob[offset:len(blob)]
+			offset = 0
+		}
 
-		var copied int
+		copied := copy(dst, blob)
-		if len(blob) > len(content) {
+		remainingBytes -= copied
-			copied = copy(content[0:], blob[:len(content)])
+		readBytes += copied
-		} else {
+
-			copied = copy(content[0:], blob)
+		dst = dst[copied:]
-		}
+	}
-		content = content[copied:]
+	resp.Data = resp.Data[:readBytes]
-		if len(content) == 0 {
+
-			break
+	return nil
-		}
+}
+
+func (f *file) Release(ctx context.Context, req *fuse.ReleaseRequest) error {
+	for i := range f.blobs {
+		if f.blobs[i] != nil {
+			blobPool.Put(f.blobs[i])
+			f.blobs[i] = nil
+		}
 	}
-	resp.Data = allContent
 	return nil
 }

node.go

@@ -209,8 +209,19 @@ func (node Node) createFileAt(path string, repo *repository.Repository) error {
 		return errors.Annotate(err, "OpenFile")
 	}
 
+	var buf []byte
 	for _, id := range node.Content {
-		buf, err := repo.LoadBlob(pack.Data, id)
+		_, _, _, length, err := repo.Index().Lookup(id)
+		if err != nil {
+			return err
+		}
+
+		buf = buf[:cap(buf)]
+		if uint(len(buf)) < length {
+			buf = make([]byte, length)
+		}
+
+		buf, err := repo.LoadBlob(pack.Data, id, buf)
 		if err != nil {
 			return errors.Annotate(err, "Load")
 		}

repository/repository.go

@@ -55,7 +55,6 @@ func (r *Repository) PrefixLength(t backend.Type) (int, error) {
 func (r *Repository) LoadAndDecrypt(t backend.Type, id backend.ID) ([]byte, error) {
 	debug.Log("Repo.Load", "load %v with id %v", t, id.Str())
 
-	// load blob from pack
 	rd, err := r.be.Get(t, id.String())
 	if err != nil {
 		debug.Log("Repo.Load", "error loading %v: %v", id.Str(), err)
@@ -87,8 +86,9 @@ func (r *Repository) LoadAndDecrypt(t backend.Type, id backend.ID) ([]byte, erro
 }
 
 // LoadBlob tries to load and decrypt content identified by t and id from a
-// pack from the backend.
+// pack from the backend, the result is stored in buf, which must be large
-func (r *Repository) LoadBlob(t pack.BlobType, id backend.ID) ([]byte, error) {
+// enough to hold the complete blob.
+func (r *Repository) LoadBlob(t pack.BlobType, id backend.ID, buf []byte) ([]byte, error) {
 	debug.Log("Repo.LoadBlob", "load %v with id %v", t, id.Str())
 	// lookup pack
 	packID, tpe, offset, length, err := r.idx.Lookup(id)
@@ -97,6 +97,10 @@ func (r *Repository) LoadBlob(t pack.BlobType, id backend.ID) ([]byte, error) {
 		return nil, err
 	}
 
+	if length > uint(cap(buf))+crypto.Extension {
+		return nil, errors.New("buf is too small")
+	}
+
 	if tpe != t {
 		debug.Log("Repo.LoadBlob", "wrong type returned for %v: wanted %v, got %v", id.Str(), t, tpe)
 		return nil, fmt.Errorf("blob has wrong type %v (wanted: %v)", tpe, t)
@@ -111,7 +115,9 @@ func (r *Repository) LoadBlob(t pack.BlobType, id backend.ID) ([]byte, error) {
 		return nil, err
 	}
 
-	buf, err := ioutil.ReadAll(rd)
+	// make buffer that is large enough for the complete blob
+	cbuf := make([]byte, length)
+	_, err = io.ReadFull(rd, cbuf)
 	if err != nil {
 		return nil, err
 	}
@@ -122,17 +128,17 @@ func (r *Repository) LoadBlob(t pack.BlobType, id backend.ID) ([]byte, error) {
 	}
 
 	// decrypt
-	plain, err := r.Decrypt(buf)
+	buf, err = r.decryptTo(buf, cbuf)
 	if err != nil {
 		return nil, err
 	}
 
 	// check hash
-	if !backend.Hash(plain).Equal(id) {
+	if !backend.Hash(buf).Equal(id) {
 		return nil, errors.New("invalid data returned")
 	}
 
-	return plain, nil
+	return buf, nil
 }
 
 // LoadJSONUnpacked decrypts the data and afterwards calls json.Unmarshal on
@@ -580,6 +586,12 @@ func (r *Repository) Init(password string) error {
 
 // Decrypt authenticates and decrypts ciphertext and returns the plaintext.
 func (r *Repository) Decrypt(ciphertext []byte) ([]byte, error) {
+	return r.decryptTo(nil, ciphertext)
+}
+
+// decrypt authenticates and decrypts ciphertext and stores the result in
+// plaintext.
+func (r *Repository) decryptTo(plaintext, ciphertext []byte) ([]byte, error) {
 	if r.key == nil {
 		return nil, errors.New("key for repository not set")
 	}

repository/repository_test.go

@@ -90,7 +90,7 @@ func TestSave(t *testing.T) {
 		OK(t, repo.Flush())
 
 		// read back
-		buf, err := repo.LoadBlob(pack.Data, id)
+		buf, err := repo.LoadBlob(pack.Data, id, make([]byte, size))
 
 		Assert(t, len(buf) == len(data),
 			"number of bytes read back does not match: expected %d, got %d",
@@ -120,7 +120,7 @@ func TestSaveFrom(t *testing.T) {
 		OK(t, repo.Flush())
 
 		// read back
-		buf, err := repo.LoadBlob(pack.Data, id)
+		buf, err := repo.LoadBlob(pack.Data, id, make([]byte, size))
 
 		Assert(t, len(buf) == len(data),
 			"number of bytes read back does not match: expected %d, got %d",