octoleo/restic
2
2
Fork 0
mirror of https://github.com/octoleo/restic.git synced 2024-12-28 04:56:04 +00:00
Code Issues Releases Wiki Activity

Reduce memory usage while decoding index

Browse Source
This commit is contained in:
Alexander Neumann 2017-01-13 22:05:34 +01:00
parent 73e7a2bea8
commit caabc4ec44
2 changed files with 12 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
src/restic/repository/index.go
View File

@ -1,7 +1,6 @@
package repository
import (
"bytes"
"encoding/json"
"io"
"restic"
@ -451,12 +450,11 @@ func isErrOldIndex(err error) bool {
var ErrOldIndexFormat = errors.New("index has old format")
// DecodeIndex loads and unserializes an index from rd.
func DecodeIndex(rd io.Reader) (idx *Index, err error) {
func DecodeIndex(buf []byte) (idx *Index, err error) {
debug.Log("Start decoding index")
idxJSON := jsonIndex{}
idxJSON := &jsonIndex{}
dec := json.NewDecoder(rd)
err = dec.Decode(&idxJSON)
err = json.Unmarshal(buf, idxJSON)
if err != nil {
debug.Log("Error %v", err)
@ -490,12 +488,11 @@ func DecodeIndex(rd io.Reader) (idx *Index, err error) {
}
// DecodeOldIndex loads and unserializes an index in the old format from rd.
func DecodeOldIndex(rd io.Reader) (idx *Index, err error) {
func DecodeOldIndex(buf []byte) (idx *Index, err error) {
debug.Log("Start decoding old index")
list := []*packJSON{}
dec := json.NewDecoder(rd)
err = dec.Decode(&list)
err = json.Unmarshal(buf, &list)
if err != nil {
debug.Log("Error %#v", err)
return nil, errors.Wrap(err, "Decode")
@ -522,7 +519,7 @@ func DecodeOldIndex(rd io.Reader) (idx *Index, err error) {
}
// LoadIndexWithDecoder loads the index and decodes it with fn.
func LoadIndexWithDecoder(repo restic.Repository, id restic.ID, fn func(io.Reader) (*Index, error)) (idx *Index, err error) {
func LoadIndexWithDecoder(repo restic.Repository, id restic.ID, fn func([]byte) (*Index, error)) (idx *Index, err error) {
debug.Log("Loading index %v", id.Str())
buf, err := repo.LoadAndDecrypt(restic.IndexFile, id)
@ -530,7 +527,7 @@ func LoadIndexWithDecoder(repo restic.Repository, id restic.ID, fn func(io.Reade
return nil, err
}
idx, err = fn(bytes.NewReader(buf))
idx, err = fn(buf)
if err != nil {
debug.Log("error while decoding index %v: %v", id, err)
return nil, err

14
src/restic/repository/index_test.go
View File

@ -54,7 +54,7 @@ func TestIndexSerialize(t *testing.T) {
err := idx.Encode(wr)
OK(t, err)
idx2, err := repository.DecodeIndex(wr)
idx2, err := repository.DecodeIndex(wr.Bytes())
OK(t, err)
Assert(t, idx2 != nil,
"nil returned for decoded index")
@ -136,7 +136,7 @@ func TestIndexSerialize(t *testing.T) {
Assert(t, id2.Equal(id),
"wrong ID returned: want %v, got %v", id, id2)
idx3, err := repository.DecodeIndex(wr3)
idx3, err := repository.DecodeIndex(wr3.Bytes())
OK(t, err)
Assert(t, idx3 != nil,
"nil returned for decoded index")
@ -288,7 +288,7 @@ var exampleLookupTest = struct {
func TestIndexUnserialize(t *testing.T) {
oldIdx := restic.IDs{restic.TestParseID("ed54ae36197f4745ebc4b54d10e0f623eaaaedd03013eb7ae90df881b7781452")}
idx, err := repository.DecodeIndex(bytes.NewReader(docExample))
idx, err := repository.DecodeIndex(docExample)
OK(t, err)
for _, test := range exampleTests {
@ -327,20 +327,16 @@ func TestIndexUnserialize(t *testing.T) {
}
func BenchmarkDecodeIndex(b *testing.B) {
rd := bytes.NewReader(docExample)
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, err := rd.Seek(0, 0)
OK(b, err)
_, err = repository.DecodeIndex(rd)
_, err := repository.DecodeIndex(docExample)
OK(b, err)
}
}
func TestIndexUnserializeOld(t *testing.T) {
idx, err := repository.DecodeOldIndex(bytes.NewReader(docOldExample))
idx, err := repository.DecodeOldIndex(docOldExample)
OK(t, err)
for _, test := range exampleTests {