octoleo/restic
2
2
Fork 0
mirror of https://github.com/octoleo/restic.git synced 2024-12-28 21:02:47 +00:00
Code Issues Releases Wiki Activity

Reduce memory usage while decoding index

Browse Source
This commit is contained in:
Alexander Neumann 2017-01-13 22:05:34 +01:00
parent 73e7a2bea8
commit caabc4ec44
2 changed files with 12 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
src/restic/repository/index.go
View File

@ -1,7 +1,6 @@
package repository package repository
import ( import (
"bytes"
"encoding/json" "encoding/json"
"io" "io"
"restic" "restic"
@ -451,12 +450,11 @@ func isErrOldIndex(err error) bool {
var ErrOldIndexFormat = errors.New("index has old format") var ErrOldIndexFormat = errors.New("index has old format")
// DecodeIndex loads and unserializes an index from rd. // DecodeIndex loads and unserializes an index from rd.
func DecodeIndex(rd io.Reader) (idx *Index, err error) { func DecodeIndex(buf []byte) (idx *Index, err error) {
debug.Log("Start decoding index") debug.Log("Start decoding index")
idxJSON := jsonIndex{} idxJSON := &jsonIndex{}
dec := json.NewDecoder(rd) err = json.Unmarshal(buf, idxJSON)
err = dec.Decode(&idxJSON)
if err != nil { if err != nil {
debug.Log("Error %v", err) debug.Log("Error %v", err)
@ -490,12 +488,11 @@ func DecodeIndex(rd io.Reader) (idx *Index, err error) {
} }
// DecodeOldIndex loads and unserializes an index in the old format from rd. // DecodeOldIndex loads and unserializes an index in the old format from rd.
func DecodeOldIndex(rd io.Reader) (idx *Index, err error) { func DecodeOldIndex(buf []byte) (idx *Index, err error) {
debug.Log("Start decoding old index") debug.Log("Start decoding old index")
list := []*packJSON{} list := []*packJSON{}
dec := json.NewDecoder(rd) err = json.Unmarshal(buf, &list)
err = dec.Decode(&list)
if err != nil { if err != nil {
debug.Log("Error %#v", err) debug.Log("Error %#v", err)
return nil, errors.Wrap(err, "Decode") return nil, errors.Wrap(err, "Decode")
@ -522,7 +519,7 @@ func DecodeOldIndex(rd io.Reader) (idx *Index, err error) {
} }
// LoadIndexWithDecoder loads the index and decodes it with fn. // LoadIndexWithDecoder loads the index and decodes it with fn.
func LoadIndexWithDecoder(repo restic.Repository, id restic.ID, fn func(io.Reader) (*Index, error)) (idx *Index, err error) { func LoadIndexWithDecoder(repo restic.Repository, id restic.ID, fn func([]byte) (*Index, error)) (idx *Index, err error) {
debug.Log("Loading index %v", id.Str()) debug.Log("Loading index %v", id.Str())
buf, err := repo.LoadAndDecrypt(restic.IndexFile, id) buf, err := repo.LoadAndDecrypt(restic.IndexFile, id)
@ -530,7 +527,7 @@ func LoadIndexWithDecoder(repo restic.Repository, id restic.ID, fn func(io.Reade
return nil, err return nil, err
} }
idx, err = fn(bytes.NewReader(buf)) idx, err = fn(buf)
if err != nil { if err != nil {
debug.Log("error while decoding index %v: %v", id, err) debug.Log("error while decoding index %v: %v", id, err)
return nil, err return nil, err

14
src/restic/repository/index_test.go
View File

@ -54,7 +54,7 @@ func TestIndexSerialize(t *testing.T) {
err := idx.Encode(wr) err := idx.Encode(wr)
OK(t, err) OK(t, err)
idx2, err := repository.DecodeIndex(wr) idx2, err := repository.DecodeIndex(wr.Bytes())
OK(t, err) OK(t, err)
Assert(t, idx2 != nil, Assert(t, idx2 != nil,
"nil returned for decoded index") "nil returned for decoded index")
@ -136,7 +136,7 @@ func TestIndexSerialize(t *testing.T) {
Assert(t, id2.Equal(id), Assert(t, id2.Equal(id),
"wrong ID returned: want %v, got %v", id, id2) "wrong ID returned: want %v, got %v", id, id2)
idx3, err := repository.DecodeIndex(wr3) idx3, err := repository.DecodeIndex(wr3.Bytes())
OK(t, err) OK(t, err)
Assert(t, idx3 != nil, Assert(t, idx3 != nil,
"nil returned for decoded index") "nil returned for decoded index")
@ -288,7 +288,7 @@ var exampleLookupTest = struct {
func TestIndexUnserialize(t *testing.T) { func TestIndexUnserialize(t *testing.T) {
oldIdx := restic.IDs{restic.TestParseID("ed54ae36197f4745ebc4b54d10e0f623eaaaedd03013eb7ae90df881b7781452")} oldIdx := restic.IDs{restic.TestParseID("ed54ae36197f4745ebc4b54d10e0f623eaaaedd03013eb7ae90df881b7781452")}
idx, err := repository.DecodeIndex(bytes.NewReader(docExample)) idx, err := repository.DecodeIndex(docExample)
OK(t, err) OK(t, err)
for _, test := range exampleTests { for _, test := range exampleTests {
@ -327,20 +327,16 @@ func TestIndexUnserialize(t *testing.T) {
} }
func BenchmarkDecodeIndex(b *testing.B) { func BenchmarkDecodeIndex(b *testing.B) {
rd := bytes.NewReader(docExample)
b.ResetTimer() b.ResetTimer()
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
_, err := rd.Seek(0, 0) _, err := repository.DecodeIndex(docExample)
OK(b, err)
_, err = repository.DecodeIndex(rd)
OK(b, err) OK(b, err)
} }
} }
func TestIndexUnserializeOld(t *testing.T) { func TestIndexUnserializeOld(t *testing.T) {
idx, err := repository.DecodeOldIndex(bytes.NewReader(docOldExample)) idx, err := repository.DecodeOldIndex(docOldExample)
OK(t, err) OK(t, err)
for _, test := range exampleTests { for _, test := range exampleTests {