octoleo/restic
2
2
Fork 0
mirror of https://github.com/octoleo/restic.git synced 2024-11-26 14:56:29 +00:00
Code Issues Releases Wiki Activity

Merge pull request #1635 from ifedorenko/1633-negative-load-offset

Browse Source 
Fixed unexpected 'pack file cannot be listed' error
This commit is contained in:
Alexander Neumann 2018-02-21 19:51:38 +01:00
commit 2c7dd3edf4
3 changed files with 35 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
changelog/0.8.3/issue-1633 Normal file
View File

@ -0,0 +1,6 @@
Bugfix: Fixed unexpected 'pack file cannot be listed' error
Due to regression introduced in 0.8.2, `rebuild-index` and `prune` commands
failed to read pack files with size of 587, 588, 589 or 590 bytes.
https://github.com/restic/restic/issues/1633

15
internal/pack/pack.go
View File

@ -172,8 +172,11 @@ func (p *Packer) String() string {
const maxHeaderSize = 16 * 1024 * 1024 const maxHeaderSize = 16 * 1024 * 1024
// size of the header-length field at the end of the file
var headerLengthSize = binary.Size(uint32(0))
// we require at least one entry in the header, and one blob for a pack file // we require at least one entry in the header, and one blob for a pack file
var minFileSize = entrySize + crypto.Extension var minFileSize = entrySize + crypto.Extension + uint(headerLengthSize)
// number of header enries to download as part of header-length request // number of header enries to download as part of header-length request
var eagerEntries = uint(15) var eagerEntries = uint(15)
@ -197,10 +200,10 @@ func readHeader(rd io.ReaderAt, size int64) ([]byte, error) {
// only make second request if actual number of entries is greater than eagerEntries // only make second request if actual number of entries is greater than eagerEntries
eagerHl := uint32((eagerEntries * entrySize) + crypto.Extension) eagerHl := uint32((eagerEntries * entrySize) + crypto.Extension)
if int64(eagerHl) > size { if int64(eagerHl)+int64(headerLengthSize) > size {
eagerHl = uint32(size) - uint32(binary.Size(uint32(0))) eagerHl = uint32(size) - uint32(headerLengthSize)
} }
eagerBuf := make([]byte, eagerHl+uint32(binary.Size(uint32(0)))) eagerBuf := make([]byte, eagerHl+uint32(headerLengthSize))
n, err := rd.ReadAt(eagerBuf, size-int64(len(eagerBuf))) n, err := rd.ReadAt(eagerBuf, size-int64(len(eagerBuf)))
if err != nil { if err != nil {
@ -228,7 +231,7 @@ func readHeader(rd io.ReaderAt, size int64) ([]byte, error) {
return nil, errors.Wrap(err, "readHeader") return nil, errors.Wrap(err, "readHeader")
} }
if int64(hl) > size-int64(binary.Size(hl)) { if int64(hl) > size-int64(headerLengthSize) {
err := InvalidFileError{Message: "header is larger than file"} err := InvalidFileError{Message: "header is larger than file"}
return nil, errors.Wrap(err, "readHeader") return nil, errors.Wrap(err, "readHeader")
} }
@ -248,7 +251,7 @@ func readHeader(rd io.ReaderAt, size int64) ([]byte, error) {
// need more header bytes // need more header bytes
buf = make([]byte, hl) buf = make([]byte, hl)
missingHl := hl - eagerHl missingHl := hl - eagerHl
n, err := rd.ReadAt(buf[:missingHl], size-int64(hl)-int64(binary.Size(hl))) n, err := rd.ReadAt(buf[:missingHl], size-int64(hl)-int64(headerLengthSize))
if err != nil { if err != nil {
return nil, errors.Wrap(err, "ReadAt") return nil, errors.Wrap(err, "ReadAt")
} }

26
internal/pack/pack_internal_test.go
View File

@ -22,11 +22,11 @@ func (rd *countingReaderAt) ReadAt(p []byte, off int64) (n int, err error) {
func TestReadHeaderEagerLoad(t *testing.T) { func TestReadHeaderEagerLoad(t *testing.T) {
testReadHeader := func(entryCount uint, expectedReadInvocationCount int) { testReadHeader := func(dataSize int, entryCount uint, expectedReadInvocationCount int) {
expectedHeader := rtest.Random(0, int(entryCount*entrySize)+crypto.Extension) expectedHeader := rtest.Random(0, int(entryCount*entrySize)+crypto.Extension)
buf := &bytes.Buffer{} buf := &bytes.Buffer{}
buf.Write(rtest.Random(0, 100)) // pack blobs data buf.Write(rtest.Random(0, dataSize)) // pack blobs data
buf.Write(expectedHeader) // pack header buf.Write(expectedHeader) // pack header
binary.Write(buf, binary.LittleEndian, uint32(len(expectedHeader))) // pack header length binary.Write(buf, binary.LittleEndian, uint32(len(expectedHeader))) // pack header length
@ -39,8 +39,22 @@ func TestReadHeaderEagerLoad(t *testing.T) {
rtest.Equals(t, expectedReadInvocationCount, rd.invocationCount) rtest.Equals(t, expectedReadInvocationCount, rd.invocationCount)
} }
testReadHeader(1, 1) // basic
testReadHeader(eagerEntries-1, 1) testReadHeader(100, 1, 1)
testReadHeader(eagerEntries, 1)
testReadHeader(eagerEntries+1, 2) // header entries == eager entries
testReadHeader(100, eagerEntries-1, 1)
testReadHeader(100, eagerEntries, 1)
testReadHeader(100, eagerEntries+1, 2)
// file size == eager header load size
eagerLoadSize := int((eagerEntries * entrySize) + crypto.Extension)
headerSize := int(1*entrySize) + crypto.Extension
dataSize := eagerLoadSize - headerSize - binary.Size(uint32(0))
testReadHeader(dataSize-1, 1, 1)
testReadHeader(dataSize, 1, 1)
testReadHeader(dataSize+1, 1, 1)
testReadHeader(dataSize+2, 1, 1)
testReadHeader(dataSize+3, 1, 1)
testReadHeader(dataSize+4, 1, 1)
} }