octoleo
/
restic
mirror of https://github.com/octoleo/restic.git
2
2
Fork 0
Code Issues Releases Wiki Activity

Fixed unexpected 'pack file cannot be listed' error 

Fixes #1633

Signed-off-by: Igor Fedorenko <igor@ifedorenko.com>
This commit is contained in:
Igor Fedorenko 2018-02-20 21:28:37 -05:00
parent 9f0605766c
commit 19e7803ac6
3 changed files with 35 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
changelog/0.8.3/issue-1633 Normal file
View File

@ -0,0 +1,6 @@
Bugfix: Fixed unexpected 'pack file cannot be listed' error
Due to regression introduced in 0.8.2, `rebuild-index` and `prune` commands
failed to read pack files with size of 587, 588, 589 or 590 bytes.
https://github.com/restic/restic/issues/1633

15
internal/pack/pack.go
View File

@ -172,8 +172,11 @@ func (p *Packer) String() string {
const maxHeaderSize = 16 * 1024 * 1024
// size of the header-length field at the end of the file
var headerLengthSize = binary.Size(uint32(0))
// we require at least one entry in the header, and one blob for a pack file
var minFileSize = entrySize + crypto.Extension
var minFileSize = entrySize + crypto.Extension + uint(headerLengthSize)
// number of header enries to download as part of header-length request
var eagerEntries = uint(15)
@ -197,10 +200,10 @@ func readHeader(rd io.ReaderAt, size int64) ([]byte, error) {
// only make second request if actual number of entries is greater than eagerEntries
eagerHl := uint32((eagerEntries * entrySize) + crypto.Extension)
if int64(eagerHl) > size {
eagerHl = uint32(size) - uint32(binary.Size(uint32(0)))
if int64(eagerHl)+int64(headerLengthSize) > size {
eagerHl = uint32(size) - uint32(headerLengthSize)
}
eagerBuf := make([]byte, eagerHl+uint32(binary.Size(uint32(0))))
eagerBuf := make([]byte, eagerHl+uint32(headerLengthSize))
n, err := rd.ReadAt(eagerBuf, size-int64(len(eagerBuf)))
if err != nil {
@ -228,7 +231,7 @@ func readHeader(rd io.ReaderAt, size int64) ([]byte, error) {
return nil, errors.Wrap(err, "readHeader")
}
if int64(hl) > size-int64(binary.Size(hl)) {
if int64(hl) > size-int64(headerLengthSize) {
err := InvalidFileError{Message: "header is larger than file"}
return nil, errors.Wrap(err, "readHeader")
}
@ -248,7 +251,7 @@ func readHeader(rd io.ReaderAt, size int64) ([]byte, error) {
// need more header bytes
buf = make([]byte, hl)
missingHl := hl - eagerHl
n, err := rd.ReadAt(buf[:missingHl], size-int64(hl)-int64(binary.Size(hl)))
n, err := rd.ReadAt(buf[:missingHl], size-int64(hl)-int64(headerLengthSize))
if err != nil {
return nil, errors.Wrap(err, "ReadAt")
}

26
internal/pack/pack_internal_test.go
View File

@ -22,11 +22,11 @@ func (rd *countingReaderAt) ReadAt(p []byte, off int64) (n int, err error) {
func TestReadHeaderEagerLoad(t *testing.T) {
testReadHeader := func(entryCount uint, expectedReadInvocationCount int) {
testReadHeader := func(dataSize int, entryCount uint, expectedReadInvocationCount int) {
expectedHeader := rtest.Random(0, int(entryCount*entrySize)+crypto.Extension)
buf := &bytes.Buffer{}
buf.Write(rtest.Random(0, 100)) // pack blobs data
buf.Write(rtest.Random(0, dataSize)) // pack blobs data
buf.Write(expectedHeader) // pack header
binary.Write(buf, binary.LittleEndian, uint32(len(expectedHeader))) // pack header length
@ -39,8 +39,22 @@ func TestReadHeaderEagerLoad(t *testing.T) {
rtest.Equals(t, expectedReadInvocationCount, rd.invocationCount)
}
testReadHeader(1, 1)
testReadHeader(eagerEntries-1, 1)
testReadHeader(eagerEntries, 1)
testReadHeader(eagerEntries+1, 2)
// basic
testReadHeader(100, 1, 1)
// header entries == eager entries
testReadHeader(100, eagerEntries-1, 1)
testReadHeader(100, eagerEntries, 1)
testReadHeader(100, eagerEntries+1, 2)
// file size == eager header load size
eagerLoadSize := int((eagerEntries * entrySize) + crypto.Extension)
headerSize := int(1*entrySize) + crypto.Extension
dataSize := eagerLoadSize - headerSize - binary.Size(uint32(0))
testReadHeader(dataSize-1, 1, 1)
testReadHeader(dataSize, 1, 1)
testReadHeader(dataSize+1, 1, 1)
testReadHeader(dataSize+2, 1, 1)
testReadHeader(dataSize+3, 1, 1)
testReadHeader(dataSize+4, 1, 1)
}