mirror of https://github.com/octoleo/restic.git
Update the chunker
This commit is contained in:
Alexander Neumann
parent
81c0b031f9
commit
b229aa5cf1
2
go.mod
2
go.mod
|
|
@ -26,7 +26,7 @@ require (
|
|||
github.com/pkg/profile v1.3.0
|
||||
github.com/pkg/sftp v1.10.0
|
||||
github.com/pkg/xattr v0.4.1
|
||||
github.com/restic/chunker v0.2.0
|
||||
github.com/restic/chunker v0.3.0
|
||||
github.com/satori/go.uuid v1.2.0 // indirect
|
||||
github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3 // indirect
|
||||
github.com/spf13/cobra v0.0.3
|
||||
|
|
|
|||
4
go.sum
4
go.sum
|
|
@ -132,8 +132,8 @@ github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8
|
|||
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
|
||||
github.com/restic/chunker v0.2.0 h1:GjvmvFuv2mx0iekZs+iAlrioo2UtgsGSSplvoXaVHDU=
|
||||
github.com/restic/chunker v0.2.0/go.mod h1:VdjruEj+7BU1ZZTW8Qqi1exxRx2Omf2JH0NsUEkQ29s=
|
||||
github.com/restic/chunker v0.3.0 h1:8OGNG5ALPTmHTdfuNkwqHqbzifrIc3MeL8CL7q9BY34=
|
||||
github.com/restic/chunker v0.3.0/go.mod h1:VdjruEj+7BU1ZZTW8Qqi1exxRx2Omf2JH0NsUEkQ29s=
|
||||
github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo=
|
||||
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
|
||||
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
|
||||
|
|
|
|||
|
|
@ -1,18 +1,20 @@
|
|||
language: go
|
||||
sudo: false
|
||||
|
||||
go:
|
||||
- 1.6.4
|
||||
- 1.7.4
|
||||
- tip
|
||||
|
||||
os:
|
||||
- linux
|
||||
- osx
|
||||
matrix:
|
||||
include:
|
||||
- os: linux
|
||||
go: "1.9.x"
|
||||
- os: linux
|
||||
go: "1.10.x"
|
||||
- os: linux
|
||||
go: "tip"
|
||||
- os: osx
|
||||
go: "1.10.x"
|
||||
|
||||
install:
|
||||
- go get -t ./...
|
||||
- go get -u github.com/golang/lint/golint
|
||||
- go get -u golang.org/x/lint/golint
|
||||
- go get -u golang.org/x/tools/cmd/goimports
|
||||
|
||||
script:
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
[](http://godoc.org/github.com/restic/chunker)
|
||||
[](https://travis-ci.org/restic/chunker)
|
||||
[](https://travis-ci.com/restic/chunker)
|
||||
|
||||
The package `chunker` implements content-defined-chunking (CDC) based on a
|
||||
rolling Rabin Hash. The library is part of the [restic backup
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ type Chunk struct {
|
|||
|
||||
type chunkerState struct {
|
||||
window [windowSize]byte
|
||||
wpos int
|
||||
wpos uint
|
||||
|
||||
buf []byte
|
||||
bpos uint
|
||||
|
|
@ -225,6 +225,12 @@ func (c *Chunker) Next(data []byte) (Chunk, error) {
|
|||
tabout := c.tables.out
|
||||
tabmod := c.tables.mod
|
||||
polShift := c.polShift
|
||||
// go guarantees the expected behavior for bit shifts even for shift counts
|
||||
// larger than the value width. Bounding the value of polShift allows the compiler
|
||||
// to optimize the code for 'digest >> polShift'
|
||||
if polShift > 53-8 {
|
||||
return Chunk{}, errors.New("the polynomial must have a degree less than or equal 53")
|
||||
}
|
||||
minSize := c.MinSize
|
||||
maxSize := c.MaxSize
|
||||
buf := c.buf
|
||||
|
|
@ -291,10 +297,12 @@ func (c *Chunker) Next(data []byte) (Chunk, error) {
|
|||
wpos := c.wpos
|
||||
for _, b := range buf[c.bpos:c.bmax] {
|
||||
// slide(b)
|
||||
// limit wpos before to elide array bound checks
|
||||
wpos = wpos % windowSize
|
||||
out := win[wpos]
|
||||
win[wpos] = b
|
||||
digest ^= uint64(tabout[out])
|
||||
wpos = (wpos + 1) % windowSize
|
||||
wpos++
|
||||
|
||||
// updateDigest
|
||||
index := byte(digest >> polShift)
|
||||
|
|
@ -331,7 +339,7 @@ func (c *Chunker) Next(data []byte) (Chunk, error) {
|
|||
}
|
||||
c.digest = digest
|
||||
c.window = win
|
||||
c.wpos = wpos
|
||||
c.wpos = wpos % windowSize
|
||||
|
||||
steps := c.bmax - c.bpos
|
||||
if steps > 0 {
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
module github.com/restic/chunker
|
||||
|
|
@ -94,7 +94,6 @@ func (x Pol) Deg() int {
|
|||
}
|
||||
|
||||
if uint64(x)&0x2 > 0 {
|
||||
x >>= 1
|
||||
r |= 1
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -78,7 +78,7 @@ github.com/pkg/profile
|
|||
github.com/pkg/sftp
|
||||
# github.com/pkg/xattr v0.4.1
|
||||
github.com/pkg/xattr
|
||||
# github.com/restic/chunker v0.2.0
|
||||
# github.com/restic/chunker v0.3.0
|
||||
github.com/restic/chunker
|
||||
# github.com/russross/blackfriday v1.5.2
|
||||
github.com/russross/blackfriday
|
||||
|
|
|
|||
Loading…
Reference in New Issue