mirror of
https://github.com/octoleo/restic.git
synced 2024-12-23 11:28:54 +00:00
commit
4615bdfd70
7
changelog/unreleased/pull-2576
Normal file
7
changelog/unreleased/pull-2576
Normal file
@ -0,0 +1,7 @@
|
|||||||
|
Enhancement: Improve the chunking algorithm
|
||||||
|
|
||||||
|
We've updated the chunker library responsible for splitting files into smaller
|
||||||
|
blocks. It should improve the chunking throughput by 5-10% depending on the
|
||||||
|
CPU.
|
||||||
|
|
||||||
|
https://github.com/restic/restic/pull/2576
|
2
go.mod
2
go.mod
@ -26,7 +26,7 @@ require (
|
|||||||
github.com/pkg/profile v1.3.0
|
github.com/pkg/profile v1.3.0
|
||||||
github.com/pkg/sftp v1.10.0
|
github.com/pkg/sftp v1.10.0
|
||||||
github.com/pkg/xattr v0.4.1
|
github.com/pkg/xattr v0.4.1
|
||||||
github.com/restic/chunker v0.2.0
|
github.com/restic/chunker v0.3.0
|
||||||
github.com/satori/go.uuid v1.2.0 // indirect
|
github.com/satori/go.uuid v1.2.0 // indirect
|
||||||
github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3 // indirect
|
github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3 // indirect
|
||||||
github.com/spf13/cobra v0.0.3
|
github.com/spf13/cobra v0.0.3
|
||||||
|
4
go.sum
4
go.sum
@ -132,8 +132,8 @@ github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8
|
|||||||
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||||
github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||||
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
|
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
|
||||||
github.com/restic/chunker v0.2.0 h1:GjvmvFuv2mx0iekZs+iAlrioo2UtgsGSSplvoXaVHDU=
|
github.com/restic/chunker v0.3.0 h1:8OGNG5ALPTmHTdfuNkwqHqbzifrIc3MeL8CL7q9BY34=
|
||||||
github.com/restic/chunker v0.2.0/go.mod h1:VdjruEj+7BU1ZZTW8Qqi1exxRx2Omf2JH0NsUEkQ29s=
|
github.com/restic/chunker v0.3.0/go.mod h1:VdjruEj+7BU1ZZTW8Qqi1exxRx2Omf2JH0NsUEkQ29s=
|
||||||
github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo=
|
github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo=
|
||||||
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
|
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
|
||||||
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
|
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
|
||||||
|
20
vendor/github.com/restic/chunker/.travis.yml
generated
vendored
20
vendor/github.com/restic/chunker/.travis.yml
generated
vendored
@ -1,18 +1,20 @@
|
|||||||
language: go
|
language: go
|
||||||
sudo: false
|
sudo: false
|
||||||
|
|
||||||
go:
|
matrix:
|
||||||
- 1.6.4
|
include:
|
||||||
- 1.7.4
|
- os: linux
|
||||||
- tip
|
go: "1.9.x"
|
||||||
|
- os: linux
|
||||||
os:
|
go: "1.10.x"
|
||||||
- linux
|
- os: linux
|
||||||
- osx
|
go: "tip"
|
||||||
|
- os: osx
|
||||||
|
go: "1.10.x"
|
||||||
|
|
||||||
install:
|
install:
|
||||||
- go get -t ./...
|
- go get -t ./...
|
||||||
- go get -u github.com/golang/lint/golint
|
- go get -u golang.org/x/lint/golint
|
||||||
- go get -u golang.org/x/tools/cmd/goimports
|
- go get -u golang.org/x/tools/cmd/goimports
|
||||||
|
|
||||||
script:
|
script:
|
||||||
|
2
vendor/github.com/restic/chunker/README.md
generated
vendored
2
vendor/github.com/restic/chunker/README.md
generated
vendored
@ -1,5 +1,5 @@
|
|||||||
[![GoDoc](https://godoc.org/github.com/restic/chunker?status.svg)](http://godoc.org/github.com/restic/chunker)
|
[![GoDoc](https://godoc.org/github.com/restic/chunker?status.svg)](http://godoc.org/github.com/restic/chunker)
|
||||||
[![Build Status](https://travis-ci.org/restic/chunker.svg?branch=master)](https://travis-ci.org/restic/chunker)
|
[![Build Status](https://travis-ci.com/restic/chunker.svg?branch=master)](https://travis-ci.com/restic/chunker)
|
||||||
|
|
||||||
The package `chunker` implements content-defined-chunking (CDC) based on a
|
The package `chunker` implements content-defined-chunking (CDC) based on a
|
||||||
rolling Rabin Hash. The library is part of the [restic backup
|
rolling Rabin Hash. The library is part of the [restic backup
|
||||||
|
14
vendor/github.com/restic/chunker/chunker.go
generated
vendored
14
vendor/github.com/restic/chunker/chunker.go
generated
vendored
@ -47,7 +47,7 @@ type Chunk struct {
|
|||||||
|
|
||||||
type chunkerState struct {
|
type chunkerState struct {
|
||||||
window [windowSize]byte
|
window [windowSize]byte
|
||||||
wpos int
|
wpos uint
|
||||||
|
|
||||||
buf []byte
|
buf []byte
|
||||||
bpos uint
|
bpos uint
|
||||||
@ -225,6 +225,12 @@ func (c *Chunker) Next(data []byte) (Chunk, error) {
|
|||||||
tabout := c.tables.out
|
tabout := c.tables.out
|
||||||
tabmod := c.tables.mod
|
tabmod := c.tables.mod
|
||||||
polShift := c.polShift
|
polShift := c.polShift
|
||||||
|
// go guarantees the expected behavior for bit shifts even for shift counts
|
||||||
|
// larger than the value width. Bounding the value of polShift allows the compiler
|
||||||
|
// to optimize the code for 'digest >> polShift'
|
||||||
|
if polShift > 53-8 {
|
||||||
|
return Chunk{}, errors.New("the polynomial must have a degree less than or equal 53")
|
||||||
|
}
|
||||||
minSize := c.MinSize
|
minSize := c.MinSize
|
||||||
maxSize := c.MaxSize
|
maxSize := c.MaxSize
|
||||||
buf := c.buf
|
buf := c.buf
|
||||||
@ -291,10 +297,12 @@ func (c *Chunker) Next(data []byte) (Chunk, error) {
|
|||||||
wpos := c.wpos
|
wpos := c.wpos
|
||||||
for _, b := range buf[c.bpos:c.bmax] {
|
for _, b := range buf[c.bpos:c.bmax] {
|
||||||
// slide(b)
|
// slide(b)
|
||||||
|
// limit wpos before to elide array bound checks
|
||||||
|
wpos = wpos % windowSize
|
||||||
out := win[wpos]
|
out := win[wpos]
|
||||||
win[wpos] = b
|
win[wpos] = b
|
||||||
digest ^= uint64(tabout[out])
|
digest ^= uint64(tabout[out])
|
||||||
wpos = (wpos + 1) % windowSize
|
wpos++
|
||||||
|
|
||||||
// updateDigest
|
// updateDigest
|
||||||
index := byte(digest >> polShift)
|
index := byte(digest >> polShift)
|
||||||
@ -331,7 +339,7 @@ func (c *Chunker) Next(data []byte) (Chunk, error) {
|
|||||||
}
|
}
|
||||||
c.digest = digest
|
c.digest = digest
|
||||||
c.window = win
|
c.window = win
|
||||||
c.wpos = wpos
|
c.wpos = wpos % windowSize
|
||||||
|
|
||||||
steps := c.bmax - c.bpos
|
steps := c.bmax - c.bpos
|
||||||
if steps > 0 {
|
if steps > 0 {
|
||||||
|
1
vendor/github.com/restic/chunker/go.mod
generated
vendored
Normal file
1
vendor/github.com/restic/chunker/go.mod
generated
vendored
Normal file
@ -0,0 +1 @@
|
|||||||
|
module github.com/restic/chunker
|
1
vendor/github.com/restic/chunker/polynomials.go
generated
vendored
1
vendor/github.com/restic/chunker/polynomials.go
generated
vendored
@ -94,7 +94,6 @@ func (x Pol) Deg() int {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if uint64(x)&0x2 > 0 {
|
if uint64(x)&0x2 > 0 {
|
||||||
x >>= 1
|
|
||||||
r |= 1
|
r |= 1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
2
vendor/modules.txt
vendored
2
vendor/modules.txt
vendored
@ -78,7 +78,7 @@ github.com/pkg/profile
|
|||||||
github.com/pkg/sftp
|
github.com/pkg/sftp
|
||||||
# github.com/pkg/xattr v0.4.1
|
# github.com/pkg/xattr v0.4.1
|
||||||
github.com/pkg/xattr
|
github.com/pkg/xattr
|
||||||
# github.com/restic/chunker v0.2.0
|
# github.com/restic/chunker v0.3.0
|
||||||
github.com/restic/chunker
|
github.com/restic/chunker
|
||||||
# github.com/russross/blackfriday v1.5.2
|
# github.com/russross/blackfriday v1.5.2
|
||||||
github.com/russross/blackfriday
|
github.com/russross/blackfriday
|
||||||
|
Loading…
Reference in New Issue
Block a user