Merge pull request #2576 from restic/update-chunker

Update the chunker

changelog/unreleased/pull-2576

@@ -0,0 +1,7 @@
+Enhancement: Improve the chunking algorithm
+
+We've updated the chunker library responsible for splitting files into smaller
+blocks. It should improve the chunking throughput by 5-10% depending on the
+CPU.
+
+https://github.com/restic/restic/pull/2576

go.mod

@@ -26,7 +26,7 @@ require (
 	github.com/pkg/profile v1.3.0
 	github.com/pkg/sftp v1.10.0
 	github.com/pkg/xattr v0.4.1
-	github.com/restic/chunker v0.2.0
+	github.com/restic/chunker v0.3.0
 	github.com/satori/go.uuid v1.2.0 // indirect
 	github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3 // indirect
 	github.com/spf13/cobra v0.0.3

go.sum

@@ -132,8 +132,8 @@ github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
-github.com/restic/chunker v0.2.0 h1:GjvmvFuv2mx0iekZs+iAlrioo2UtgsGSSplvoXaVHDU=
-github.com/restic/chunker v0.2.0/go.mod h1:VdjruEj+7BU1ZZTW8Qqi1exxRx2Omf2JH0NsUEkQ29s=
+github.com/restic/chunker v0.3.0 h1:8OGNG5ALPTmHTdfuNkwqHqbzifrIc3MeL8CL7q9BY34=
+github.com/restic/chunker v0.3.0/go.mod h1:VdjruEj+7BU1ZZTW8Qqi1exxRx2Omf2JH0NsUEkQ29s=
 github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=

vendor/github.com/restic/chunker/.travis.yml

@@ -1,18 +1,20 @@
 language: go
 sudo: false
 
-go:
-  - 1.6.4
-  - 1.7.4
-  - tip
-
-os:
-  - linux
-  - osx
+matrix:
+  include:
+    - os: linux
+      go: "1.9.x"
+    - os: linux
+      go: "1.10.x"
+    - os: linux
+      go: "tip"
+    - os: osx
+      go: "1.10.x"
 
 install:
   - go get -t ./...
-  - go get -u github.com/golang/lint/golint
+  - go get -u golang.org/x/lint/golint
   - go get -u golang.org/x/tools/cmd/goimports
 
 script:

vendor/github.com/restic/chunker/README.md

@@ -1,5 +1,5 @@
 [![GoDoc](https://godoc.org/github.com/restic/chunker?status.svg)](http://godoc.org/github.com/restic/chunker)
-[![Build Status](https://travis-ci.org/restic/chunker.svg?branch=master)](https://travis-ci.org/restic/chunker)
+[![Build Status](https://travis-ci.com/restic/chunker.svg?branch=master)](https://travis-ci.com/restic/chunker)
 
 The package `chunker` implements content-defined-chunking (CDC) based on a
 rolling Rabin Hash. The library is part of the [restic backup

vendor/github.com/restic/chunker/chunker.go

@@ -47,7 +47,7 @@ type Chunk struct {
 
 type chunkerState struct {
 	window [windowSize]byte
-	wpos   int
+	wpos   uint
 
 	buf  []byte
 	bpos uint
@@ -225,6 +225,12 @@ func (c *Chunker) Next(data []byte) (Chunk, error) {
 	tabout := c.tables.out
 	tabmod := c.tables.mod
 	polShift := c.polShift
+	// go guarantees the expected behavior for bit shifts even for shift counts
+	// larger than the value width. Bounding the value of polShift allows the compiler
+	// to optimize the code for 'digest >> polShift'
+	if polShift > 53-8 {
+		return Chunk{}, errors.New("the polynomial must have a degree less than or equal 53")
+	}
 	minSize := c.MinSize
 	maxSize := c.MaxSize
 	buf := c.buf
@@ -291,10 +297,12 @@ func (c *Chunker) Next(data []byte) (Chunk, error) {
 		wpos := c.wpos
 		for _, b := range buf[c.bpos:c.bmax] {
 			// slide(b)
+			// limit wpos before to elide array bound checks
+			wpos = wpos % windowSize
 			out := win[wpos]
 			win[wpos] = b
 			digest ^= uint64(tabout[out])
-			wpos = (wpos + 1) % windowSize
+			wpos++
 
 			// updateDigest
 			index := byte(digest >> polShift)
@@ -331,7 +339,7 @@ func (c *Chunker) Next(data []byte) (Chunk, error) {
 		}
 		c.digest = digest
 		c.window = win
-		c.wpos = wpos
+		c.wpos = wpos % windowSize
 
 		steps := c.bmax - c.bpos
 		if steps > 0 {

vendor/github.com/restic/chunker/go.mod

@@ -0,0 +1 @@
+module github.com/restic/chunker

vendor/github.com/restic/chunker/polynomials.go

@@ -94,7 +94,6 @@ func (x Pol) Deg() int {
 	}
 
 	if uint64(x)&0x2 > 0 {
-		x >>= 1
 		r |= 1
 	}
 

vendor/modules.txt

@@ -78,7 +78,7 @@ github.com/pkg/profile
 github.com/pkg/sftp
 # github.com/pkg/xattr v0.4.1
 github.com/pkg/xattr
-# github.com/restic/chunker v0.2.0
+# github.com/restic/chunker v0.3.0
 github.com/restic/chunker
 # github.com/russross/blackfriday v1.5.2
 github.com/russross/blackfriday