Merge pull request #3519 from greatroar/maphash

Replace siphash by hash/maphash

.github/workflows/tests.yml

@@ -45,11 +45,6 @@ jobs:
             os: ubuntu-latest
             test_fuse: true
 
-          - job_name: Linux
-            go: 1.13.x
-            os: ubuntu-latest
-            test_fuse: true
-
     name: ${{ matrix.job_name }} Go ${{ matrix.go }}
     runs-on: ${{ matrix.os }}
 

CONTRIBUTING.md

@@ -66,7 +66,7 @@ Development Environment
 The repository contains the code written for restic in the directories
 `cmd/` and `internal/`.
 
-Restic requires Go version 1.13 or later for compiling. Clone the repo (without
+Restic requires Go version 1.14 or later for compiling. Clone the repo (without
 having `$GOPATH` set) and `cd` into the directory:
 
     $ unset GOPATH

build.go

@@ -35,6 +35,7 @@
 // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+//go:build ignore_build_go
 // +build ignore_build_go
 
 package main
@@ -123,17 +124,8 @@ func printEnv(env []string) {
 
 // build runs "go build args..." with GOPATH set to gopath.
 func build(cwd string, env map[string]string, args ...string) error {
-	a := []string{"build"}
+	// -trimpath removes all absolute paths from the binary.
-
+	a := []string{"build", "-trimpath"}
-	// try to remove all absolute paths from resulting binary
-	if goVersion.AtLeast(GoVersion{1, 13, 0}) {
-		// use the new flag introduced by Go 1.13
-		a = append(a, "-trimpath")
-	} else {
-		// otherwise try to trim as many paths as possible
-		a = append(a, "-asmflags", fmt.Sprintf("all=-trimpath=%s", cwd))
-		a = append(a, "-gcflags", fmt.Sprintf("all=-trimpath=%s", cwd))
-	}
 
 	if enablePIE {
 		a = append(a, "-buildmode=pie")

changelog/unreleased/pull-3519

@@ -0,0 +1,6 @@
+Change: Require Go 1.14 or newer
+
+Restic now requires Go 1.14 to build. This allows it to use new
+standard library features instead of an external dependency.
+
+https://github.com/restic/restic/issues/3519

doc/020_installation.rst

@@ -255,7 +255,7 @@ From Source
 ***********
 
 restic is written in the Go programming language and you need at least
-Go version 1.13. Building restic may also work with older versions of Go,
+Go version 1.14. Building restic may also work with older versions of Go,
 but that's not supported. See the `Getting
 started <https://golang.org/doc/install>`__ guide of the Go project for
 instructions how to install Go.

go.mod

@@ -8,7 +8,6 @@ require (
 	github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
 	github.com/cenkalti/backoff/v4 v4.1.1
 	github.com/cespare/xxhash/v2 v2.1.1
-	github.com/dchest/siphash v1.2.2
 	github.com/dnaeon/go-vcr v1.2.0 // indirect
 	github.com/elithrar/simple-scrypt v1.3.0
 	github.com/go-ole/go-ole v1.2.5
@@ -37,4 +36,4 @@ require (
 	gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637
 )
 
-go 1.13
+go 1.14

go.sum

@@ -89,8 +89,6 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsr
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dchest/siphash v1.2.2 h1:9DFz8tQwl9pTVt5iok/9zKyzA1Q6bRGiF3HPiEEVr9I=
-github.com/dchest/siphash v1.2.2/go.mod h1:q+IRvb2gOSrUnYoPqHiyHXS0FOBBOdl6tONBlVnOnt4=
 github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
 github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
 github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=

internal/repository/indexmap.go

@@ -1,12 +1,9 @@
 package repository
 
 import (
-	"crypto/rand"
+	"hash/maphash"
-	"encoding/binary"
 
 	"github.com/restic/restic/internal/restic"
-
-	"github.com/dchest/siphash"
 )
 
 // An indexMap is a chained hash table that maps blob IDs to indexEntries.
@@ -23,7 +20,7 @@ type indexMap struct {
 	buckets    []*indexEntry
 	numentries uint
 
-	key0, key1 uint64 // Key for hash randomization.
+	mh maphash.Hash
 
 	free *indexEntry // Free list.
 }
@@ -113,25 +110,20 @@ func (m *indexMap) grow() {
 }
 
 func (m *indexMap) hash(id restic.ID) uint {
-	// We use siphash with a randomly generated 128-bit key, to prevent
+	// We use maphash to prevent backups of specially crafted inputs
-	// backups of specially crafted inputs from degrading performance.
+	// from degrading performance.
 	// While SHA-256 should be collision-resistant, for hash table indices
 	// we use only a few bits of it and finding collisions for those is
 	// much easier than breaking the whole algorithm.
-	h := uint(siphash.Hash(m.key0, m.key1, id[:]))
+	m.mh.Reset()
+	_, _ = m.mh.Write(id[:])
+	h := uint(m.mh.Sum64())
 	return h & uint(len(m.buckets)-1)
 }
 
 func (m *indexMap) init() {
 	const initialBuckets = 64
 	m.buckets = make([]*indexEntry, initialBuckets)
-
-	var buf [16]byte
-	if _, err := rand.Read(buf[:]); err != nil {
-		panic(err) // Very little we can do here.
-	}
-	m.key0 = binary.LittleEndian.Uint64(buf[:8])
-	m.key1 = binary.LittleEndian.Uint64(buf[8:])
 }
 
 func (m *indexMap) len() uint { return m.numentries }

internal/repository/indexmap_test.go

@@ -107,32 +107,6 @@ func TestIndexMapForeachWithID(t *testing.T) {
 	}
 }
 
-func TestIndexMapHash(t *testing.T) {
-	t.Parallel()
-
-	var m1, m2 indexMap
-
-	id := restic.NewRandomID()
-	// Add to both maps to initialize them.
-	m1.add(id, 0, 0, 0)
-	m2.add(id, 0, 0, 0)
-
-	h1 := m1.hash(id)
-	h2 := m2.hash(id)
-
-	rtest.Equals(t, len(m1.buckets), len(m2.buckets)) // just to be sure
-
-	if h1 == h2 {
-		// The probability of the zero key should be 2^(-128).
-		if m1.key0 == 0 && m1.key1 == 0 {
-			t.Error("siphash key not set for m1")
-		}
-		if m2.key0 == 0 && m2.key1 == 0 {
-			t.Error("siphash key not set for m2")
-		}
-	}
-}
-
 func BenchmarkIndexMapHash(b *testing.B) {
 	var m indexMap
 	m.add(restic.ID{}, 0, 0, 0) // Trigger lazy initialization.