diff --git a/lib/scanner/blocks.go b/lib/scanner/blocks.go index dea7c0305..ec6d946c4 100644 --- a/lib/scanner/blocks.go +++ b/lib/scanner/blocks.go @@ -9,9 +9,9 @@ package scanner import ( "bytes" "fmt" - "hash/adler32" "io" + "github.com/chmduquesne/rollinghash/adler32" "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/sha256" ) diff --git a/lib/scanner/blocks_test.go b/lib/scanner/blocks_test.go index dc5c8d3c8..47d816a05 100644 --- a/lib/scanner/blocks_test.go +++ b/lib/scanner/blocks_test.go @@ -8,9 +8,13 @@ package scanner import ( "bytes" + "crypto/rand" "fmt" + origAdler32 "hash/adler32" "testing" + "testing/quick" + rollingAdler32 "github.com/chmduquesne/rollinghash/adler32" "github.com/syncthing/syncthing/lib/protocol" ) @@ -160,3 +164,66 @@ func TestDiffEmpty(t *testing.T) { } } } + +func TestAdler32Variants(t *testing.T) { + // Verify that the two adler32 functions give matching results for a few + // different blocks of data. + + hf1 := origAdler32.New() + hf2 := rollingAdler32.New() + + checkFn := func(data []byte) bool { + hf1.Write(data) + sum1 := hf1.Sum32() + + hf2.Write(data) + sum2 := hf2.Sum32() + + hf1.Reset() + hf2.Reset() + + return sum1 == sum2 + } + + // protocol block sized data + data := make([]byte, protocol.BlockSize) + for i := 0; i < 5; i++ { + rand.Read(data) + if !checkFn(data) { + t.Errorf("Hash mismatch on block sized data") + } + } + + // random small blocks + if err := quick.Check(checkFn, nil); err != nil { + t.Error(err) + } + + // rolling should have the same result as the individual blocks + // themselves. Which is not the same as the original non-rollind adler32 + // blocks. + + windowSize := 128 + + hf2.Reset() + + hf3 := rollingAdler32.New() + hf3.Write(data[:windowSize]) + + for i := windowSize; i < len(data); i++ { + if i%windowSize == 0 { + // let the reference function catch up + hf2.Write(data[i-windowSize : i]) + + // verify that they are in sync with the rolling function + sum2 := hf2.Sum32() + sum3 := hf3.Sum32() + t.Logf("At i=%d, sum2=%08x, sum3=%08x", i, sum2, sum3) + if sum2 != sum3 { + t.Errorf("Mismatch after roll; i=%d, sum2=%08x, sum3=%08x", i, sum2, sum3) + break + } + } + hf3.Roll(data[i]) + } +} diff --git a/vendor/github.com/chmduquesne/rollinghash/adler32/adler32.go b/vendor/github.com/chmduquesne/rollinghash/adler32/adler32.go index a05df02d2..643a5987a 100644 --- a/vendor/github.com/chmduquesne/rollinghash/adler32/adler32.go +++ b/vendor/github.com/chmduquesne/rollinghash/adler32/adler32.go @@ -3,6 +3,7 @@ package adler32 import ( + "hash" vanilla "hash/adler32" "github.com/chmduquesne/rollinghash" @@ -22,6 +23,8 @@ type digest struct { window []byte oldest int n uint32 + + vanilla hash.Hash32 } // Reset resets the Hash to its initial state. @@ -37,7 +40,7 @@ func (d *digest) Reset() { // only used to determine which is the oldest element (leaving the // window). The calls to Roll() do not recompute the whole checksum. func New() rollinghash.Hash32 { - return &digest{a: 1, b: 0, window: nil, oldest: 0} + return &digest{a: 1, b: 0, window: nil, oldest: 0, vanilla: vanilla.New()} } // Size returns the number of bytes Sum will return. @@ -53,13 +56,15 @@ func (d *digest) BlockSize() int { return 1 } // running hash. It never returns an error. func (d *digest) Write(p []byte) (int, error) { // Copy the window - d.window = make([]byte, len(p)) + if len(d.window) != len(p) { + d.window = make([]byte, len(p)) + } copy(d.window, p) // Piggy-back on the core implementation - h := vanilla.New() - h.Write(p) - s := h.Sum32() + d.vanilla.Reset() + d.vanilla.Write(p) + s := d.vanilla.Sum32() d.a, d.b = s&0xffff, s>>16 d.n = uint32(len(p)) % mod return len(d.window), nil diff --git a/vendor/manifest b/vendor/manifest index e9947b4ef..7156200c9 100644 --- a/vendor/manifest +++ b/vendor/manifest @@ -46,10 +46,9 @@ }, { "importpath": "github.com/chmduquesne/rollinghash", - "repository": "https://github.com/chmduquesne/rollinghash", + "repository": "https://github.com/kastelo/rollinghash", "vcs": "git", - "revision": "88b86a92826991b14d01fb43456909fcb8a76b8b", - "branch": "master", + "branch": "reducealloc", "notests": true }, { @@ -422,4 +421,4 @@ "notests": true } ] -} \ No newline at end of file +}