Add LZ4 compression

Godeps/Godeps.json

@@ -1,6 +1,6 @@
 {
 	"ImportPath": "github.com/calmh/syncthing",
-	"GoVersion": "devel +6bdbf9086c00 Thu Jul 17 17:02:46 2014 +1000",
+	"GoVersion": "go1.3",
 	"Packages": [
 		"./cmd/..."
 	],
@@ -40,6 +40,10 @@
 			"Comment": "null-15",
 			"Rev": "12e4b4183793ac4b061921e7980845e750679fd0"
 		},
+		{
+			"ImportPath": "github.com/bkaradzic/go-lz4",
+			"Rev": "77e2ba877bde9da31213bec75dbbe197fa507c21"
+		},
 		{
 			"ImportPath": "github.com/golang/groupcache/lru",
 			"Rev": "8b25adc0f62632c810997cb38c21111a3f256bf4"

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/.gitignore

@@ -0,0 +1 @@
+/lz4-example/lz4-example

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/.travis.yml

@@ -0,0 +1,7 @@
+language: go
+
+go:
+  - 1.1
+  - 1.2
+  - 1.3
+  - tip

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/LICENSE

@@ -0,0 +1,24 @@
+Copyright 2011-2012 Branimir Karadzic. All rights reserved.
+Copyright 2013 Damian Gryski. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+   1. Redistributions of source code must retain the above copyright notice, this
+      list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright notice,
+      this list of conditions and the following disclaimer in the documentation
+      and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDER ``AS IS'' AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
+SHALL COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+THE POSSIBILITY OF SUCH DAMAGE.
+

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/README.md

@@ -0,0 +1,71 @@
+go-lz4
+======
+
+go-lz4 is port of LZ4 lossless compression algorithm to Go. The original C code
+is located at:
+
+https://code.google.com/p/lz4/
+
+Status
+------
+[![Build Status](https://secure.travis-ci.org/bkaradzic/go-lz4.png)](http://travis-ci.org/bkaradzic/go-lz4)  
+[![GoDoc](https://godoc.org/github.com/bkaradzic/go-lz4?status.png)](https://godoc.org/github.com/bkaradzic/go-lz4)
+
+Usage
+-----
+
+    go get github.com/bkaradzic/go-lz4
+
+    import "github.com/bkaradzic/go-lz4"
+
+The package name is `lz4`
+
+Notes
+-----
+
+* go-lz4 saves a uint32 with the original uncompressed length at the beginning
+  of the encoded buffer.  They may get in the way of interoperability with
+  other implementations.
+
+Contributors
+------------
+
+Damian Gryski ([@dgryski](https://github.com/dgryski))  
+Dustin Sallings ([@dustin](https://github.com/dustin))
+
+Contact
+-------
+
+[@bkaradzic](https://twitter.com/bkaradzic)  
+http://www.stuckingeometry.com
+
+Project page  
+https://github.com/bkaradzic/go-lz4
+
+License
+-------
+
+Copyright 2011-2012 Branimir Karadzic. All rights reserved.  
+Copyright 2013 Damian Gryski. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+   1. Redistributions of source code must retain the above copyright notice, this
+      list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright notice,
+      this list of conditions and the following disclaimer in the documentation
+      and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDER ``AS IS'' AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
+SHALL COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+THE POSSIBILITY OF SUCH DAMAGE.
+

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/fuzzer/main.go

@@ -0,0 +1,74 @@
+package main
+
+import (
+	"math/rand"
+
+	"github.com/bkaradzic/go-lz4"
+
+	// lz4's API matches snappy's, so we can easily see how it performs
+	// lz4 "code.google.com/p/snappy-go/snappy"
+)
+
+var input = `
+ADVENTURE I. A SCANDAL IN BOHEMIA
+
+I.
+
+To Sherlock Holmes she is always THE woman. I have seldom heard
+him mention her under any other name. In his eyes she eclipses
+and predominates the whole of her sex. It was not that he felt
+any emotion akin to love for Irene Adler. All emotions, and that
+one particularly, were abhorrent to his cold, precise but
+admirably balanced mind. He was, I take it, the most perfect
+reasoning and observing machine that the world has seen, but as a
+lover he would have placed himself in a false position. He never
+spoke of the softer passions, save with a gibe and a sneer. They
+were admirable things for the observer--excellent for drawing the
+veil from men's motives and actions. But for the trained reasoner
+to admit such intrusions into his own delicate and finely
+adjusted temperament was to introduce a distracting factor which
+might throw a doubt upon all his mental results. Grit in a
+sensitive instrument, or a crack in one of his own high-power
+lenses, would not be more disturbing than a strong emotion in a
+nature such as his. And yet there was but one woman to him, and
+that woman was the late Irene Adler, of dubious and questionable
+memory.
+
+I had seen little of Holmes lately. My marriage had drifted us
+away from each other. My own complete happiness, and the
+home-centred interests which rise up around the man who first
+finds himself master of his own establishment, were sufficient to
+absorb all my attention, while Holmes, who loathed every form of
+society with his whole Bohemian soul, remained in our lodgings in
+Baker Street, buried among his old books, and alternating from
+week to week between cocaine and ambition, the drowsiness of the
+drug, and the fierce energy of his own keen nature. He was still,
+as ever, deeply attracted by the study of crime, and occupied his
+immense faculties and extraordinary powers of observation in
+following out those clues, and clearing up those mysteries which
+had been abandoned as hopeless by the official police. From time
+to time I heard some vague account of his doings: of his summons
+to Odessa in the case of the Trepoff murder, of his clearing up
+of the singular tragedy of the Atkinson brothers at Trincomalee,
+and finally of the mission which he had accomplished so
+delicately and successfully for the reigning family of Holland.
+Beyond these signs of his activity, however, which I merely
+shared with all the readers of the daily press, I knew little of
+my former friend and companion.
+`
+
+func main() {
+
+	compressed, _ := lz4.Encode(nil, []byte(input))
+
+	modified := make([]byte, len(compressed))
+
+	for {
+		copy(modified, compressed)
+		for i := 0; i < 100; i++ {
+			modified[rand.Intn(len(compressed)-4)+4] = byte(rand.Intn(256))
+		}
+		lz4.Decode(nil, modified)
+	}
+
+}

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/lz4-example/main.go

@@ -0,0 +1,86 @@
+/*
+ * Copyright 2011 Branimir Karadzic. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *    1. Redistributions of source code must retain the above copyright notice, this
+ *       list of conditions and the following disclaimer.
+ *
+ *    2. Redistributions in binary form must reproduce the above copyright notice,
+ *       this list of conditions and the following disclaimer in the documentation
+ *       and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDER ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
+ * SHALL COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"runtime/pprof"
+
+	lz4 "github.com/bkaradzic/go-lz4"
+)
+
+var (
+	decompress = flag.Bool("d", false, "decompress")
+)
+
+func main() {
+
+	var optCPUProfile = flag.String("cpuprofile", "", "profile")
+	flag.Parse()
+
+	if *optCPUProfile != "" {
+		f, err := os.Create(*optCPUProfile)
+		if err != nil {
+			log.Fatal(err)
+		}
+		pprof.StartCPUProfile(f)
+		defer pprof.StopCPUProfile()
+	}
+
+	args := flag.Args()
+
+	var data []byte
+
+	if len(args) < 2 {
+		fmt.Print("Usage: lz4 [-d] <input> <output>\n")
+		os.Exit(1)
+	}
+
+	input, err := os.OpenFile(args[0], os.O_RDONLY, 0644)
+	if err != nil {
+		fmt.Printf("Failed to open input file %s\n", args[0])
+		os.Exit(1)
+	}
+	defer input.Close()
+
+	if *decompress {
+		data, _ = ioutil.ReadAll(input)
+		data, _ = lz4.Decode(nil, data)
+	} else {
+		data, _ = ioutil.ReadAll(input)
+		data, _ = lz4.Encode(nil, data)
+	}
+
+	err = ioutil.WriteFile(args[1], data, 0644)
+	if err != nil {
+		fmt.Printf("Failed to open output file %s\n", args[1])
+		os.Exit(1)
+	}
+}

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/lz4_test.go

@@ -0,0 +1,63 @@
+package lz4
+
+import (
+	"bytes"
+	"io/ioutil"
+	"testing"
+)
+
+var testfile, _ = ioutil.ReadFile("testdata/pg1661.txt")
+
+func roundtrip(t *testing.T, input []byte) {
+
+	dst, err := Encode(nil, input)
+	if err != nil {
+		t.Errorf("got error during compression: %s", err)
+	}
+
+	output, err := Decode(nil, dst)
+
+	if err != nil {
+		t.Errorf("got error during decompress: %s", err)
+	}
+
+	if !bytes.Equal(output, input) {
+		t.Errorf("roundtrip failed")
+	}
+}
+
+func TestEmpty(t *testing.T) {
+	roundtrip(t, nil)
+}
+
+func TestLengths(t *testing.T) {
+
+	for i := 0; i < 1024; i++ {
+		roundtrip(t, testfile[:i])
+	}
+
+	for i := 1024; i < 4096; i += 23 {
+		roundtrip(t, testfile[:i])
+	}
+}
+
+func TestWords(t *testing.T) {
+	roundtrip(t, testfile)
+}
+
+func BenchmarkLZ4Encode(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		Encode(nil, testfile)
+	}
+}
+
+func BenchmarkLZ4Decode(b *testing.B) {
+
+	var compressed, _ = Encode(nil, testfile)
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		Decode(nil, compressed)
+	}
+}

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/reader.go

@@ -0,0 +1,194 @@
+/*
+ * Copyright 2011-2012 Branimir Karadzic. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *    1. Redistributions of source code must retain the above copyright notice, this
+ *       list of conditions and the following disclaimer.
+ *
+ *    2. Redistributions in binary form must reproduce the above copyright notice,
+ *       this list of conditions and the following disclaimer in the documentation
+ *       and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDER ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
+ * SHALL COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package lz4
+
+import (
+	"encoding/binary"
+	"errors"
+	"io"
+)
+
+var (
+	// ErrCorrupt indicates the input was corrupt
+	ErrCorrupt = errors.New("corrupt input")
+)
+
+const (
+	mlBits  = 4
+	mlMask  = (1 << mlBits) - 1
+	runBits = 8 - mlBits
+	runMask = (1 << runBits) - 1
+)
+
+type decoder struct {
+	src  []byte
+	dst  []byte
+	spos uint32
+	dpos uint32
+	ref  uint32
+}
+
+func (d *decoder) readByte() (uint8, error) {
+	if int(d.spos) == len(d.src) {
+		return 0, io.EOF
+	}
+	b := d.src[d.spos]
+	d.spos++
+	return b, nil
+}
+
+func (d *decoder) getLen() (uint32, error) {
+
+	length := uint32(0)
+	ln, err := d.readByte()
+	if err != nil {
+		return 0, ErrCorrupt
+	}
+	for ln == 255 {
+		length += 255
+		ln, err = d.readByte()
+		if err != nil {
+			return 0, ErrCorrupt
+		}
+	}
+	length += uint32(ln)
+
+	return length, nil
+}
+
+func (d *decoder) cp(length, decr uint32) {
+
+	if int(d.ref+length) < int(d.dpos) {
+		copy(d.dst[d.dpos:], d.dst[d.ref:d.ref+length])
+	} else {
+		for ii := uint32(0); ii < length; ii++ {
+			d.dst[d.dpos+ii] = d.dst[d.ref+ii]
+		}
+	}
+	d.dpos += length
+	d.ref += length - decr
+}
+
+func (d *decoder) finish(err error) error {
+	if err == io.EOF {
+		return nil
+	}
+
+	return err
+}
+
+// Decode returns the decoded form of src.  The returned slice may be a
+// subslice of dst if it was large enough to hold the entire decoded block.
+func Decode(dst, src []byte) ([]byte, error) {
+
+	if len(src) < 4 {
+		return nil, ErrCorrupt
+	}
+
+	uncompressedLen := binary.LittleEndian.Uint32(src)
+
+	if uncompressedLen == 0 {
+		return nil, nil
+	}
+
+	if uncompressedLen > MaxInputSize {
+		return nil, ErrTooLarge
+	}
+
+	if dst == nil || len(dst) < int(uncompressedLen) {
+		dst = make([]byte, uncompressedLen)
+	}
+
+	d := decoder{src: src, dst: dst[:uncompressedLen], spos: 4}
+
+	decr := []uint32{0, 3, 2, 3}
+
+	for {
+		code, err := d.readByte()
+		if err != nil {
+			return d.dst, d.finish(err)
+		}
+
+		length := uint32(code >> mlBits)
+		if length == runMask {
+			ln, err := d.getLen()
+			if err != nil {
+				return nil, ErrCorrupt
+			}
+			length += ln
+		}
+
+		if int(d.spos+length) > len(d.src) {
+			return nil, ErrCorrupt
+		}
+
+		for ii := uint32(0); ii < length; ii++ {
+			d.dst[d.dpos+ii] = d.src[d.spos+ii]
+		}
+
+		d.spos += length
+		d.dpos += length
+
+		if int(d.spos) == len(d.src) {
+			return d.dst, nil
+		}
+
+		if int(d.spos+2) >= len(d.src) {
+			return nil, ErrCorrupt
+		}
+
+		back := uint32(d.src[d.spos]) | uint32(d.src[d.spos+1])<<8
+
+		if back > d.dpos {
+			return nil, ErrCorrupt
+		}
+
+		d.spos += 2
+		d.ref = d.dpos - back
+
+		length = uint32(code & mlMask)
+		if length == mlMask {
+			ln, err := d.getLen()
+			if err != nil {
+				return nil, ErrCorrupt
+			}
+			length += ln
+		}
+
+		literal := d.dpos - d.ref
+		if literal < 4 {
+			d.cp(4, decr[literal])
+		} else {
+			length += 4
+		}
+
+		if d.dpos+length > uncompressedLen {
+			return nil, ErrCorrupt
+		}
+
+		d.cp(length, 0)
+	}
+}

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/writer.go

@@ -0,0 +1,188 @@
+/*
+ * Copyright 2011-2012 Branimir Karadzic. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *    1. Redistributions of source code must retain the above copyright notice, this
+ *       list of conditions and the following disclaimer.
+ *
+ *    2. Redistributions in binary form must reproduce the above copyright notice,
+ *       this list of conditions and the following disclaimer in the documentation
+ *       and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDER ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
+ * SHALL COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package lz4
+
+import "encoding/binary"
+import "errors"
+
+const (
+	minMatch              = 4
+	hashLog               = 17
+	hashTableSize         = 1 << hashLog
+	hashShift             = (minMatch * 8) - hashLog
+	incompressible uint32 = 128
+	uninitHash            = 0x88888888
+
+	// MaxInputSize is the largest buffer than can be compressed in a single block
+	MaxInputSize = 0x7E000000
+)
+
+var (
+	// ErrTooLarge indicates the input buffer was too large
+	ErrTooLarge = errors.New("input too large")
+)
+
+type encoder struct {
+	src       []byte
+	dst       []byte
+	hashTable []uint32
+	pos       uint32
+	anchor    uint32
+	dpos      uint32
+}
+
+// CompressBound returns the maximum length of a lz4 block, given it's uncompressed length
+func CompressBound(isize int) int {
+	if isize > MaxInputSize {
+		return 0
+	}
+	return isize + ((isize) / 255) + 16 + 4
+}
+
+func (e *encoder) writeLiterals(length, mlLen, pos uint32) {
+
+	ln := length
+
+	var code byte
+	if ln > runMask-1 {
+		code = runMask
+	} else {
+		code = byte(ln)
+	}
+
+	if mlLen > mlMask-1 {
+		e.dst[e.dpos] = (code << mlBits) + byte(mlMask)
+	} else {
+		e.dst[e.dpos] = (code << mlBits) + byte(mlLen)
+	}
+	e.dpos++
+
+	if code == runMask {
+		ln -= runMask
+		for ; ln > 254; ln -= 255 {
+			e.dst[e.dpos] = 255
+			e.dpos++
+		}
+
+		e.dst[e.dpos] = byte(ln)
+		e.dpos++
+	}
+
+	for ii := uint32(0); ii < length; ii++ {
+		e.dst[e.dpos+ii] = e.src[pos+ii]
+	}
+
+	e.dpos += length
+}
+
+// Encode returns the encoded form of src.  The returned array may be a
+// sub-slice of dst if it was large enough to hold the entire output.
+func Encode(dst, src []byte) ([]byte, error) {
+
+	if len(src) >= MaxInputSize {
+		return nil, ErrTooLarge
+	}
+
+	if n := CompressBound(len(src)); len(dst) < n {
+		dst = make([]byte, n)
+	}
+
+	e := encoder{src: src, dst: dst, hashTable: make([]uint32, hashTableSize)}
+
+	binary.LittleEndian.PutUint32(dst, uint32(len(src)))
+	e.dpos = 4
+
+	var (
+		step  uint32 = 1
+		limit        = incompressible
+	)
+
+	for {
+		if int(e.pos)+4 >= len(e.src) {
+			e.writeLiterals(uint32(len(e.src))-e.anchor, 0, e.anchor)
+			return e.dst[:e.dpos], nil
+		}
+
+		sequence := uint32(e.src[e.pos+3])<<24 | uint32(e.src[e.pos+2])<<16 | uint32(e.src[e.pos+1])<<8 | uint32(e.src[e.pos+0])
+
+		hash := (sequence * 2654435761) >> hashShift
+		ref := e.hashTable[hash] + uninitHash
+		e.hashTable[hash] = e.pos - uninitHash
+
+		if ((e.pos-ref)>>16) != 0 || uint32(e.src[ref+3])<<24|uint32(e.src[ref+2])<<16|uint32(e.src[ref+1])<<8|uint32(e.src[ref+0]) != sequence {
+			if e.pos-e.anchor > limit {
+				limit <<= 1
+				step += 1 + (step >> 2)
+			}
+			e.pos += step
+			continue
+		}
+
+		if step > 1 {
+			e.hashTable[hash] = ref - uninitHash
+			e.pos -= step - 1
+			step = 1
+			continue
+		}
+		limit = incompressible
+
+		ln := e.pos - e.anchor
+		back := e.pos - ref
+
+		anchor := e.anchor
+
+		e.pos += minMatch
+		ref += minMatch
+		e.anchor = e.pos
+
+		for int(e.pos) < len(e.src) && e.src[e.pos] == e.src[ref] {
+			e.pos++
+			ref++
+		}
+
+		mlLen := e.pos - e.anchor
+
+		e.writeLiterals(ln, mlLen, anchor)
+		e.dst[e.dpos] = uint8(back)
+		e.dst[e.dpos+1] = uint8(back >> 8)
+		e.dpos += 2
+
+		if mlLen > mlMask-1 {
+			mlLen -= mlMask
+			for mlLen > 254 {
+				mlLen -= 255
+
+				e.dst[e.dpos] = 255
+				e.dpos++
+			}
+
+			e.dst[e.dpos] = byte(mlLen)
+			e.dpos++
+		}
+
+		e.anchor = e.pos
+	}
+}

integration/f1/config.xml

@@ -21,7 +21,7 @@
         <localAnnounceEnabled>true</localAnnounceEnabled>
         <localAnnouncePort>21025</localAnnouncePort>
         <parallelRequests>16</parallelRequests>
-        <maxSendKbps>1000</maxSendKbps>
+        <maxSendKbps>50</maxSendKbps>
         <rescanIntervalS>10</rescanIntervalS>
         <reconnectionIntervalS>5</reconnectionIntervalS>
         <maxChangeKbps>10000</maxChangeKbps>

integration/h1/config.xml

@@ -1,36 +1,36 @@
 <configuration version="2">
     <repository id="default" directory="s1" ro="false" ignorePerms="false">
-        <node id="373HSRPQLPNLIJYKZVQFP4PKZ6R2ZE6K3YD442UJHBGBQGWWXAHA"></node>
-        <node id="I6KAH7666SLLL5PFXSOAUFJCDZYAOMLEKCP2GB3BV5RQST3PSROA"></node>
-        <node id="JMFJCXBGZDE4BOCJE3VF65GYZNAIVJRET3J6HMRAUQIGJOFKNHMQ"></node>
+        <node id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU"></node>
+        <node id="JMFJCXB-GZDE4BN-OCJE3VF-65GYZNU-AIVJRET-3J6HMRQ-AUQIGJO-FKNHMQU"></node>
+        <node id="373HSRP-QLPNLIE-JYKZVQF-P4PKZ63-R2ZE6K3-YD442U2-JHBGBQG-WWXAHAU"></node>
         <versioning></versioning>
     </repository>
     <repository id="s12" directory="s12-1" ro="false" ignorePerms="false">
-        <node id="I6KAH7666SLLL5PFXSOAUFJCDZYAOMLEKCP2GB3BV5RQST3PSROA"></node>
-        <node id="JMFJCXBGZDE4BOCJE3VF65GYZNAIVJRET3J6HMRAUQIGJOFKNHMQ"></node>
+        <node id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU"></node>
+        <node id="JMFJCXB-GZDE4BN-OCJE3VF-65GYZNU-AIVJRET-3J6HMRQ-AUQIGJO-FKNHMQU"></node>
         <versioning></versioning>
     </repository>
-    <node id="I6KAH7666SLLL5PFXSOAUFJCDZYAOMLEKCP2GB3BV5RQST3PSROA" name="s1">
+    <node id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU" name="s1">
         <address>127.0.0.1:22001</address>
     </node>
-    <node id="JMFJCXBGZDE4BOCJE3VF65GYZNAIVJRET3J6HMRAUQIGJOFKNHMQ" name="s2">
+    <node id="JMFJCXB-GZDE4BN-OCJE3VF-65GYZNU-AIVJRET-3J6HMRQ-AUQIGJO-FKNHMQU" name="s2">
         <address>127.0.0.1:22002</address>
     </node>
-    <node id="373HSRPQLPNLIJYKZVQFP4PKZ6R2ZE6K3YD442UJHBGBQGWWXAHA" name="s3">
+    <node id="373HSRP-QLPNLIE-JYKZVQF-P4PKZ63-R2ZE6K3-YD442U2-JHBGBQG-WWXAHAU" name="s3">
         <address>127.0.0.1:22003</address>
     </node>
-    <node id="EJHMPAQOGCVORISB4IS3SYYVJXTKJGLTU66DIQPGJ5D2GXGQ3OWQ" name="s4">
+    <node id="EJHMPAQ-OGCVORE-ISB4IS3-SYYVJXF-TKJGLTU-66DIQPF-GJ5D2GX-GQ3OWQK" name="s4">
         <address>127.0.0.1:22004</address>
     </node>
     <gui enabled="true" tls="false">
         <address>127.0.0.1:8081</address>
-        <apikey>abc123</apikey>
         <user>testuser</user>
-        <password>testpass</password>
+        <password>$2a$10$7tKL5uvLDGn5s2VLPM2yWOK/II45az0mTel8hxAUJDRQN1Tk2QYwu</password>
+        <apikey>abc123</apikey>
     </gui>
     <options>
         <listenAddress>127.0.0.1:22001</listenAddress>
-        <globalAnnounceServer>announce.syncthing.net:22025</globalAnnounceServer>
+        <globalAnnounceServer>announce.syncthing.net:22026</globalAnnounceServer>
         <globalAnnounceEnabled>false</globalAnnounceEnabled>
         <localAnnounceEnabled>true</localAnnounceEnabled>
         <localAnnouncePort>21025</localAnnouncePort>
@@ -41,5 +41,6 @@
         <maxChangeKbps>10000</maxChangeKbps>
         <startBrowser>false</startBrowser>
         <upnpEnabled>true</upnpEnabled>
+        <urAccepted>-1</urAccepted>
     </options>
 </configuration>

protocol/PROTOCOL.md

@@ -25,11 +25,13 @@ Transport and Authentication
 ----------------------------
 
 BEP is deployed as the highest level in a protocol stack, with the lower
-level protocols providing encryption and authentication.
+level protocols providing compression, encryption and authentication.
 
     +-----------------------------|
     |   Block Exchange Protocol   |
     |-----------------------------|
+    |      Compression (LZ4)      |
+    |-----------------------------|
     | Encryption & Auth (TLS 1.2) |
     |-----------------------------|
     |             TCP             |
@@ -60,6 +62,37 @@ requests are received.
 
 The underlying transport protocol MUST be TCP.
 
+Compression
+-----------
+
+All data is sent within compressed blocks. Blocks are compressed using
+the LZ4 format and algorithm described in
+https://code.google.com/p/lz4/. Each compressed block is preceded by a
+header consisting of three 32 bit words, in network order (big endian):
+
+     0                   1                   2                   3
+     0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                     Magic (0x0x5e63b278)                      |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                          Data Length                          |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                   Uncompressed Block Length                   |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    /                                                               /
+    \                        Compressed Data                        \
+    /                                                               /
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+The Data Length indicates the length of data following the Data Length
+field until the next header, i.e. the length of the Compressed Data
+section plus four bytes for the Uncompressed Block Length field. The
+Uncompressed Block Length indicates the amount of data that will result
+when decompressing the Compressed Data section.
+
+A single BEP message SHOULD be sent as a single compressed block. A
+single compressed block MAY NOT contain more than one BEP message.
+
 Messages
 --------
 

protocol/lz4stream.go

@@ -0,0 +1,119 @@
+package protocol
+
+import (
+	"bytes"
+	"encoding/binary"
+	"errors"
+	"io"
+	"sync"
+
+	lz4 "github.com/bkaradzic/go-lz4"
+)
+
+const lz4Magic = 0x5e63b278
+
+type lz4Writer struct {
+	wr  io.Writer
+	mut sync.Mutex
+	buf []byte
+}
+
+func newLZ4Writer(w io.Writer) *lz4Writer {
+	return &lz4Writer{wr: w}
+}
+
+func (w *lz4Writer) Write(bs []byte) (int, error) {
+	w.mut.Lock()
+	defer w.mut.Unlock()
+
+	var err error
+	w.buf, err = lz4.Encode(w.buf[:cap(w.buf)], bs)
+	if err != nil {
+		return 0, err
+	}
+
+	var hdr [8]byte
+	binary.BigEndian.PutUint32(hdr[0:], lz4Magic)
+	binary.BigEndian.PutUint32(hdr[4:], uint32(len(w.buf)))
+	_, err = w.wr.Write(hdr[:])
+	if err != nil {
+		return 0, err
+	}
+
+	_, err = w.wr.Write(w.buf)
+	if err != nil {
+		return 0, err
+	}
+
+	if debug {
+		l.Debugf("lz4 write; %d / %d bytes", len(bs), 8+len(w.buf))
+	}
+	return len(bs), nil
+}
+
+type lz4Reader struct {
+	rd     io.Reader
+	mut    sync.Mutex
+	buf    []byte
+	ebuf   []byte
+	obuf   *bytes.Buffer
+	ibytes uint64
+	obytes uint64
+}
+
+func newLZ4Reader(r io.Reader) *lz4Reader {
+	return &lz4Reader{rd: r}
+}
+
+func (r *lz4Reader) Read(bs []byte) (int, error) {
+	r.mut.Lock()
+	defer r.mut.Unlock()
+
+	if r.obuf == nil {
+		r.obuf = bytes.NewBuffer(nil)
+	}
+
+	if r.obuf.Len() == 0 {
+		if err := r.moreBits(); err != nil {
+			return 0, err
+		}
+	}
+
+	n, err := r.obuf.Read(bs)
+	if debug {
+		l.Debugf("lz4 read; %d bytes", n)
+	}
+	return n, err
+}
+
+func (r *lz4Reader) moreBits() error {
+	var hdr [8]byte
+	_, err := io.ReadFull(r.rd, hdr[:])
+	if binary.BigEndian.Uint32(hdr[0:]) != lz4Magic {
+		return errors.New("bad magic")
+	}
+
+	ln := int(binary.BigEndian.Uint32(hdr[4:]))
+	if len(r.buf) < ln {
+		r.buf = make([]byte, int(ln))
+	} else {
+		r.buf = r.buf[:ln]
+	}
+
+	_, err = io.ReadFull(r.rd, r.buf)
+	if err != nil {
+		return err
+	}
+
+	r.ebuf, err = lz4.Decode(r.ebuf[:cap(r.ebuf)], r.buf)
+	if err != nil {
+		return err
+	}
+
+	if debug {
+		l.Debugf("lz4 moreBits: %d / %d bytes", ln+8, len(r.ebuf))
+	}
+
+	_, err = r.obuf.Write(r.ebuf)
+	return err
+}

protocol/lz4stream_test.go

@@ -0,0 +1,60 @@
+package protocol
+
+import (
+	"bytes"
+	"crypto/rand"
+	"io"
+	"testing"
+)
+
+var toWrite = [][]byte{
+	[]byte("this is a short byte string that should pass through somewhat compressed this is a short byte string that should pass through somewhat compressed this is a short byte string that should pass through somewhat compressed this is a short byte string that should pass through somewhat compressed this is a short byte string that should pass through somewhat compressed this is a short byte string that should pass through somewhat compressed"),
+	[]byte("this is another short byte string that should pass through uncompressed"),
+	[]byte{0, 1, 2, 3, 4, 5},
+}
+
+func TestLZ4Stream(t *testing.T) {
+	tb := make([]byte, 128*1024)
+	rand.Reader.Read(tb)
+	toWrite = append(toWrite, tb)
+	tb = make([]byte, 512*1024)
+	rand.Reader.Read(tb)
+	toWrite = append(toWrite, tb)
+	toWrite = append(toWrite, toWrite[0])
+	toWrite = append(toWrite, toWrite[1])
+
+	rd, wr := io.Pipe()
+	lz4r := newLZ4Reader(rd)
+	lz4w := newLZ4Writer(wr)
+
+	go func() {
+		for i := 0; i < 5; i++ {
+			for _, bs := range toWrite {
+				n, err := lz4w.Write(bs)
+				if err != nil {
+					t.Error(err)
+				}
+				if n != len(bs) {
+					t.Errorf("weird write length; %d != %d", n, len(bs))
+				}
+			}
+		}
+	}()
+
+	buf := make([]byte, 512*1024)
+
+	for i := 0; i < 5; i++ {
+		for _, bs := range toWrite {
+			n, err := lz4r.Read(buf)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if n != len(bs) {
+				t.Errorf("Unexpected len %d != %d", n, len(bs))
+			}
+			if bytes.Compare(bs, buf[:n]) != 0 {
+				t.Error("Unexpected data")
+			}
+		}
+	}
+}

protocol/protocol.go

@@ -109,11 +109,17 @@ const (
 )
 
 func NewConnection(nodeID NodeID, reader io.Reader, writer io.Writer, receiver Model, name string) Connection {
+	// Byte counters are at the lowest level, counting compressed bytes
 	cr := &countingReader{Reader: reader}
 	cw := &countingWriter{Writer: writer}
 
-	rb := bufio.NewReader(cr)
-	wb := bufio.NewWriterSize(cw, 65536)
+	// Compression is just above counting
+	zr := newLZ4Reader(cr)
+	zw := newLZ4Writer(cw)
+
+	// We buffer writes on top of compression.
+	// The LZ4 reader is already internally buffered
+	wb := bufio.NewWriterSize(zw, 65536)
 
 	c := rawConnection{
 		id:       nodeID,
@@ -121,7 +127,7 @@ func NewConnection(nodeID NodeID, reader io.Reader, writer io.Writer, receiver M
 		receiver: nativeModel{receiver},
 		state:    stateInitial,
 		cr:       cr,
-		xr:       xdr.NewReader(rb),
+		xr:       xdr.NewReader(zr),
 		cw:       cw,
 		wb:       wb,
 		xw:       xdr.NewWriter(wb),
@@ -428,10 +434,6 @@ func (c *rawConnection) writerLoop() {
 	}
 }
 
-type flusher interface {
-	Flush() error
-}
-
 func (c *rawConnection) flush() error {
 	if err := c.xw.Error(); err != nil {
 		return err

protocol/protocol_test.go

@@ -73,8 +73,8 @@ func TestPing(t *testing.T) {
 func TestPingErr(t *testing.T) {
 	e := errors.New("something broke")
 
-	for i := 0; i < 12; i++ {
-		for j := 0; j < 12; j++ {
+	for i := 0; i < 16; i++ {
+		for j := 0; j < 16; j++ {
 			m0 := newTestModel()
 			m1 := newTestModel()
 
@@ -87,9 +87,9 @@ func TestPingErr(t *testing.T) {
 			NewConnection(c1ID, br, eaw, m1, "name")
 
 			res := c0.ping()
-			if (i < 4 || j < 4) && res {
+			if (i < 8 || j < 8) && res {
 				t.Errorf("Unexpected ping success; i=%d, j=%d", i, j)
-			} else if (i >= 8 && j >= 8) && !res {
+			} else if (i >= 12 && j >= 12) && !res {
 				t.Errorf("Unexpected ping fail; i=%d, j=%d", i, j)
 			}
 		}