Screw godep

This commit is contained in:
Audrius Butkevicius 2016-04-17 14:49:00 +01:00
parent 1a35c440e8
commit 1bc4c1a8ac
136 changed files with 0 additions and 22914 deletions

View File

@ -1,95 +0,0 @@
{
"ImportPath": "github.com/syncthing/relaypoolsrv",
"GoVersion": "go1.5.1",
"Packages": [
"."
],
"Deps": [
{
"ImportPath": "github.com/bkaradzic/go-lz4",
"Rev": "74ddf82598bc4745b965729e9c6a463bedd33049"
},
{
"ImportPath": "github.com/calmh/du",
"Rev": "3c0690cca16228b97741327b1b6781397afbdb24"
},
{
"ImportPath": "github.com/calmh/luhn",
"Rev": "0c8388ff95fa92d4094011e5a04fc99dea3d1632"
},
{
"ImportPath": "github.com/calmh/xdr",
"Rev": "9eb3e1a622d9364deb39c831f7e5f164393d7e37"
},
{
"ImportPath": "github.com/golang/groupcache/lru",
"Rev": "604ed5785183e59ae2789449d89e73f3a2a77987"
},
{
"ImportPath": "github.com/juju/ratelimit",
"Rev": "772f5c38e468398c4511514f4f6aa9a4185bc0a0"
},
{
"ImportPath": "github.com/oschwald/geoip2-golang",
"Comment": "v0.1.0-28-gfe8132a",
"Rev": "fe8132a35233b371398441ab4c5655ae1244564a"
},
{
"ImportPath": "github.com/oschwald/maxminddb-golang",
"Comment": "v0.2.0-41-gbfd1341",
"Rev": "bfd134128faa57c37ef41fff5fb1a9ca94450a9b"
},
{
"ImportPath": "github.com/syncthing/syncthing/lib/dialer",
"Comment": "v0.12.6-6-g38e9b92",
"Rev": "38e9b92c42d827292cdddada39d7ddde646e1677"
},
{
"ImportPath": "github.com/syncthing/syncthing/lib/logger",
"Comment": "v0.12.6-6-g38e9b92",
"Rev": "38e9b92c42d827292cdddada39d7ddde646e1677"
},
{
"ImportPath": "github.com/syncthing/syncthing/lib/osutil",
"Comment": "v0.12.6-6-g38e9b92",
"Rev": "38e9b92c42d827292cdddada39d7ddde646e1677"
},
{
"ImportPath": "github.com/syncthing/syncthing/lib/protocol",
"Comment": "v0.12.6-6-g38e9b92",
"Rev": "38e9b92c42d827292cdddada39d7ddde646e1677"
},
{
"ImportPath": "github.com/syncthing/syncthing/lib/relay/client",
"Comment": "v0.12.6-6-g38e9b92",
"Rev": "38e9b92c42d827292cdddada39d7ddde646e1677"
},
{
"ImportPath": "github.com/syncthing/syncthing/lib/relay/protocol",
"Comment": "v0.12.6-6-g38e9b92",
"Rev": "38e9b92c42d827292cdddada39d7ddde646e1677"
},
{
"ImportPath": "github.com/syncthing/syncthing/lib/sync",
"Comment": "v0.12.6-6-g38e9b92",
"Rev": "38e9b92c42d827292cdddada39d7ddde646e1677"
},
{
"ImportPath": "github.com/syncthing/syncthing/lib/tlsutil",
"Comment": "v0.12.6-6-g38e9b92",
"Rev": "38e9b92c42d827292cdddada39d7ddde646e1677"
},
{
"ImportPath": "golang.org/x/net/proxy",
"Rev": "042ba42fa6633b34205efc66ba5719cd3afd8d38"
},
{
"ImportPath": "golang.org/x/text/transform",
"Rev": "5eb8d4684c4796dd36c74f6452f2c0fa6c79597e"
},
{
"ImportPath": "golang.org/x/text/unicode/norm",
"Rev": "5eb8d4684c4796dd36c74f6452f2c0fa6c79597e"
}
]
}

View File

@ -1,5 +0,0 @@
This directory tree is generated automatically by godep.
Please do not edit.
See https://github.com/tools/godep for more information.

View File

@ -1,2 +0,0 @@
/pkg
/bin

View File

@ -1 +0,0 @@
/lz4-example/lz4-example

View File

@ -1,9 +0,0 @@
language: go
go:
- 1.1
- 1.2
- 1.3
- 1.4
- 1.5
- tip

View File

@ -1,24 +0,0 @@
Copyright 2011-2012 Branimir Karadzic. All rights reserved.
Copyright 2013 Damian Gryski. All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDER ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
SHALL COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
THE POSSIBILITY OF SUCH DAMAGE.

View File

@ -1,71 +0,0 @@
go-lz4
======
go-lz4 is port of LZ4 lossless compression algorithm to Go. The original C code
is located at:
https://github.com/Cyan4973/lz4
Status
------
[![Build Status](https://secure.travis-ci.org/bkaradzic/go-lz4.png)](http://travis-ci.org/bkaradzic/go-lz4)
[![GoDoc](https://godoc.org/github.com/bkaradzic/go-lz4?status.png)](https://godoc.org/github.com/bkaradzic/go-lz4)
Usage
-----
go get github.com/bkaradzic/go-lz4
import "github.com/bkaradzic/go-lz4"
The package name is `lz4`
Notes
-----
* go-lz4 saves a uint32 with the original uncompressed length at the beginning
of the encoded buffer. They may get in the way of interoperability with
other implementations.
Contributors
------------
Damian Gryski ([@dgryski](https://github.com/dgryski))
Dustin Sallings ([@dustin](https://github.com/dustin))
Contact
-------
[@bkaradzic](https://twitter.com/bkaradzic)
http://www.stuckingeometry.com
Project page
https://github.com/bkaradzic/go-lz4
License
-------
Copyright 2011-2012 Branimir Karadzic. All rights reserved.
Copyright 2013 Damian Gryski. All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDER ``AS IS'' AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
SHALL COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
THE POSSIBILITY OF SUCH DAMAGE.

View File

@ -1,23 +0,0 @@
// +build gofuzz
package lz4
import "encoding/binary"
func Fuzz(data []byte) int {
if len(data) < 4 {
return 0
}
ln := binary.LittleEndian.Uint32(data)
if ln > (1 << 21) {
return 0
}
if _, err := Decode(nil, data); err != nil {
return 0
}
return 1
}

View File

@ -1,74 +0,0 @@
package main
import (
"math/rand"
"github.com/bkaradzic/go-lz4"
// lz4's API matches snappy's, so we can easily see how it performs
// lz4 "code.google.com/p/snappy-go/snappy"
)
var input = `
ADVENTURE I. A SCANDAL IN BOHEMIA
I.
To Sherlock Holmes she is always THE woman. I have seldom heard
him mention her under any other name. In his eyes she eclipses
and predominates the whole of her sex. It was not that he felt
any emotion akin to love for Irene Adler. All emotions, and that
one particularly, were abhorrent to his cold, precise but
admirably balanced mind. He was, I take it, the most perfect
reasoning and observing machine that the world has seen, but as a
lover he would have placed himself in a false position. He never
spoke of the softer passions, save with a gibe and a sneer. They
were admirable things for the observer--excellent for drawing the
veil from men's motives and actions. But for the trained reasoner
to admit such intrusions into his own delicate and finely
adjusted temperament was to introduce a distracting factor which
might throw a doubt upon all his mental results. Grit in a
sensitive instrument, or a crack in one of his own high-power
lenses, would not be more disturbing than a strong emotion in a
nature such as his. And yet there was but one woman to him, and
that woman was the late Irene Adler, of dubious and questionable
memory.
I had seen little of Holmes lately. My marriage had drifted us
away from each other. My own complete happiness, and the
home-centred interests which rise up around the man who first
finds himself master of his own establishment, were sufficient to
absorb all my attention, while Holmes, who loathed every form of
society with his whole Bohemian soul, remained in our lodgings in
Baker Street, buried among his old books, and alternating from
week to week between cocaine and ambition, the drowsiness of the
drug, and the fierce energy of his own keen nature. He was still,
as ever, deeply attracted by the study of crime, and occupied his
immense faculties and extraordinary powers of observation in
following out those clues, and clearing up those mysteries which
had been abandoned as hopeless by the official police. From time
to time I heard some vague account of his doings: of his summons
to Odessa in the case of the Trepoff murder, of his clearing up
of the singular tragedy of the Atkinson brothers at Trincomalee,
and finally of the mission which he had accomplished so
delicately and successfully for the reigning family of Holland.
Beyond these signs of his activity, however, which I merely
shared with all the readers of the daily press, I knew little of
my former friend and companion.
`
func main() {
compressed, _ := lz4.Encode(nil, []byte(input))
modified := make([]byte, len(compressed))
for {
copy(modified, compressed)
for i := 0; i < 100; i++ {
modified[rand.Intn(len(compressed)-4)+4] = byte(rand.Intn(256))
}
lz4.Decode(nil, modified)
}
}

View File

@ -1,94 +0,0 @@
/*
* Copyright 2011 Branimir Karadzic. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification,
* are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDER ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
* SHALL COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
* OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
* THE POSSIBILITY OF SUCH DAMAGE.
*/
package main
import (
"flag"
"fmt"
"io/ioutil"
"log"
"os"
"runtime/pprof"
lz4 "github.com/bkaradzic/go-lz4"
)
var (
decompress = flag.Bool("d", false, "decompress")
)
func main() {
var optCPUProfile = flag.String("cpuprofile", "", "profile")
flag.Parse()
if *optCPUProfile != "" {
f, err := os.Create(*optCPUProfile)
if err != nil {
log.Fatal(err)
}
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}
args := flag.Args()
var data []byte
if len(args) < 2 {
fmt.Print("Usage: lz4 [-d] <input> <output>\n")
os.Exit(1)
}
input, err := os.OpenFile(args[0], os.O_RDONLY, 0644)
if err != nil {
fmt.Printf("Failed to open input file %s\n", args[0])
os.Exit(1)
}
defer input.Close()
if *decompress {
data, _ = ioutil.ReadAll(input)
data, err = lz4.Decode(nil, data)
if err != nil {
fmt.Println("Failed to decode:", err)
return
}
} else {
data, _ = ioutil.ReadAll(input)
data, err = lz4.Encode(nil, data)
if err != nil {
fmt.Println("Failed to encode:", err)
return
}
}
err = ioutil.WriteFile(args[1], data, 0644)
if err != nil {
fmt.Printf("Failed to open output file %s\n", args[1])
os.Exit(1)
}
}

View File

@ -1,199 +0,0 @@
/*
* Copyright 2011-2012 Branimir Karadzic. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification,
* are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDER ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
* SHALL COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
* OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
* THE POSSIBILITY OF SUCH DAMAGE.
*/
package lz4
import (
"encoding/binary"
"errors"
"io"
)
var (
// ErrCorrupt indicates the input was corrupt
ErrCorrupt = errors.New("corrupt input")
)
const (
mlBits = 4
mlMask = (1 << mlBits) - 1
runBits = 8 - mlBits
runMask = (1 << runBits) - 1
)
type decoder struct {
src []byte
dst []byte
spos uint32
dpos uint32
ref uint32
}
func (d *decoder) readByte() (uint8, error) {
if int(d.spos) == len(d.src) {
return 0, io.EOF
}
b := d.src[d.spos]
d.spos++
return b, nil
}
func (d *decoder) getLen() (uint32, error) {
length := uint32(0)
ln, err := d.readByte()
if err != nil {
return 0, ErrCorrupt
}
for ln == 255 {
length += 255
ln, err = d.readByte()
if err != nil {
return 0, ErrCorrupt
}
}
length += uint32(ln)
return length, nil
}
func (d *decoder) cp(length, decr uint32) {
if int(d.ref+length) < int(d.dpos) {
copy(d.dst[d.dpos:], d.dst[d.ref:d.ref+length])
} else {
for ii := uint32(0); ii < length; ii++ {
d.dst[d.dpos+ii] = d.dst[d.ref+ii]
}
}
d.dpos += length
d.ref += length - decr
}
func (d *decoder) finish(err error) error {
if err == io.EOF {
return nil
}
return err
}
// Decode returns the decoded form of src. The returned slice may be a
// subslice of dst if it was large enough to hold the entire decoded block.
func Decode(dst, src []byte) ([]byte, error) {
if len(src) < 4 {
return nil, ErrCorrupt
}
uncompressedLen := binary.LittleEndian.Uint32(src)
if uncompressedLen == 0 {
return nil, nil
}
if uncompressedLen > MaxInputSize {
return nil, ErrTooLarge
}
if dst == nil || len(dst) < int(uncompressedLen) {
dst = make([]byte, uncompressedLen)
}
d := decoder{src: src, dst: dst[:uncompressedLen], spos: 4}
decr := []uint32{0, 3, 2, 3}
for {
code, err := d.readByte()
if err != nil {
return d.dst, d.finish(err)
}
length := uint32(code >> mlBits)
if length == runMask {
ln, err := d.getLen()
if err != nil {
return nil, ErrCorrupt
}
length += ln
}
if int(d.spos+length) > len(d.src) || int(d.dpos+length) > len(d.dst) {
return nil, ErrCorrupt
}
for ii := uint32(0); ii < length; ii++ {
d.dst[d.dpos+ii] = d.src[d.spos+ii]
}
d.spos += length
d.dpos += length
if int(d.spos) == len(d.src) {
return d.dst, nil
}
if int(d.spos+2) >= len(d.src) {
return nil, ErrCorrupt
}
back := uint32(d.src[d.spos]) | uint32(d.src[d.spos+1])<<8
if back > d.dpos {
return nil, ErrCorrupt
}
d.spos += 2
d.ref = d.dpos - back
length = uint32(code & mlMask)
if length == mlMask {
ln, err := d.getLen()
if err != nil {
return nil, ErrCorrupt
}
length += ln
}
literal := d.dpos - d.ref
if literal < 4 {
if int(d.dpos+4) > len(d.dst) {
return nil, ErrCorrupt
}
d.cp(4, decr[literal])
} else {
length += 4
}
if d.dpos+length > uncompressedLen {
return nil, ErrCorrupt
}
d.cp(length, 0)
}
}

View File

@ -1,190 +0,0 @@
/*
* Copyright 2011-2012 Branimir Karadzic. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification,
* are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDER ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
* SHALL COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
* OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
* THE POSSIBILITY OF SUCH DAMAGE.
*/
package lz4
import (
"encoding/binary"
"errors"
)
const (
minMatch = 4
hashLog = 17
hashTableSize = 1 << hashLog
hashShift = (minMatch * 8) - hashLog
incompressible uint32 = 128
uninitHash = 0x88888888
// MaxInputSize is the largest buffer than can be compressed in a single block
MaxInputSize = 0x7E000000
)
var (
// ErrTooLarge indicates the input buffer was too large
ErrTooLarge = errors.New("input too large")
)
type encoder struct {
src []byte
dst []byte
hashTable []uint32
pos uint32
anchor uint32
dpos uint32
}
// CompressBound returns the maximum length of a lz4 block, given it's uncompressed length
func CompressBound(isize int) int {
if isize > MaxInputSize {
return 0
}
return isize + ((isize) / 255) + 16 + 4
}
func (e *encoder) writeLiterals(length, mlLen, pos uint32) {
ln := length
var code byte
if ln > runMask-1 {
code = runMask
} else {
code = byte(ln)
}
if mlLen > mlMask-1 {
e.dst[e.dpos] = (code << mlBits) + byte(mlMask)
} else {
e.dst[e.dpos] = (code << mlBits) + byte(mlLen)
}
e.dpos++
if code == runMask {
ln -= runMask
for ; ln > 254; ln -= 255 {
e.dst[e.dpos] = 255
e.dpos++
}
e.dst[e.dpos] = byte(ln)
e.dpos++
}
for ii := uint32(0); ii < length; ii++ {
e.dst[e.dpos+ii] = e.src[pos+ii]
}
e.dpos += length
}
// Encode returns the encoded form of src. The returned array may be a
// sub-slice of dst if it was large enough to hold the entire output.
func Encode(dst, src []byte) ([]byte, error) {
if len(src) >= MaxInputSize {
return nil, ErrTooLarge
}
if n := CompressBound(len(src)); len(dst) < n {
dst = make([]byte, n)
}
e := encoder{src: src, dst: dst, hashTable: make([]uint32, hashTableSize)}
binary.LittleEndian.PutUint32(dst, uint32(len(src)))
e.dpos = 4
var (
step uint32 = 1
limit = incompressible
)
for {
if int(e.pos)+12 >= len(e.src) {
e.writeLiterals(uint32(len(e.src))-e.anchor, 0, e.anchor)
return e.dst[:e.dpos], nil
}
sequence := uint32(e.src[e.pos+3])<<24 | uint32(e.src[e.pos+2])<<16 | uint32(e.src[e.pos+1])<<8 | uint32(e.src[e.pos+0])
hash := (sequence * 2654435761) >> hashShift
ref := e.hashTable[hash] + uninitHash
e.hashTable[hash] = e.pos - uninitHash
if ((e.pos-ref)>>16) != 0 || uint32(e.src[ref+3])<<24|uint32(e.src[ref+2])<<16|uint32(e.src[ref+1])<<8|uint32(e.src[ref+0]) != sequence {
if e.pos-e.anchor > limit {
limit <<= 1
step += 1 + (step >> 2)
}
e.pos += step
continue
}
if step > 1 {
e.hashTable[hash] = ref - uninitHash
e.pos -= step - 1
step = 1
continue
}
limit = incompressible
ln := e.pos - e.anchor
back := e.pos - ref
anchor := e.anchor
e.pos += minMatch
ref += minMatch
e.anchor = e.pos
for int(e.pos) < len(e.src)-5 && e.src[e.pos] == e.src[ref] {
e.pos++
ref++
}
mlLen := e.pos - e.anchor
e.writeLiterals(ln, mlLen, anchor)
e.dst[e.dpos] = uint8(back)
e.dst[e.dpos+1] = uint8(back >> 8)
e.dpos += 2
if mlLen > mlMask-1 {
mlLen -= mlMask
for mlLen > 254 {
mlLen -= 255
e.dst[e.dpos] = 255
e.dpos++
}
e.dst[e.dpos] = byte(mlLen)
e.dpos++
}
e.anchor = e.pos
}
}

View File

@ -1,24 +0,0 @@
This is free and unencumbered software released into the public domain.
Anyone is free to copy, modify, publish, use, compile, sell, or
distribute this software, either in source code form or as a compiled
binary, for any purpose, commercial or non-commercial, and by any
means.
In jurisdictions that recognize copyright laws, the author or authors
of this software dedicate any and all copyright interest in the
software to the public domain. We make this dedication for the benefit
of the public at large and to the detriment of our heirs and
successors. We intend this dedication to be an overt act of
relinquishment in perpetuity of all present and future rights to this
software under copyright law.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE.
For more information, please refer to <http://unlicense.org>

View File

@ -1,14 +0,0 @@
du
==
Get total and available disk space on a given volume.
Documentation
-------------
http://godoc.org/github.com/calmh/du
License
-------
Public Domain

View File

@ -1,21 +0,0 @@
package main
import (
"fmt"
"log"
"os"
"github.com/calmh/du"
)
var KB = int64(1024)
func main() {
usage, err := du.Get(os.Args[1])
if err != nil {
log.Fatal(err)
}
fmt.Println("Free:", usage.FreeBytes/(KB*KB), "MiB")
fmt.Println("Available:", usage.AvailBytes/(KB*KB), "MiB")
fmt.Println("Size:", usage.TotalBytes/(KB*KB), "MiB")
}

View File

@ -1,8 +0,0 @@
package du
// Usage holds information about total and available storage on a volume.
type Usage struct {
TotalBytes int64 // Size of volume
FreeBytes int64 // Unused size
AvailBytes int64 // Available to a non-privileged user
}

View File

@ -1,24 +0,0 @@
// +build !windows,!netbsd,!openbsd,!solaris
package du
import (
"path/filepath"
"syscall"
)
// Get returns the Usage of a given path, or an error if usage data is
// unavailable.
func Get(path string) (Usage, error) {
var stat syscall.Statfs_t
err := syscall.Statfs(filepath.Clean(path), &stat)
if err != nil {
return Usage{}, err
}
u := Usage{
FreeBytes: int64(stat.Bfree) * int64(stat.Bsize),
TotalBytes: int64(stat.Blocks) * int64(stat.Bsize),
AvailBytes: int64(stat.Bavail) * int64(stat.Bsize),
}
return u, nil
}

View File

@ -1,13 +0,0 @@
// +build netbsd openbsd solaris
package du
import "errors"
var ErrUnsupported = errors.New("unsupported platform")
// Get returns the Usage of a given path, or an error if usage data is
// unavailable.
func Get(path string) (Usage, error) {
return Usage{}, ErrUnsupported
}

View File

@ -1,27 +0,0 @@
package du
import (
"syscall"
"unsafe"
)
// Get returns the Usage of a given path, or an error if usage data is
// unavailable.
func Get(path string) (Usage, error) {
h := syscall.MustLoadDLL("kernel32.dll")
c := h.MustFindProc("GetDiskFreeSpaceExW")
var u Usage
ret, _, err := c.Call(
uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(path))),
uintptr(unsafe.Pointer(&u.FreeBytes)),
uintptr(unsafe.Pointer(&u.TotalBytes)),
uintptr(unsafe.Pointer(&u.AvailBytes)))
if ret == 0 {
return u, err
}
return u, nil
}

View File

@ -1,19 +0,0 @@
Copyright (C) 2014 Jakob Borg
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:
- The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

View File

@ -1,70 +0,0 @@
// Copyright (C) 2014 Jakob Borg
// Package luhn generates and validates Luhn mod N check digits.
package luhn
import (
"fmt"
"strings"
)
// An alphabet is a string of N characters, representing the digits of a given
// base N.
type Alphabet string
var (
Base32 Alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"
)
// Generate returns a check digit for the string s, which should be composed
// of characters from the Alphabet a.
func (a Alphabet) Generate(s string) (rune, error) {
if err := a.check(); err != nil {
return 0, err
}
factor := 1
sum := 0
n := len(a)
for i := range s {
codepoint := strings.IndexByte(string(a), s[i])
if codepoint == -1 {
return 0, fmt.Errorf("Digit %q not valid in alphabet %q", s[i], a)
}
addend := factor * codepoint
if factor == 2 {
factor = 1
} else {
factor = 2
}
addend = (addend / n) + (addend % n)
sum += addend
}
remainder := sum % n
checkCodepoint := (n - remainder) % n
return rune(a[checkCodepoint]), nil
}
// Validate returns true if the last character of the string s is correct, for
// a string s composed of characters in the alphabet a.
func (a Alphabet) Validate(s string) bool {
t := s[:len(s)-1]
c, err := a.Generate(t)
if err != nil {
return false
}
return rune(s[len(s)-1]) == c
}
// check returns an error if the given alphabet does not consist of unique characters
func (a Alphabet) check() error {
cm := make(map[byte]bool, len(a))
for i := range a {
if cm[a[i]] {
return fmt.Errorf("Digit %q non-unique in alphabet %q", a[i], a)
}
cm[a[i]] = true
}
return nil
}

View File

@ -1 +0,0 @@
coverage.out

View File

@ -1,19 +0,0 @@
language: go
go:
- tip
install:
- export PATH=$PATH:$HOME/gopath/bin
- go get golang.org/x/tools/cover
- go get github.com/mattn/goveralls
script:
- ./generate.sh
- go test -coverprofile=coverage.out
after_success:
- goveralls -coverprofile=coverage.out -service=travis-ci -package=calmh/xdr -repotoken="$COVERALLS_TOKEN"
env:
global:
secure: SmgnrGfp2zLrA44ChRMpjPeujubt9veZ8Fx/OseMWECmacyV5N/TuDhzIbwo6QwV4xB0sBacoPzvxQbJRVjNKsPiSu72UbcQmQ7flN4Tf7nW09tSh1iW8NgrpBCq/3UYLoBu2iPBEBKm93IK0aGNAKs6oEkB0fU27iTVBwiTXOY=

View File

@ -1,19 +0,0 @@
Copyright (C) 2014 Jakob Borg.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
deal in the Software without restriction, including without limitation the
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
- The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
IN THE SOFTWARE.

View File

@ -1,12 +0,0 @@
xdr
===
[![Build Status](https://img.shields.io/circleci/project/calmh/xdr.svg?style=flat-square)](https://circleci.com/gh/calmh/xdr)
[![Coverage Status](https://img.shields.io/coveralls/calmh/xdr.svg?style=flat)](https://coveralls.io/r/calmh/xdr?branch=master)
[![API Documentation](http://img.shields.io/badge/api-Godoc-blue.svg?style=flat)](http://godoc.org/github.com/calmh/xdr)
[![MIT License](http://img.shields.io/badge/license-MIT-blue.svg?style=flat)](http://opensource.org/licenses/MIT)
This is an XDR encoding/decoding library. It uses code generation and
not reflection. It supports the IPDR bastardized XDR format when built
with `-tags ipdr`.

View File

@ -1,3 +0,0 @@
dependencies:
post:
- ./generate.sh

View File

@ -1,530 +0,0 @@
// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
// is governed by an MIT-style license that can be found in the LICENSE file.
package main
import (
"bytes"
"flag"
"fmt"
"go/ast"
"go/format"
"go/parser"
"go/token"
"io"
"log"
"os"
"regexp"
"strconv"
"strings"
"text/template"
)
type fieldInfo struct {
Name string
IsBasic bool // handled by one the native Read/WriteUint64 etc functions
IsSlice bool // field is a slice of FieldType
FieldType string // original type of field, i.e. "int"
Encoder string // the encoder name, i.e. "Uint64" for Read/WriteUint64
Convert string // what to convert to when encoding, i.e. "uint64"
Max int // max size for slices and strings
Submax int // max size for strings inside slices
}
type structInfo struct {
Name string
Fields []fieldInfo
}
var headerTpl = template.Must(template.New("header").Parse(`// ************************************************************
// This file is automatically generated by genxdr. Do not edit.
// ************************************************************
package {{.Package}}
import (
"bytes"
"io"
"github.com/calmh/xdr"
)
`))
var encodeTpl = template.Must(template.New("encoder").Parse(`
func (o {{.TypeName}}) EncodeXDR(w io.Writer) (int, error) {
var xw = xdr.NewWriter(w)
return o.EncodeXDRInto(xw)
}//+n
func (o {{.TypeName}}) MarshalXDR() ([]byte, error) {
return o.AppendXDR(make([]byte, 0, 128))
}//+n
func (o {{.TypeName}}) MustMarshalXDR() []byte {
bs, err := o.MarshalXDR()
if err != nil {
panic(err)
}
return bs
}//+n
func (o {{.TypeName}}) AppendXDR(bs []byte) ([]byte, error) {
var aw = xdr.AppendWriter(bs)
var xw = xdr.NewWriter(&aw)
_, err := o.EncodeXDRInto(xw)
return []byte(aw), err
}//+n
func (o {{.TypeName}}) EncodeXDRInto(xw *xdr.Writer) (int, error) {
{{range $fieldInfo := .Fields}}
{{if not $fieldInfo.IsSlice}}
{{if ne $fieldInfo.Convert ""}}
xw.Write{{$fieldInfo.Encoder}}({{$fieldInfo.Convert}}(o.{{$fieldInfo.Name}}))
{{else if $fieldInfo.IsBasic}}
{{if ge $fieldInfo.Max 1}}
if l := len(o.{{$fieldInfo.Name}}); l > {{$fieldInfo.Max}} {
return xw.Tot(), xdr.ElementSizeExceeded("{{$fieldInfo.Name}}", l, {{$fieldInfo.Max}})
}
{{end}}
xw.Write{{$fieldInfo.Encoder}}(o.{{$fieldInfo.Name}})
{{else}}
_, err := o.{{$fieldInfo.Name}}.EncodeXDRInto(xw)
if err != nil {
return xw.Tot(), err
}
{{end}}
{{else}}
{{if ge $fieldInfo.Max 1}}
if l := len(o.{{$fieldInfo.Name}}); l > {{$fieldInfo.Max}} {
return xw.Tot(), xdr.ElementSizeExceeded("{{$fieldInfo.Name}}", l, {{$fieldInfo.Max}})
}
{{end}}
xw.WriteUint32(uint32(len(o.{{$fieldInfo.Name}})))
for i := range o.{{$fieldInfo.Name}} {
{{if ne $fieldInfo.Convert ""}}
xw.Write{{$fieldInfo.Encoder}}({{$fieldInfo.Convert}}(o.{{$fieldInfo.Name}}[i]))
{{else if $fieldInfo.IsBasic}}
xw.Write{{$fieldInfo.Encoder}}(o.{{$fieldInfo.Name}}[i])
{{else}}
_, err := o.{{$fieldInfo.Name}}[i].EncodeXDRInto(xw)
if err != nil {
return xw.Tot(), err
}
{{end}}
}
{{end}}
{{end}}
return xw.Tot(), xw.Error()
}//+n
func (o *{{.TypeName}}) DecodeXDR(r io.Reader) error {
xr := xdr.NewReader(r)
return o.DecodeXDRFrom(xr)
}//+n
func (o *{{.TypeName}}) UnmarshalXDR(bs []byte) error {
var br = bytes.NewReader(bs)
var xr = xdr.NewReader(br)
return o.DecodeXDRFrom(xr)
}//+n
func (o *{{.TypeName}}) DecodeXDRFrom(xr *xdr.Reader) error {
{{range $fieldInfo := .Fields}}
{{if not $fieldInfo.IsSlice}}
{{if ne $fieldInfo.Convert ""}}
o.{{$fieldInfo.Name}} = {{$fieldInfo.FieldType}}(xr.Read{{$fieldInfo.Encoder}}())
{{else if $fieldInfo.IsBasic}}
{{if ge $fieldInfo.Max 1}}
o.{{$fieldInfo.Name}} = xr.Read{{$fieldInfo.Encoder}}Max({{$fieldInfo.Max}})
{{else}}
o.{{$fieldInfo.Name}} = xr.Read{{$fieldInfo.Encoder}}()
{{end}}
{{else}}
(&o.{{$fieldInfo.Name}}).DecodeXDRFrom(xr)
{{end}}
{{else}}
_{{$fieldInfo.Name}}Size := int(xr.ReadUint32())
if _{{$fieldInfo.Name}}Size < 0 {
return xdr.ElementSizeExceeded("{{$fieldInfo.Name}}", _{{$fieldInfo.Name}}Size, {{$fieldInfo.Max}})
}
{{if ge $fieldInfo.Max 1}}
if _{{$fieldInfo.Name}}Size > {{$fieldInfo.Max}} {
return xdr.ElementSizeExceeded("{{$fieldInfo.Name}}", _{{$fieldInfo.Name}}Size, {{$fieldInfo.Max}})
}
{{end}}
o.{{$fieldInfo.Name}} = make([]{{$fieldInfo.FieldType}}, _{{$fieldInfo.Name}}Size)
for i := range o.{{$fieldInfo.Name}} {
{{if ne $fieldInfo.Convert ""}}
o.{{$fieldInfo.Name}}[i] = {{$fieldInfo.FieldType}}(xr.Read{{$fieldInfo.Encoder}}())
{{else if $fieldInfo.IsBasic}}
{{if ge $fieldInfo.Submax 1}}
o.{{$fieldInfo.Name}}[i] = xr.Read{{$fieldInfo.Encoder}}Max({{$fieldInfo.Submax}})
{{else}}
o.{{$fieldInfo.Name}}[i] = xr.Read{{$fieldInfo.Encoder}}()
{{end}}
{{else}}
(&o.{{$fieldInfo.Name}}[i]).DecodeXDRFrom(xr)
{{end}}
}
{{end}}
{{end}}
return xr.Error()
}`))
var emptyTypeTpl = template.Must(template.New("encoder").Parse(`
func (o {{.TypeName}}) EncodeXDR(w io.Writer) (int, error) {
return 0, nil
}//+n
func (o {{.TypeName}}) MarshalXDR() ([]byte, error) {
return nil, nil
}//+n
func (o {{.TypeName}}) MustMarshalXDR() []byte {
return nil
}//+n
func (o {{.TypeName}}) AppendXDR(bs []byte) ([]byte, error) {
return bs, nil
}//+n
func (o {{.TypeName}}) EncodeXDRInto(xw *xdr.Writer) (int, error) {
return xw.Tot(), xw.Error()
}//+n
func (o *{{.TypeName}}) DecodeXDR(r io.Reader) error {
return nil
}//+n
func (o *{{.TypeName}}) UnmarshalXDR(bs []byte) error {
return nil
}//+n
func (o *{{.TypeName}}) DecodeXDRFrom(xr *xdr.Reader) error {
return xr.Error()
}`))
var maxRe = regexp.MustCompile(`(?:\Wmax:)(\d+)(?:\s*,\s*(\d+))?`)
type typeSet struct {
Type string
Encoder string
}
var xdrEncoders = map[string]typeSet{
"int8": typeSet{"uint8", "Uint8"},
"uint8": typeSet{"", "Uint8"},
"int16": typeSet{"uint16", "Uint16"},
"uint16": typeSet{"", "Uint16"},
"int32": typeSet{"uint32", "Uint32"},
"uint32": typeSet{"", "Uint32"},
"int64": typeSet{"uint64", "Uint64"},
"uint64": typeSet{"", "Uint64"},
"int": typeSet{"uint64", "Uint64"},
"string": typeSet{"", "String"},
"[]byte": typeSet{"", "Bytes"},
"bool": typeSet{"", "Bool"},
}
func handleStruct(t *ast.StructType) []fieldInfo {
var fs []fieldInfo
for _, sf := range t.Fields.List {
if len(sf.Names) == 0 {
// We don't handle anonymous fields
continue
}
fn := sf.Names[0].Name
var max1, max2 int
if sf.Comment != nil {
c := sf.Comment.List[0].Text
m := maxRe.FindStringSubmatch(c)
if len(m) >= 2 {
max1, _ = strconv.Atoi(m[1])
}
if len(m) >= 3 {
max2, _ = strconv.Atoi(m[2])
}
if strings.Contains(c, "noencode") {
continue
}
}
var f fieldInfo
switch ft := sf.Type.(type) {
case *ast.Ident:
tn := ft.Name
if enc, ok := xdrEncoders[tn]; ok {
f = fieldInfo{
Name: fn,
IsBasic: true,
FieldType: tn,
Encoder: enc.Encoder,
Convert: enc.Type,
Max: max1,
Submax: max2,
}
} else {
f = fieldInfo{
Name: fn,
IsBasic: false,
FieldType: tn,
Max: max1,
Submax: max2,
}
}
case *ast.ArrayType:
if ft.Len != nil {
// We don't handle arrays
continue
}
tn := ft.Elt.(*ast.Ident).Name
if enc, ok := xdrEncoders["[]"+tn]; ok {
f = fieldInfo{
Name: fn,
IsBasic: true,
FieldType: tn,
Encoder: enc.Encoder,
Convert: enc.Type,
Max: max1,
Submax: max2,
}
} else if enc, ok := xdrEncoders[tn]; ok {
f = fieldInfo{
Name: fn,
IsBasic: true,
IsSlice: true,
FieldType: tn,
Encoder: enc.Encoder,
Convert: enc.Type,
Max: max1,
Submax: max2,
}
} else {
f = fieldInfo{
Name: fn,
IsSlice: true,
FieldType: tn,
Max: max1,
Submax: max2,
}
}
case *ast.SelectorExpr:
f = fieldInfo{
Name: fn,
FieldType: ft.Sel.Name,
Max: max1,
Submax: max2,
}
}
fs = append(fs, f)
}
return fs
}
func generateCode(output io.Writer, s structInfo) {
name := s.Name
fs := s.Fields
var buf bytes.Buffer
var err error
if len(fs) == 0 {
// This is an empty type. We can create a quite simple codec for it.
err = emptyTypeTpl.Execute(&buf, map[string]interface{}{"TypeName": name})
} else {
// Generate with the default template.
err = encodeTpl.Execute(&buf, map[string]interface{}{"TypeName": name, "Fields": fs})
}
if err != nil {
panic(err)
}
bs := regexp.MustCompile(`(\s*\n)+`).ReplaceAll(buf.Bytes(), []byte("\n"))
bs = bytes.Replace(bs, []byte("//+n"), []byte("\n"), -1)
bs, err = format.Source(bs)
if err != nil {
panic(err)
}
fmt.Fprintln(output, string(bs))
}
func uncamelize(s string) string {
return regexp.MustCompile("[a-z][A-Z]").ReplaceAllStringFunc(s, func(camel string) string {
return camel[:1] + " " + camel[1:]
})
}
func generateDiagram(output io.Writer, s structInfo) {
sn := s.Name
fs := s.Fields
fmt.Fprintln(output, sn+" Structure:")
if len(fs) == 0 {
fmt.Fprintln(output, "(contains no fields)")
fmt.Fprintln(output)
fmt.Fprintln(output)
return
}
fmt.Fprintln(output)
fmt.Fprintln(output, " 0 1 2 3")
fmt.Fprintln(output, " 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1")
line := "+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+"
fmt.Fprintln(output, line)
for _, f := range fs {
tn := f.FieldType
name := uncamelize(f.Name)
if f.IsSlice {
fmt.Fprintf(output, "| %s |\n", center("Number of "+name, 61))
fmt.Fprintln(output, line)
}
switch tn {
case "bool":
fmt.Fprintf(output, "| %s |V|\n", center(name+" (V=0 or 1)", 59))
fmt.Fprintln(output, line)
case "int16", "uint16":
fmt.Fprintf(output, "| %s | %s |\n", center("0x0000", 29), center(name, 29))
fmt.Fprintln(output, line)
case "int32", "uint32":
fmt.Fprintf(output, "| %s |\n", center(name, 61))
fmt.Fprintln(output, line)
case "int64", "uint64":
fmt.Fprintf(output, "| %-61s |\n", "")
fmt.Fprintf(output, "+ %s +\n", center(name+" (64 bits)", 61))
fmt.Fprintf(output, "| %-61s |\n", "")
fmt.Fprintln(output, line)
case "string", "byte": // XXX We assume slice of byte!
fmt.Fprintf(output, "| %s |\n", center("Length of "+name, 61))
fmt.Fprintln(output, line)
fmt.Fprintf(output, "/ %61s /\n", "")
fmt.Fprintf(output, "\\ %s \\\n", center(name+" (variable length)", 61))
fmt.Fprintf(output, "/ %61s /\n", "")
fmt.Fprintln(output, line)
default:
if f.IsSlice {
tn = "Zero or more " + tn + " Structures"
fmt.Fprintf(output, "/ %s /\n", center("", 61))
fmt.Fprintf(output, "\\ %s \\\n", center(tn, 61))
fmt.Fprintf(output, "/ %s /\n", center("", 61))
} else {
tn = tn + " Structure"
fmt.Fprintf(output, "/ %s /\n", center("", 61))
fmt.Fprintf(output, "\\ %s \\\n", center(tn, 61))
fmt.Fprintf(output, "/ %s /\n", center("", 61))
}
fmt.Fprintln(output, line)
}
}
fmt.Fprintln(output)
fmt.Fprintln(output)
}
func generateXdr(output io.Writer, s structInfo) {
sn := s.Name
fs := s.Fields
fmt.Fprintf(output, "struct %s {\n", sn)
for _, f := range fs {
tn := f.FieldType
fn := f.Name
suf := ""
l := ""
if f.Max > 0 {
l = strconv.Itoa(f.Max)
}
if f.IsSlice {
suf = "<" + l + ">"
}
switch tn {
case "int16", "int32":
fmt.Fprintf(output, "\tint %s%s;\n", fn, suf)
case "uint16", "uint32":
fmt.Fprintf(output, "\tunsigned int %s%s;\n", fn, suf)
case "int64":
fmt.Fprintf(output, "\thyper %s%s;\n", fn, suf)
case "uint64":
fmt.Fprintf(output, "\tunsigned hyper %s%s;\n", fn, suf)
case "string":
fmt.Fprintf(output, "\tstring %s<%s>;\n", fn, l)
case "byte":
fmt.Fprintf(output, "\topaque %s<%s>;\n", fn, l)
default:
fmt.Fprintf(output, "\t%s %s%s;\n", tn, fn, suf)
}
}
fmt.Fprintln(output, "}")
fmt.Fprintln(output)
}
func center(s string, w int) string {
w -= len(s)
l := w / 2
r := l
if l+r < w {
r++
}
return strings.Repeat(" ", l) + s + strings.Repeat(" ", r)
}
func inspector(structs *[]structInfo) func(ast.Node) bool {
return func(n ast.Node) bool {
switch n := n.(type) {
case *ast.TypeSpec:
switch t := n.Type.(type) {
case *ast.StructType:
name := n.Name.Name
fs := handleStruct(t)
*structs = append(*structs, structInfo{name, fs})
}
return false
default:
return true
}
}
}
func main() {
outputFile := flag.String("o", "", "Output file, blank for stdout")
flag.Parse()
fname := flag.Arg(0)
fset := token.NewFileSet()
f, err := parser.ParseFile(fset, fname, nil, parser.ParseComments)
if err != nil {
log.Fatal(err)
}
var structs []structInfo
i := inspector(&structs)
ast.Inspect(f, i)
var output io.Writer = os.Stdout
if *outputFile != "" {
fd, err := os.Create(*outputFile)
if err != nil {
log.Fatal(err)
}
output = fd
}
headerTpl.Execute(output, map[string]string{"Package": f.Name.Name})
for _, s := range structs {
fmt.Fprintf(output, "\n/*\n\n")
generateDiagram(output, s)
generateXdr(output, s)
fmt.Fprintf(output, "*/\n")
generateCode(output, s)
}
}

View File

@ -1,16 +0,0 @@
// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
// is governed by an MIT-style license that can be found in the LICENSE file.
package xdr
import (
"log"
"os"
)
var (
debug = len(os.Getenv("XDRTRACE")) > 0
dl = log.New(os.Stdout, "xdr: ", log.Lshortfile|log.Ltime|log.Lmicroseconds)
)
const maxDebugBytes = 32

View File

@ -1,5 +0,0 @@
// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
// is governed by an MIT-style license that can be found in the LICENSE file.
// Package xdr implements an XDR (RFC 4506) encoder/decoder.
package xdr

View File

@ -1,4 +0,0 @@
#!/bin/sh
go run cmd/genxdr/main.go -- bench_test.go > bench_xdr_test.go
go run cmd/genxdr/main.go -- encdec_test.go > encdec_xdr_test.go

View File

@ -1,10 +0,0 @@
// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
// is governed by an MIT-style license that can be found in the LICENSE file.
// +build ipdr
package xdr
func pad(l int) int {
return 0
}

View File

@ -1,14 +0,0 @@
// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
// is governed by an MIT-style license that can be found in the LICENSE file.
// +build !ipdr
package xdr
func pad(l int) int {
d := l % 4
if d == 0 {
return 0
}
return 4 - d
}

View File

@ -1,171 +0,0 @@
// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
// All rights reserved. Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.
package xdr
import (
"fmt"
"io"
"reflect"
"unsafe"
)
type Reader struct {
r io.Reader
err error
b [8]byte
}
func NewReader(r io.Reader) *Reader {
return &Reader{
r: r,
}
}
func (r *Reader) ReadRaw(bs []byte) (int, error) {
if r.err != nil {
return 0, r.err
}
var n int
n, r.err = io.ReadFull(r.r, bs)
return n, r.err
}
func (r *Reader) ReadString() string {
return r.ReadStringMax(0)
}
func (r *Reader) ReadStringMax(max int) string {
buf := r.ReadBytesMaxInto(max, nil)
bh := (*reflect.SliceHeader)(unsafe.Pointer(&buf))
sh := reflect.StringHeader{
Data: bh.Data,
Len: bh.Len,
}
return *((*string)(unsafe.Pointer(&sh)))
}
func (r *Reader) ReadBytes() []byte {
return r.ReadBytesInto(nil)
}
func (r *Reader) ReadBytesMax(max int) []byte {
return r.ReadBytesMaxInto(max, nil)
}
func (r *Reader) ReadBytesInto(dst []byte) []byte {
return r.ReadBytesMaxInto(0, dst)
}
func (r *Reader) ReadBytesMaxInto(max int, dst []byte) []byte {
if r.err != nil {
return nil
}
l := int(r.ReadUint32())
if r.err != nil {
return nil
}
if l < 0 || max > 0 && l > max {
// l may be negative on 32 bit builds
r.err = ElementSizeExceeded("bytes field", l, max)
return nil
}
if fullLen := l + pad(l); fullLen > len(dst) {
dst = make([]byte, fullLen)
} else {
dst = dst[:fullLen]
}
var n int
n, r.err = io.ReadFull(r.r, dst)
if r.err != nil {
if debug {
dl.Printf("rd bytes (%d): %v", len(dst), r.err)
}
return nil
}
if debug {
if n > maxDebugBytes {
dl.Printf("rd bytes (%d): %x...", len(dst), dst[:maxDebugBytes])
} else {
dl.Printf("rd bytes (%d): %x", len(dst), dst)
}
}
return dst[:l]
}
func (r *Reader) ReadBool() bool {
return r.ReadUint8() != 0
}
func (r *Reader) ReadUint32() uint32 {
if r.err != nil {
return 0
}
_, r.err = io.ReadFull(r.r, r.b[:4])
if r.err != nil {
if debug {
dl.Printf("rd uint32: %v", r.err)
}
return 0
}
v := uint32(r.b[3]) | uint32(r.b[2])<<8 | uint32(r.b[1])<<16 | uint32(r.b[0])<<24
if debug {
dl.Printf("rd uint32=%d (0x%08x)", v, v)
}
return v
}
func (r *Reader) ReadUint64() uint64 {
if r.err != nil {
return 0
}
_, r.err = io.ReadFull(r.r, r.b[:8])
if r.err != nil {
if debug {
dl.Printf("rd uint64: %v", r.err)
}
return 0
}
v := uint64(r.b[7]) | uint64(r.b[6])<<8 | uint64(r.b[5])<<16 | uint64(r.b[4])<<24 |
uint64(r.b[3])<<32 | uint64(r.b[2])<<40 | uint64(r.b[1])<<48 | uint64(r.b[0])<<56
if debug {
dl.Printf("rd uint64=%d (0x%016x)", v, v)
}
return v
}
type XDRError struct {
op string
err error
}
func (e XDRError) Error() string {
return "xdr " + e.op + ": " + e.err.Error()
}
func (e XDRError) IsEOF() bool {
return e.err == io.EOF
}
func (r *Reader) Error() error {
if r.err == nil {
return nil
}
return XDRError{"read", r.err}
}
func ElementSizeExceeded(field string, size, limit int) error {
return fmt.Errorf("%s exceeds size limit; %d > %d", field, size, limit)
}

View File

@ -1,49 +0,0 @@
// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
// All rights reserved. Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.
// +build ipdr
package xdr
import "io"
func (r *Reader) ReadUint8() uint8 {
if r.err != nil {
return 0
}
_, r.err = io.ReadFull(r.r, r.b[:1])
if r.err != nil {
if debug {
dl.Printf("rd uint8: %v", r.err)
}
return 0
}
if debug {
dl.Printf("rd uint8=%d (0x%02x)", r.b[0], r.b[0])
}
return r.b[0]
}
func (r *Reader) ReadUint16() uint16 {
if r.err != nil {
return 0
}
_, r.err = io.ReadFull(r.r, r.b[:2])
if r.err != nil {
if debug {
dl.Printf("rd uint16: %v", r.err)
}
return 0
}
v := uint16(r.b[1]) | uint16(r.b[0])<<8
if debug {
dl.Printf("rd uint16=%d (0x%04x)", v, v)
}
return v
}

View File

@ -1,15 +0,0 @@
// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
// All rights reserved. Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.
// +build !ipdr
package xdr
func (r *Reader) ReadUint8() uint8 {
return uint8(r.ReadUint32())
}
func (r *Reader) ReadUint16() uint16 {
return uint16(r.ReadUint32())
}

View File

@ -1,146 +0,0 @@
// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
// is governed by an MIT-style license that can be found in the LICENSE file.
package xdr
import (
"io"
"reflect"
"unsafe"
)
var padBytes = []byte{0, 0, 0}
type Writer struct {
w io.Writer
tot int
err error
b [8]byte
}
type AppendWriter []byte
func (w *AppendWriter) Write(bs []byte) (int, error) {
*w = append(*w, bs...)
return len(bs), nil
}
func NewWriter(w io.Writer) *Writer {
return &Writer{
w: w,
}
}
func (w *Writer) WriteRaw(bs []byte) (int, error) {
if w.err != nil {
return 0, w.err
}
var n int
n, w.err = w.w.Write(bs)
return n, w.err
}
func (w *Writer) WriteString(s string) (int, error) {
sh := *((*reflect.StringHeader)(unsafe.Pointer(&s)))
bh := reflect.SliceHeader{
Data: sh.Data,
Len: sh.Len,
Cap: sh.Len,
}
return w.WriteBytes(*(*[]byte)(unsafe.Pointer(&bh)))
}
func (w *Writer) WriteBytes(bs []byte) (int, error) {
if w.err != nil {
return 0, w.err
}
w.WriteUint32(uint32(len(bs)))
if w.err != nil {
return 0, w.err
}
if debug {
if len(bs) > maxDebugBytes {
dl.Printf("wr bytes (%d): %x...", len(bs), bs[:maxDebugBytes])
} else {
dl.Printf("wr bytes (%d): %x", len(bs), bs)
}
}
var l, n int
n, w.err = w.w.Write(bs)
l += n
if p := pad(len(bs)); w.err == nil && p > 0 {
n, w.err = w.w.Write(padBytes[:p])
l += n
}
w.tot += l
return l, w.err
}
func (w *Writer) WriteBool(v bool) (int, error) {
if v {
return w.WriteUint8(1)
} else {
return w.WriteUint8(0)
}
}
func (w *Writer) WriteUint32(v uint32) (int, error) {
if w.err != nil {
return 0, w.err
}
if debug {
dl.Printf("wr uint32=%d", v)
}
w.b[0] = byte(v >> 24)
w.b[1] = byte(v >> 16)
w.b[2] = byte(v >> 8)
w.b[3] = byte(v)
var l int
l, w.err = w.w.Write(w.b[:4])
w.tot += l
return l, w.err
}
func (w *Writer) WriteUint64(v uint64) (int, error) {
if w.err != nil {
return 0, w.err
}
if debug {
dl.Printf("wr uint64=%d", v)
}
w.b[0] = byte(v >> 56)
w.b[1] = byte(v >> 48)
w.b[2] = byte(v >> 40)
w.b[3] = byte(v >> 32)
w.b[4] = byte(v >> 24)
w.b[5] = byte(v >> 16)
w.b[6] = byte(v >> 8)
w.b[7] = byte(v)
var l int
l, w.err = w.w.Write(w.b[:8])
w.tot += l
return l, w.err
}
func (w *Writer) Tot() int {
return w.tot
}
func (w *Writer) Error() error {
if w.err == nil {
return nil
}
return XDRError{"write", w.err}
}

View File

@ -1,41 +0,0 @@
// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
// is governed by an MIT-style license that can be found in the LICENSE file.
// +build ipdr
package xdr
func (w *Writer) WriteUint8(v uint8) (int, error) {
if w.err != nil {
return 0, w.err
}
if debug {
dl.Printf("wr uint8=%d", v)
}
w.b[0] = byte(v)
var l int
l, w.err = w.w.Write(w.b[:1])
w.tot += l
return l, w.err
}
func (w *Writer) WriteUint16(v uint16) (int, error) {
if w.err != nil {
return 0, w.err
}
if debug {
dl.Printf("wr uint8=%d", v)
}
w.b[0] = byte(v >> 8)
w.b[1] = byte(v)
var l int
l, w.err = w.w.Write(w.b[:2])
w.tot += l
return l, w.err
}

View File

@ -1,14 +0,0 @@
// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
// is governed by an MIT-style license that can be found in the LICENSE file.
// +build !ipdr
package xdr
func (w *Writer) WriteUint8(v uint8) (int, error) {
return w.WriteUint32(uint32(v))
}
func (w *Writer) WriteUint16(v uint16) (int, error) {
return w.WriteUint32(uint32(v))
}

View File

@ -1,121 +0,0 @@
/*
Copyright 2013 Google Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package lru implements an LRU cache.
package lru
import "container/list"
// Cache is an LRU cache. It is not safe for concurrent access.
type Cache struct {
// MaxEntries is the maximum number of cache entries before
// an item is evicted. Zero means no limit.
MaxEntries int
// OnEvicted optionally specificies a callback function to be
// executed when an entry is purged from the cache.
OnEvicted func(key Key, value interface{})
ll *list.List
cache map[interface{}]*list.Element
}
// A Key may be any value that is comparable. See http://golang.org/ref/spec#Comparison_operators
type Key interface{}
type entry struct {
key Key
value interface{}
}
// New creates a new Cache.
// If maxEntries is zero, the cache has no limit and it's assumed
// that eviction is done by the caller.
func New(maxEntries int) *Cache {
return &Cache{
MaxEntries: maxEntries,
ll: list.New(),
cache: make(map[interface{}]*list.Element),
}
}
// Add adds a value to the cache.
func (c *Cache) Add(key Key, value interface{}) {
if c.cache == nil {
c.cache = make(map[interface{}]*list.Element)
c.ll = list.New()
}
if ee, ok := c.cache[key]; ok {
c.ll.MoveToFront(ee)
ee.Value.(*entry).value = value
return
}
ele := c.ll.PushFront(&entry{key, value})
c.cache[key] = ele
if c.MaxEntries != 0 && c.ll.Len() > c.MaxEntries {
c.RemoveOldest()
}
}
// Get looks up a key's value from the cache.
func (c *Cache) Get(key Key) (value interface{}, ok bool) {
if c.cache == nil {
return
}
if ele, hit := c.cache[key]; hit {
c.ll.MoveToFront(ele)
return ele.Value.(*entry).value, true
}
return
}
// Remove removes the provided key from the cache.
func (c *Cache) Remove(key Key) {
if c.cache == nil {
return
}
if ele, hit := c.cache[key]; hit {
c.removeElement(ele)
}
}
// RemoveOldest removes the oldest item from the cache.
func (c *Cache) RemoveOldest() {
if c.cache == nil {
return
}
ele := c.ll.Back()
if ele != nil {
c.removeElement(ele)
}
}
func (c *Cache) removeElement(e *list.Element) {
c.ll.Remove(e)
kv := e.Value.(*entry)
delete(c.cache, kv.key)
if c.OnEvicted != nil {
c.OnEvicted(kv.key, kv.value)
}
}
// Len returns the number of items in the cache.
func (c *Cache) Len() int {
if c.cache == nil {
return 0
}
return c.ll.Len()
}

View File

@ -1,191 +0,0 @@
All files in this repository are licensed as follows. If you contribute
to this repository, it is assumed that you license your contribution
under the same license unless you state otherwise.
All files Copyright (C) 2015 Canonical Ltd. unless otherwise specified in the file.
This software is licensed under the LGPLv3, included below.
As a special exception to the GNU Lesser General Public License version 3
("LGPL3"), the copyright holders of this Library give you permission to
convey to a third party a Combined Work that links statically or dynamically
to this Library without providing any Minimal Corresponding Source or
Minimal Application Code as set out in 4d or providing the installation
information set out in section 4e, provided that you comply with the other
provisions of LGPL3 and provided that you meet, for the Application the
terms and conditions of the license(s) which apply to the Application.
Except as stated in this special exception, the provisions of LGPL3 will
continue to comply in full to this Library. If you modify this Library, you
may apply this exception to your version of this Library, but you are not
obliged to do so. If you do not wish to do so, delete this exception
statement from your version. This exception does not (and cannot) modify any
license terms which apply to the Application, with which you must still
comply.
GNU LESSER GENERAL PUBLIC LICENSE
Version 3, 29 June 2007
Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
This version of the GNU Lesser General Public License incorporates
the terms and conditions of version 3 of the GNU General Public
License, supplemented by the additional permissions listed below.
0. Additional Definitions.
As used herein, "this License" refers to version 3 of the GNU Lesser
General Public License, and the "GNU GPL" refers to version 3 of the GNU
General Public License.
"The Library" refers to a covered work governed by this License,
other than an Application or a Combined Work as defined below.
An "Application" is any work that makes use of an interface provided
by the Library, but which is not otherwise based on the Library.
Defining a subclass of a class defined by the Library is deemed a mode
of using an interface provided by the Library.
A "Combined Work" is a work produced by combining or linking an
Application with the Library. The particular version of the Library
with which the Combined Work was made is also called the "Linked
Version".
The "Minimal Corresponding Source" for a Combined Work means the
Corresponding Source for the Combined Work, excluding any source code
for portions of the Combined Work that, considered in isolation, are
based on the Application, and not on the Linked Version.
The "Corresponding Application Code" for a Combined Work means the
object code and/or source code for the Application, including any data
and utility programs needed for reproducing the Combined Work from the
Application, but excluding the System Libraries of the Combined Work.
1. Exception to Section 3 of the GNU GPL.
You may convey a covered work under sections 3 and 4 of this License
without being bound by section 3 of the GNU GPL.
2. Conveying Modified Versions.
If you modify a copy of the Library, and, in your modifications, a
facility refers to a function or data to be supplied by an Application
that uses the facility (other than as an argument passed when the
facility is invoked), then you may convey a copy of the modified
version:
a) under this License, provided that you make a good faith effort to
ensure that, in the event an Application does not supply the
function or data, the facility still operates, and performs
whatever part of its purpose remains meaningful, or
b) under the GNU GPL, with none of the additional permissions of
this License applicable to that copy.
3. Object Code Incorporating Material from Library Header Files.
The object code form of an Application may incorporate material from
a header file that is part of the Library. You may convey such object
code under terms of your choice, provided that, if the incorporated
material is not limited to numerical parameters, data structure
layouts and accessors, or small macros, inline functions and templates
(ten or fewer lines in length), you do both of the following:
a) Give prominent notice with each copy of the object code that the
Library is used in it and that the Library and its use are
covered by this License.
b) Accompany the object code with a copy of the GNU GPL and this license
document.
4. Combined Works.
You may convey a Combined Work under terms of your choice that,
taken together, effectively do not restrict modification of the
portions of the Library contained in the Combined Work and reverse
engineering for debugging such modifications, if you also do each of
the following:
a) Give prominent notice with each copy of the Combined Work that
the Library is used in it and that the Library and its use are
covered by this License.
b) Accompany the Combined Work with a copy of the GNU GPL and this license
document.
c) For a Combined Work that displays copyright notices during
execution, include the copyright notice for the Library among
these notices, as well as a reference directing the user to the
copies of the GNU GPL and this license document.
d) Do one of the following:
0) Convey the Minimal Corresponding Source under the terms of this
License, and the Corresponding Application Code in a form
suitable for, and under terms that permit, the user to
recombine or relink the Application with a modified version of
the Linked Version to produce a modified Combined Work, in the
manner specified by section 6 of the GNU GPL for conveying
Corresponding Source.
1) Use a suitable shared library mechanism for linking with the
Library. A suitable mechanism is one that (a) uses at run time
a copy of the Library already present on the user's computer
system, and (b) will operate properly with a modified version
of the Library that is interface-compatible with the Linked
Version.
e) Provide Installation Information, but only if you would otherwise
be required to provide such information under section 6 of the
GNU GPL, and only to the extent that such information is
necessary to install and execute a modified version of the
Combined Work produced by recombining or relinking the
Application with a modified version of the Linked Version. (If
you use option 4d0, the Installation Information must accompany
the Minimal Corresponding Source and Corresponding Application
Code. If you use option 4d1, you must provide the Installation
Information in the manner specified by section 6 of the GNU GPL
for conveying Corresponding Source.)
5. Combined Libraries.
You may place library facilities that are a work based on the
Library side by side in a single library together with other library
facilities that are not Applications and are not covered by this
License, and convey such a combined library under terms of your
choice, if you do both of the following:
a) Accompany the combined library with a copy of the same work based
on the Library, uncombined with any other library facilities,
conveyed under the terms of this License.
b) Give prominent notice with the combined library that part of it
is a work based on the Library, and explaining where to find the
accompanying uncombined form of the same work.
6. Revised Versions of the GNU Lesser General Public License.
The Free Software Foundation may publish revised and/or new versions
of the GNU Lesser General Public License from time to time. Such new
versions will be similar in spirit to the present version, but may
differ in detail to address new problems or concerns.
Each version is given a distinguishing version number. If the
Library as you received it specifies that a certain numbered version
of the GNU Lesser General Public License "or any later version"
applies to it, you have the option of following the terms and
conditions either of that published version or of any later version
published by the Free Software Foundation. If the Library as you
received it does not specify a version number of the GNU Lesser
General Public License, you may choose any version of the GNU Lesser
General Public License ever published by the Free Software Foundation.
If the Library as you received it specifies that a proxy can decide
whether future versions of the GNU Lesser General Public License shall
apply, that proxy's public statement of acceptance of any version is
permanent authorization for you to choose that version for the
Library.

View File

@ -1,117 +0,0 @@
# ratelimit
--
import "github.com/juju/ratelimit"
The ratelimit package provides an efficient token bucket implementation. See
http://en.wikipedia.org/wiki/Token_bucket.
## Usage
#### func Reader
```go
func Reader(r io.Reader, bucket *Bucket) io.Reader
```
Reader returns a reader that is rate limited by the given token bucket. Each
token in the bucket represents one byte.
#### func Writer
```go
func Writer(w io.Writer, bucket *Bucket) io.Writer
```
Writer returns a writer that is rate limited by the given token bucket. Each
token in the bucket represents one byte.
#### type Bucket
```go
type Bucket struct {
}
```
Bucket represents a token bucket that fills at a predetermined rate. Methods on
Bucket may be called concurrently.
#### func NewBucket
```go
func NewBucket(fillInterval time.Duration, capacity int64) *Bucket
```
NewBucket returns a new token bucket that fills at the rate of one token every
fillInterval, up to the given maximum capacity. Both arguments must be positive.
The bucket is initially full.
#### func NewBucketWithQuantum
```go
func NewBucketWithQuantum(fillInterval time.Duration, capacity, quantum int64) *Bucket
```
NewBucketWithQuantum is similar to NewBucket, but allows the specification of
the quantum size - quantum tokens are added every fillInterval.
#### func NewBucketWithRate
```go
func NewBucketWithRate(rate float64, capacity int64) *Bucket
```
NewBucketWithRate returns a token bucket that fills the bucket at the rate of
rate tokens per second up to the given maximum capacity. Because of limited
clock resolution, at high rates, the actual rate may be up to 1% different from
the specified rate.
#### func (*Bucket) Rate
```go
func (tb *Bucket) Rate() float64
```
Rate returns the fill rate of the bucket, in tokens per second.
#### func (*Bucket) Take
```go
func (tb *Bucket) Take(count int64) time.Duration
```
Take takes count tokens from the bucket without blocking. It returns the time
that the caller should wait until the tokens are actually available.
Note that if the request is irrevocable - there is no way to return tokens to
the bucket once this method commits us to taking them.
#### func (*Bucket) TakeAvailable
```go
func (tb *Bucket) TakeAvailable(count int64) int64
```
TakeAvailable takes up to count immediately available tokens from the bucket. It
returns the number of tokens removed, or zero if there are no available tokens.
It does not block.
#### func (*Bucket) TakeMaxDuration
```go
func (tb *Bucket) TakeMaxDuration(count int64, maxWait time.Duration) (time.Duration, bool)
```
TakeMaxDuration is like Take, except that it will only take tokens from the
bucket if the wait time for the tokens is no greater than maxWait.
If it would take longer than maxWait for the tokens to become available, it does
nothing and reports false, otherwise it returns the time that the caller should
wait until the tokens are actually available, and reports true.
#### func (*Bucket) Wait
```go
func (tb *Bucket) Wait(count int64)
```
Wait takes count tokens from the bucket, waiting until they are available.
#### func (*Bucket) WaitMaxDuration
```go
func (tb *Bucket) WaitMaxDuration(count int64, maxWait time.Duration) bool
```
WaitMaxDuration is like Wait except that it will only take tokens from the
bucket if it needs to wait for no greater than maxWait. It reports whether any
tokens have been removed from the bucket If no tokens have been removed, it
returns immediately.

View File

@ -1,221 +0,0 @@
// Copyright 2014 Canonical Ltd.
// Licensed under the LGPLv3 with static-linking exception.
// See LICENCE file for details.
// The ratelimit package provides an efficient token bucket implementation
// that can be used to limit the rate of arbitrary things.
// See http://en.wikipedia.org/wiki/Token_bucket.
package ratelimit
import (
"strconv"
"sync"
"time"
"math"
)
// Bucket represents a token bucket that fills at a predetermined rate.
// Methods on Bucket may be called concurrently.
type Bucket struct {
startTime time.Time
capacity int64
quantum int64
fillInterval time.Duration
// The mutex guards the fields following it.
mu sync.Mutex
// avail holds the number of available tokens
// in the bucket, as of availTick ticks from startTime.
// It will be negative when there are consumers
// waiting for tokens.
avail int64
availTick int64
}
// NewBucket returns a new token bucket that fills at the
// rate of one token every fillInterval, up to the given
// maximum capacity. Both arguments must be
// positive. The bucket is initially full.
func NewBucket(fillInterval time.Duration, capacity int64) *Bucket {
return NewBucketWithQuantum(fillInterval, capacity, 1)
}
// rateMargin specifes the allowed variance of actual
// rate from specified rate. 1% seems reasonable.
const rateMargin = 0.01
// NewBucketWithRate returns a token bucket that fills the bucket
// at the rate of rate tokens per second up to the given
// maximum capacity. Because of limited clock resolution,
// at high rates, the actual rate may be up to 1% different from the
// specified rate.
func NewBucketWithRate(rate float64, capacity int64) *Bucket {
for quantum := int64(1); quantum < 1<<50; quantum = nextQuantum(quantum) {
fillInterval := time.Duration(1e9 * float64(quantum) / rate)
if fillInterval <= 0 {
continue
}
tb := NewBucketWithQuantum(fillInterval, capacity, quantum)
if diff := math.Abs(tb.Rate() - rate); diff/rate <= rateMargin {
return tb
}
}
panic("cannot find suitable quantum for " + strconv.FormatFloat(rate, 'g', -1, 64))
}
// nextQuantum returns the next quantum to try after q.
// We grow the quantum exponentially, but slowly, so we
// get a good fit in the lower numbers.
func nextQuantum(q int64) int64 {
q1 := q * 11 / 10
if q1 == q {
q1++
}
return q1
}
// NewBucketWithQuantum is similar to NewBucket, but allows
// the specification of the quantum size - quantum tokens
// are added every fillInterval.
func NewBucketWithQuantum(fillInterval time.Duration, capacity, quantum int64) *Bucket {
if fillInterval <= 0 {
panic("token bucket fill interval is not > 0")
}
if capacity <= 0 {
panic("token bucket capacity is not > 0")
}
if quantum <= 0 {
panic("token bucket quantum is not > 0")
}
return &Bucket{
startTime: time.Now(),
capacity: capacity,
quantum: quantum,
avail: capacity,
fillInterval: fillInterval,
}
}
// Wait takes count tokens from the bucket, waiting until they are
// available.
func (tb *Bucket) Wait(count int64) {
if d := tb.Take(count); d > 0 {
time.Sleep(d)
}
}
// WaitMaxDuration is like Wait except that it will
// only take tokens from the bucket if it needs to wait
// for no greater than maxWait. It reports whether
// any tokens have been removed from the bucket
// If no tokens have been removed, it returns immediately.
func (tb *Bucket) WaitMaxDuration(count int64, maxWait time.Duration) bool {
d, ok := tb.TakeMaxDuration(count, maxWait)
if d > 0 {
time.Sleep(d)
}
return ok
}
const infinityDuration time.Duration = 0x7fffffffffffffff
// Take takes count tokens from the bucket without blocking. It returns
// the time that the caller should wait until the tokens are actually
// available.
//
// Note that if the request is irrevocable - there is no way to return
// tokens to the bucket once this method commits us to taking them.
func (tb *Bucket) Take(count int64) time.Duration {
d, _ := tb.take(time.Now(), count, infinityDuration)
return d
}
// TakeMaxDuration is like Take, except that
// it will only take tokens from the bucket if the wait
// time for the tokens is no greater than maxWait.
//
// If it would take longer than maxWait for the tokens
// to become available, it does nothing and reports false,
// otherwise it returns the time that the caller should
// wait until the tokens are actually available, and reports
// true.
func (tb *Bucket) TakeMaxDuration(count int64, maxWait time.Duration) (time.Duration, bool) {
return tb.take(time.Now(), count, maxWait)
}
// TakeAvailable takes up to count immediately available tokens from the
// bucket. It returns the number of tokens removed, or zero if there are
// no available tokens. It does not block.
func (tb *Bucket) TakeAvailable(count int64) int64 {
return tb.takeAvailable(time.Now(), count)
}
// takeAvailable is the internal version of TakeAvailable - it takes the
// current time as an argument to enable easy testing.
func (tb *Bucket) takeAvailable(now time.Time, count int64) int64 {
if count <= 0 {
return 0
}
tb.mu.Lock()
defer tb.mu.Unlock()
tb.adjust(now)
if tb.avail <= 0 {
return 0
}
if count > tb.avail {
count = tb.avail
}
tb.avail -= count
return count
}
// Rate returns the fill rate of the bucket, in tokens per second.
func (tb *Bucket) Rate() float64 {
return 1e9 * float64(tb.quantum) / float64(tb.fillInterval)
}
// take is the internal version of Take - it takes the current time as
// an argument to enable easy testing.
func (tb *Bucket) take(now time.Time, count int64, maxWait time.Duration) (time.Duration, bool) {
if count <= 0 {
return 0, true
}
tb.mu.Lock()
defer tb.mu.Unlock()
currentTick := tb.adjust(now)
avail := tb.avail - count
if avail >= 0 {
tb.avail = avail
return 0, true
}
// Round up the missing tokens to the nearest multiple
// of quantum - the tokens won't be available until
// that tick.
endTick := currentTick + (-avail+tb.quantum-1)/tb.quantum
endTime := tb.startTime.Add(time.Duration(endTick) * tb.fillInterval)
waitTime := endTime.Sub(now)
if waitTime > maxWait {
return 0, false
}
tb.avail = avail
return waitTime, true
}
// adjust adjusts the current bucket capacity based on the current time.
// It returns the current tick.
func (tb *Bucket) adjust(now time.Time) (currentTick int64) {
currentTick = int64(now.Sub(tb.startTime) / tb.fillInterval)
if tb.avail >= tb.capacity {
return
}
tb.avail += (currentTick - tb.availTick) * tb.quantum
if tb.avail > tb.capacity {
tb.avail = tb.capacity
}
tb.availTick = currentTick
return
}

View File

@ -1,51 +0,0 @@
// Copyright 2014 Canonical Ltd.
// Licensed under the LGPLv3 with static-linking exception.
// See LICENCE file for details.
package ratelimit
import "io"
type reader struct {
r io.Reader
bucket *Bucket
}
// Reader returns a reader that is rate limited by
// the given token bucket. Each token in the bucket
// represents one byte.
func Reader(r io.Reader, bucket *Bucket) io.Reader {
return &reader{
r: r,
bucket: bucket,
}
}
func (r *reader) Read(buf []byte) (int, error) {
n, err := r.r.Read(buf)
if n <= 0 {
return n, err
}
r.bucket.Wait(int64(n))
return n, err
}
type writer struct {
w io.Writer
bucket *Bucket
}
// Writer returns a reader that is rate limited by
// the given token bucket. Each token in the bucket
// represents one byte.
func Writer(w io.Writer, bucket *Bucket) io.Writer {
return &writer{
w: w,
bucket: bucket,
}
}
func (w *writer) Write(buf []byte) (int, error) {
w.bucket.Wait(int64(len(buf)))
return w.w.Write(buf)
}

View File

@ -1,3 +0,0 @@
[submodule "test-data"]
path = test-data
url = git://github.com/maxmind/MaxMind-DB.git

View File

@ -1,20 +0,0 @@
language: go
go:
- 1.2
- 1.3
- 1.4
- 1.5
- 1.6
- tip
before_install:
- "if [[ $TRAVIS_GO_VERSION == 1.6 ]]; then go get -v github.com/golang/lint/golint; fi"
script:
- go test -race -cpu 1,4 -v
- go test -race -v -tags appengine
- "if [[ $TRAVIS_GO_VERSION == 1.6 ]]; then go vet ./...; fi"
- "if [[ $TRAVIS_GO_VERSION == 1.6 ]]; then golint .; fi"
sudo: false

View File

@ -1,13 +0,0 @@
Copyright (c) 2015, Gregory J. Oschwald <oschwald@gmail.com>
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
PERFORMANCE OF THIS SOFTWARE.

View File

@ -1,90 +0,0 @@
# GeoIP2 Reader for Go #
[![Build Status](https://travis-ci.org/oschwald/geoip2-golang.png?branch=master)](https://travis-ci.org/oschwald/geoip2-golang)
[![GoDoc](https://godoc.org/github.com/oschwald/geoip2-golang?status.png)](https://godoc.org/github.com/oschwald/geoip2-golang)
This library reads MaxMind [GeoLite2](http://dev.maxmind.com/geoip/geoip2/geolite2/)
and [GeoIP2](http://www.maxmind.com/en/geolocation_landing) databases.
This library is built using
[the Go maxminddb reader](https://github.com/oschwald/maxminddb-golang).
All data for the database record is decoded using this library. If you only
need several fields, you may get superior performance by using maxminddb's
`Lookup` directly with a result struct that only contains the required fields.
(See [example_test.go](https://github.com/oschwald/maxminddb-golang/blob/master/example_test.go)
in the maxminddb repository for an example of this.)
## Installation ##
```
go get github.com/oschwald/geoip2-golang
```
## Usage ##
[See GoDoc](http://godoc.org/github.com/oschwald/geoip2-golang) for
documentation and examples.
## Example ##
```go
package main
import (
"fmt"
"github.com/oschwald/geoip2-golang"
"log"
"net"
)
func main() {
db, err := geoip2.Open("GeoIP2-City.mmdb")
if err != nil {
log.Fatal(err)
}
defer db.Close()
// If you are using strings that may be invalid, check that ip is not nil
ip := net.ParseIP("81.2.69.142")
record, err := db.City(ip)
if err != nil {
log.Fatal(err)
}
fmt.Printf("Portuguese (BR) city name: %v\n", record.City.Names["pt-BR"])
fmt.Printf("English subdivision name: %v\n", record.Subdivisions[0].Names["en"])
fmt.Printf("Russian country name: %v\n", record.Country.Names["ru"])
fmt.Printf("ISO country code: %v\n", record.Country.IsoCode)
fmt.Printf("Time zone: %v\n", record.Location.TimeZone)
fmt.Printf("Coordinates: %v, %v\n", record.Location.Latitude, record.Location.Longitude)
// Output:
// Portuguese (BR) city name: Londres
// English subdivision name: England
// Russian country name: Великобритания
// ISO country code: GB
// Time zone: Europe/London
// Coordinates: 51.5142, -0.0931
}
```
## Testing ##
Make sure you checked out test data submodule:
```
git submodule init
git submodule update
```
Execute test suite:
```
go test
```
## Contributing ##
Contributions welcome! Please fork the repository and open a pull request
with your changes.
## License ##
This is free software, licensed under the ISC license.

View File

@ -1,202 +0,0 @@
// Package geoip2 provides a wrapper around the maxminddb package for
// easy use with the MaxMind GeoIP2 and GeoLite2 databases. The records for
// the IP address is returned from this package as well-formed structures
// that match the internal layout of data from MaxMind.
package geoip2
import (
"net"
"github.com/oschwald/maxminddb-golang"
)
// The City structure corresponds to the data in the GeoIP2/GeoLite2 City
// databases.
type City struct {
City struct {
GeoNameID uint `maxminddb:"geoname_id"`
Names map[string]string `maxminddb:"names"`
} `maxminddb:"city"`
Continent struct {
Code string `maxminddb:"code"`
GeoNameID uint `maxminddb:"geoname_id"`
Names map[string]string `maxminddb:"names"`
} `maxminddb:"continent"`
Country struct {
GeoNameID uint `maxminddb:"geoname_id"`
IsoCode string `maxminddb:"iso_code"`
Names map[string]string `maxminddb:"names"`
} `maxminddb:"country"`
Location struct {
Latitude float64 `maxminddb:"latitude"`
Longitude float64 `maxminddb:"longitude"`
MetroCode uint `maxminddb:"metro_code"`
TimeZone string `maxminddb:"time_zone"`
} `maxminddb:"location"`
Postal struct {
Code string `maxminddb:"code"`
} `maxminddb:"postal"`
RegisteredCountry struct {
GeoNameID uint `maxminddb:"geoname_id"`
IsoCode string `maxminddb:"iso_code"`
Names map[string]string `maxminddb:"names"`
} `maxminddb:"registered_country"`
RepresentedCountry struct {
GeoNameID uint `maxminddb:"geoname_id"`
IsoCode string `maxminddb:"iso_code"`
Names map[string]string `maxminddb:"names"`
Type string `maxminddb:"type"`
} `maxminddb:"represented_country"`
Subdivisions []struct {
GeoNameID uint `maxminddb:"geoname_id"`
IsoCode string `maxminddb:"iso_code"`
Names map[string]string `maxminddb:"names"`
} `maxminddb:"subdivisions"`
Traits struct {
IsAnonymousProxy bool `maxminddb:"is_anonymous_proxy"`
IsSatelliteProvider bool `maxminddb:"is_satellite_provider"`
} `maxminddb:"traits"`
}
// The Country structure corresponds to the data in the GeoIP2/GeoLite2
// Country databases.
type Country struct {
Continent struct {
Code string `maxminddb:"code"`
GeoNameID uint `maxminddb:"geoname_id"`
Names map[string]string `maxminddb:"names"`
} `maxminddb:"continent"`
Country struct {
GeoNameID uint `maxminddb:"geoname_id"`
IsoCode string `maxminddb:"iso_code"`
Names map[string]string `maxminddb:"names"`
} `maxminddb:"country"`
RegisteredCountry struct {
GeoNameID uint `maxminddb:"geoname_id"`
IsoCode string `maxminddb:"iso_code"`
Names map[string]string `maxminddb:"names"`
} `maxminddb:"registered_country"`
RepresentedCountry struct {
GeoNameID uint `maxminddb:"geoname_id"`
IsoCode string `maxminddb:"iso_code"`
Names map[string]string `maxminddb:"names"`
Type string `maxminddb:"type"`
} `maxminddb:"represented_country"`
Traits struct {
IsAnonymousProxy bool `maxminddb:"is_anonymous_proxy"`
IsSatelliteProvider bool `maxminddb:"is_satellite_provider"`
} `maxminddb:"traits"`
}
// The AnonymousIP structure corresponds to the data in the GeoIP2
// Anonymous IP database.
type AnonymousIP struct {
IsAnonymous bool `maxminddb:"is_anonymous"`
IsAnonymousVPN bool `maxminddb:"is_anonymous_vpn"`
IsHostingProvider bool `maxminddb:"is_hosting_provider"`
IsPublicProxy bool `maxminddb:"is_public_proxy"`
IsTorExitNode bool `maxminddb:"is_tor_exit_node"`
}
// The ConnectionType structure corresponds to the data in the GeoIP2
// Connection-Type database.
type ConnectionType struct {
ConnectionType string `maxminddb:"connection_type"`
}
// The Domain structure corresponds to the data in the GeoIP2 Domain database.
type Domain struct {
Domain string `maxminddb:"domain"`
}
// The ISP structure corresponds to the data in the GeoIP2 ISP database.
type ISP struct {
AutonomousSystemNumber uint `maxminddb:"autonomous_system_number"`
AutonomousSystemOrganization string `maxminddb:"autonomous_system_organization"`
ISP string `maxminddb:"isp"`
Organization string `maxminddb:"organization"`
}
// Reader holds the maxminddb.Reader structure. It should be created
// using the Open function.
type Reader struct {
mmdbReader *maxminddb.Reader
}
// Open takes a string path to a file and returns a Reader structure or an
// error. The database file is opened using a memory map. Use the Close method
// on the Reader object to return the resources to the system.
func Open(file string) (*Reader, error) {
reader, err := maxminddb.Open(file)
return &Reader{mmdbReader: reader}, err
}
// FromBytes takes a byte slice corresponding to a GeoIP2/GeoLite2 database
// file and returns a Reader structure or an error.
func FromBytes(bytes []byte) (*Reader, error) {
reader, err := maxminddb.FromBytes(bytes)
return &Reader{mmdbReader: reader}, err
}
// City takes an IP address as a net.IP struct and returns a City struct
// and/or an error. Although this can be used with other databases, this
// method generally should be used with the GeoIP2 or GeoLite2 City databases.
func (r *Reader) City(ipAddress net.IP) (*City, error) {
var city City
err := r.mmdbReader.Lookup(ipAddress, &city)
return &city, err
}
// Country takes an IP address as a net.IP struct and returns a Country struct
// and/or an error. Although this can be used with other databases, this
// method generally should be used with the GeoIP2 or GeoLite2 Country
// databases.
func (r *Reader) Country(ipAddress net.IP) (*Country, error) {
var country Country
err := r.mmdbReader.Lookup(ipAddress, &country)
return &country, err
}
// AnonymousIP takes an IP address as a net.IP struct and returns a
// AnonymousIP struct and/or an error.
func (r *Reader) AnonymousIP(ipAddress net.IP) (*AnonymousIP, error) {
var anonIP AnonymousIP
err := r.mmdbReader.Lookup(ipAddress, &anonIP)
return &anonIP, err
}
// ConnectionType takes an IP address as a net.IP struct and returns a
// ConnectionType struct and/or an error
func (r *Reader) ConnectionType(ipAddress net.IP) (*ConnectionType, error) {
var val ConnectionType
err := r.mmdbReader.Lookup(ipAddress, &val)
return &val, err
}
// Domain takes an IP address as a net.IP struct and returns a
// Domain struct and/or an error
func (r *Reader) Domain(ipAddress net.IP) (*Domain, error) {
var val Domain
err := r.mmdbReader.Lookup(ipAddress, &val)
return &val, err
}
// ISP takes an IP address as a net.IP struct and returns a ISP struct and/or
// an error
func (r *Reader) ISP(ipAddress net.IP) (*ISP, error) {
var val ISP
err := r.mmdbReader.Lookup(ipAddress, &val)
return &val, err
}
// Metadata takes no arguments and returns a struct containing metadata about
// the MaxMind database in use by the Reader.
func (r *Reader) Metadata() maxminddb.Metadata {
return r.mmdbReader.Metadata
}
// Close unmaps the database file from virtual memory and returns the
// resources to the system.
func (r *Reader) Close() error {
return r.mmdbReader.Close()
}

View File

@ -1,3 +0,0 @@
[submodule "test-data"]
path = test-data
url = git://github.com/maxmind/MaxMind-DB.git

View File

@ -1,23 +0,0 @@
language: go
go:
- 1.2
- 1.3
- 1.4
- 1.5
- 1.6
- tip
before_install:
- "if [[ $TRAVIS_GO_VERSION == 1.6 ]]; then go get -v github.com/golang/lint/golint; fi"
install:
- go get gopkg.in/check.v1
script:
- go test -race -cpu 1,4 -v
- go test -race -v -tags appengine
- "if [[ $TRAVIS_GO_VERSION == 1.6 ]]; then go vet ./...; fi"
- "if [[ $TRAVIS_GO_VERSION == 1.6 ]]; then golint .; fi"
sudo: false

View File

@ -1,13 +0,0 @@
Copyright (c) 2015, Gregory J. Oschwald <oschwald@gmail.com>
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
PERFORMANCE OF THIS SOFTWARE.

View File

@ -1,37 +0,0 @@
# MaxMind DB Reader for Go #
[![Build Status](https://travis-ci.org/oschwald/maxminddb-golang.png?branch=master)](https://travis-ci.org/oschwald/maxminddb-golang)
[![Windows Build Status](https://ci.appveyor.com/api/projects/status/4j2f9oep8nnfrmov/branch/master?svg=true)](https://ci.appveyor.com/project/oschwald/maxminddb-golang/branch/master)
[![GoDoc](https://godoc.org/github.com/oschwald/maxminddb-golang?status.png)](https://godoc.org/github.com/oschwald/maxminddb-golang)
This is a Go reader for the MaxMind DB format. This can be used to read
[GeoLite2](http://dev.maxmind.com/geoip/geoip2/geolite2/) and
[GeoIP2](http://www.maxmind.com/en/geolocation_landing) databases.
This is not an official MaxMind API.
## Installation ##
```
go get github.com/oschwald/maxminddb-golang
```
## Usage ##
[See GoDoc](http://godoc.org/github.com/oschwald/maxminddb-golang) for
documentation and examples.
## Examples ##
See [GoDoc](http://godoc.org/github.com/oschwald/maxminddb-golang) or
`example_test.go` for examples.
## Contributing ##
Contributions welcome! Please fork the repository and open a pull request
with your changes.
## License ##
This is free software, licensed under the ISC License.

View File

@ -1,19 +0,0 @@
version: "{build}"
os: Windows Server 2012 R2
clone_folder: c:\gopath\src\github.com\oschwald\maxminddb-golang
environment:
GOPATH: c:\gopath
install:
- echo %PATH%
- echo %GOPATH%
- git submodule update --init --recursive
- go version
- go env
- go get -v -t ./...
build_script:
- go test -v ./...

View File

@ -1,530 +0,0 @@
package maxminddb
import (
"encoding/binary"
"fmt"
"math"
"math/big"
"reflect"
"sync"
)
type decoder struct {
buffer []byte
}
type dataType int
const (
_Extended dataType = iota
_Pointer
_String
_Float64
_Bytes
_Uint16
_Uint32
_Map
_Int32
_Uint64
_Uint128
_Slice
_Container
_Marker
_Bool
_Float32
)
func (d *decoder) decode(offset uint, result reflect.Value) (uint, error) {
typeNum, size, newOffset := d.decodeCtrlData(offset)
return d.decodeFromType(typeNum, size, newOffset, result)
}
func (d *decoder) decodeCtrlData(offset uint) (dataType, uint, uint) {
newOffset := offset + 1
ctrlByte := d.buffer[offset]
typeNum := dataType(ctrlByte >> 5)
if typeNum == _Extended {
typeNum = dataType(d.buffer[newOffset] + 7)
newOffset++
}
var size uint
size, newOffset = d.sizeFromCtrlByte(ctrlByte, newOffset, typeNum)
return typeNum, size, newOffset
}
func (d *decoder) sizeFromCtrlByte(ctrlByte byte, offset uint, typeNum dataType) (uint, uint) {
size := uint(ctrlByte & 0x1f)
if typeNum == _Extended {
return size, offset
}
var bytesToRead uint
if size > 28 {
bytesToRead = size - 28
}
newOffset := offset + bytesToRead
sizeBytes := d.buffer[offset:newOffset]
switch {
case size == 29:
size = 29 + uint(sizeBytes[0])
case size == 30:
size = 285 + uint(uintFromBytes(0, sizeBytes))
case size > 30:
size = uint(uintFromBytes(0, sizeBytes)) + 65821
}
return size, newOffset
}
func (d *decoder) decodeFromType(dtype dataType, size uint, offset uint, result reflect.Value) (uint, error) {
if result.Kind() == reflect.Ptr {
result = reflect.Indirect(result)
}
switch dtype {
case _Bool:
return d.unmarshalBool(size, offset, result)
case _Bytes:
return d.unmarshalBytes(size, offset, result)
case _Float32:
return d.unmarshalFloat32(size, offset, result)
case _Float64:
return d.unmarshalFloat64(size, offset, result)
case _Int32:
return d.unmarshalInt32(size, offset, result)
case _Map:
return d.unmarshalMap(size, offset, result)
case _Pointer:
return d.unmarshalPointer(size, offset, result)
case _Slice:
return d.unmarshalSlice(size, offset, result)
case _String:
return d.unmarshalString(size, offset, result)
case _Uint16:
return d.unmarshalUint(size, offset, result, 16)
case _Uint32:
return d.unmarshalUint(size, offset, result, 32)
case _Uint64:
return d.unmarshalUint(size, offset, result, 64)
case _Uint128:
return d.unmarshalUint128(size, offset, result)
default:
return 0, newInvalidDatabaseError("unknown type: %d", dtype)
}
}
func (d *decoder) unmarshalBool(size uint, offset uint, result reflect.Value) (uint, error) {
if size > 1 {
return 0, newInvalidDatabaseError("the MaxMind DB file's data section contains bad data (bool size of %v)", size)
}
value, newOffset, err := d.decodeBool(size, offset)
if err != nil {
return 0, err
}
switch result.Kind() {
default:
return newOffset, fmt.Errorf("trying to unmarshal %v into %v", value, result.Type())
case reflect.Bool:
result.SetBool(value)
return newOffset, nil
case reflect.Interface:
result.Set(reflect.ValueOf(value))
return newOffset, nil
}
}
func (d *decoder) unmarshalBytes(size uint, offset uint, result reflect.Value) (uint, error) {
value, newOffset, err := d.decodeBytes(size, offset)
if err != nil {
return 0, err
}
switch result.Kind() {
default:
return newOffset, fmt.Errorf("trying to unmarshal %v into %v", value, result.Type())
case reflect.Slice:
result.SetBytes(value)
return newOffset, nil
case reflect.Interface:
result.Set(reflect.ValueOf(value))
return newOffset, nil
}
}
func (d *decoder) unmarshalFloat32(size uint, offset uint, result reflect.Value) (uint, error) {
if size != 4 {
return 0, newInvalidDatabaseError("the MaxMind DB file's data section contains bad data (float32 size of %v)", size)
}
value, newOffset, err := d.decodeFloat32(size, offset)
if err != nil {
return 0, err
}
switch result.Kind() {
default:
return newOffset, fmt.Errorf("trying to unmarshal %v into %v", value, result.Type())
case reflect.Float32, reflect.Float64:
result.SetFloat(float64(value))
return newOffset, nil
case reflect.Interface:
result.Set(reflect.ValueOf(value))
return newOffset, nil
}
}
func (d *decoder) unmarshalFloat64(size uint, offset uint, result reflect.Value) (uint, error) {
if size != 8 {
return 0, newInvalidDatabaseError("the MaxMind DB file's data section contains bad data (float 64 size of %v)", size)
}
value, newOffset, err := d.decodeFloat64(size, offset)
if err != nil {
return 0, err
}
switch result.Kind() {
default:
return newOffset, fmt.Errorf("trying to unmarshal %v into %v", value, result.Type())
case reflect.Float32, reflect.Float64:
result.SetFloat(value)
return newOffset, nil
case reflect.Interface:
result.Set(reflect.ValueOf(value))
return newOffset, nil
}
}
func (d *decoder) unmarshalInt32(size uint, offset uint, result reflect.Value) (uint, error) {
if size > 4 {
return 0, newInvalidDatabaseError("the MaxMind DB file's data section contains bad data (int32 size of %v)", size)
}
value, newOffset, err := d.decodeInt(size, offset)
if err != nil {
return 0, err
}
switch result.Kind() {
default:
return newOffset, fmt.Errorf("trying to unmarshal %v into %v", value, result.Type())
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
result.SetInt(int64(value))
return newOffset, nil
case reflect.Interface:
result.Set(reflect.ValueOf(value))
return newOffset, nil
}
}
func (d *decoder) unmarshalMap(size uint, offset uint, result reflect.Value) (uint, error) {
switch result.Kind() {
default:
return 0, fmt.Errorf("trying to unmarshal a map into %v", result.Type())
case reflect.Struct:
return d.decodeStruct(size, offset, result)
case reflect.Map:
return d.decodeMap(size, offset, result)
case reflect.Interface:
rv := reflect.ValueOf(make(map[string]interface{}, size))
newOffset, err := d.decodeMap(size, offset, rv)
result.Set(rv)
return newOffset, err
}
}
func (d *decoder) unmarshalPointer(size uint, offset uint, result reflect.Value) (uint, error) {
pointer, newOffset := d.decodePointer(size, offset)
_, err := d.decode(pointer, result)
return newOffset, err
}
func (d *decoder) unmarshalSlice(size uint, offset uint, result reflect.Value) (uint, error) {
switch result.Kind() {
default:
return 0, fmt.Errorf("trying to unmarshal an array into %v", result.Type())
case reflect.Slice:
return d.decodeSlice(size, offset, result)
case reflect.Interface:
a := []interface{}{}
rv := reflect.ValueOf(&a).Elem()
newOffset, err := d.decodeSlice(size, offset, rv)
result.Set(rv)
return newOffset, err
}
}
func (d *decoder) unmarshalString(size uint, offset uint, result reflect.Value) (uint, error) {
value, newOffset, err := d.decodeString(size, offset)
if err != nil {
return 0, err
}
switch result.Kind() {
default:
return newOffset, fmt.Errorf("trying to unmarshal %v into %v", value, result.Type())
case reflect.String:
result.SetString(value)
return newOffset, nil
case reflect.Interface:
result.Set(reflect.ValueOf(value))
return newOffset, nil
}
}
func (d *decoder) unmarshalUint(size uint, offset uint, result reflect.Value, uintType uint) (uint, error) {
if size > uintType/8 {
return 0, newInvalidDatabaseError("the MaxMind DB file's data section contains bad data (uint%v size of %v)", uintType, size)
}
value, newOffset, err := d.decodeUint(size, offset)
if err != nil {
return 0, err
}
switch result.Kind() {
default:
return newOffset, fmt.Errorf("trying to unmarshal %v into %v", value, result.Type())
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
result.SetUint(value)
return newOffset, nil
case reflect.Interface:
result.Set(reflect.ValueOf(value))
return newOffset, nil
}
}
func (d *decoder) unmarshalUint128(size uint, offset uint, result reflect.Value) (uint, error) {
if size > 16 {
return 0, newInvalidDatabaseError("the MaxMind DB file's data section contains bad data (uint128 size of %v)", size)
}
value, newOffset, err := d.decodeUint128(size, offset)
if err != nil {
return 0, err
}
// XXX - this should allow *big.Int rather than just bigInt
// Currently this is reported as invalid
switch result.Kind() {
default:
return newOffset, fmt.Errorf("trying to unmarshal %v into %v", value, result.Type())
case reflect.Struct:
result.Set(reflect.ValueOf(*value))
return newOffset, nil
case reflect.Interface, reflect.Ptr:
result.Set(reflect.ValueOf(value))
return newOffset, nil
}
}
func (d *decoder) decodeBool(size uint, offset uint) (bool, uint, error) {
return size != 0, offset, nil
}
func (d *decoder) decodeBytes(size uint, offset uint) ([]byte, uint, error) {
newOffset := offset + size
bytes := make([]byte, size)
copy(bytes, d.buffer[offset:newOffset])
return bytes, newOffset, nil
}
func (d *decoder) decodeFloat64(size uint, offset uint) (float64, uint, error) {
newOffset := offset + size
bits := binary.BigEndian.Uint64(d.buffer[offset:newOffset])
return math.Float64frombits(bits), newOffset, nil
}
func (d *decoder) decodeFloat32(size uint, offset uint) (float32, uint, error) {
newOffset := offset + size
bits := binary.BigEndian.Uint32(d.buffer[offset:newOffset])
return math.Float32frombits(bits), newOffset, nil
}
func (d *decoder) decodeInt(size uint, offset uint) (int, uint, error) {
newOffset := offset + size
var val int32
for _, b := range d.buffer[offset:newOffset] {
val = (val << 8) | int32(b)
}
return int(val), newOffset, nil
}
func (d *decoder) decodeMap(size uint, offset uint, result reflect.Value) (uint, error) {
if result.IsNil() {
result.Set(reflect.MakeMap(result.Type()))
}
for i := uint(0); i < size; i++ {
var key string
var err error
key, offset, err = d.decodeKeyString(offset)
if err != nil {
return 0, err
}
value := reflect.New(result.Type().Elem())
offset, err = d.decode(offset, value)
if err != nil {
return 0, err
}
result.SetMapIndex(reflect.ValueOf(key), value.Elem())
}
return offset, nil
}
func (d *decoder) decodePointer(size uint, offset uint) (uint, uint) {
pointerSize := ((size >> 3) & 0x3) + 1
newOffset := offset + pointerSize
pointerBytes := d.buffer[offset:newOffset]
var prefix uint64
if pointerSize == 4 {
prefix = 0
} else {
prefix = uint64(size & 0x7)
}
unpacked := uint(uintFromBytes(prefix, pointerBytes))
var pointerValueOffset uint
switch pointerSize {
case 1:
pointerValueOffset = 0
case 2:
pointerValueOffset = 2048
case 3:
pointerValueOffset = 526336
case 4:
pointerValueOffset = 0
}
pointer := unpacked + pointerValueOffset
return pointer, newOffset
}
func (d *decoder) decodeSlice(size uint, offset uint, result reflect.Value) (uint, error) {
result.Set(reflect.MakeSlice(result.Type(), int(size), int(size)))
for i := 0; i < int(size); i++ {
var err error
offset, err = d.decode(offset, result.Index(i))
if err != nil {
return 0, err
}
}
return offset, nil
}
func (d *decoder) decodeString(size uint, offset uint) (string, uint, error) {
newOffset := offset + size
return string(d.buffer[offset:newOffset]), newOffset, nil
}
var (
fieldMap = map[reflect.Type]map[string]int{}
fieldMapMu sync.RWMutex
)
func (d *decoder) decodeStruct(size uint, offset uint, result reflect.Value) (uint, error) {
resultType := result.Type()
fieldMapMu.RLock()
fields, ok := fieldMap[resultType]
fieldMapMu.RUnlock()
if !ok {
numFields := resultType.NumField()
fields = make(map[string]int, numFields)
for i := 0; i < numFields; i++ {
fieldType := resultType.Field(i)
fieldName := fieldType.Name
if tag := fieldType.Tag.Get("maxminddb"); tag != "" {
fieldName = tag
}
fields[fieldName] = i
}
fieldMapMu.Lock()
fieldMap[resultType] = fields
fieldMapMu.Unlock()
}
for i := uint(0); i < size; i++ {
var (
err error
key string
)
key, offset, err = d.decodeStructKey(offset)
if err != nil {
return 0, err
}
i, ok := fields[key]
if !ok {
offset = d.nextValueOffset(offset, 1)
continue
}
offset, err = d.decode(offset, result.Field(i))
if err != nil {
return 0, err
}
}
return offset, nil
}
func (d *decoder) decodeUint(size uint, offset uint) (uint64, uint, error) {
newOffset := offset + size
val := uintFromBytes(0, d.buffer[offset:newOffset])
return val, newOffset, nil
}
func (d *decoder) decodeUint128(size uint, offset uint) (*big.Int, uint, error) {
newOffset := offset + size
val := new(big.Int)
val.SetBytes(d.buffer[offset:newOffset])
return val, newOffset, nil
}
func uintFromBytes(prefix uint64, uintBytes []byte) uint64 {
val := prefix
for _, b := range uintBytes {
val = (val << 8) | uint64(b)
}
return val
}
func (d *decoder) decodeKeyString(offset uint) (string, uint, error) {
typeNum, size, newOffset := d.decodeCtrlData(offset)
if typeNum == _Pointer {
pointer, ptrOffset := d.decodePointer(size, newOffset)
key, _, err := d.decodeKeyString(pointer)
return key, ptrOffset, err
}
if typeNum != _String {
return "", 0, newInvalidDatabaseError("unexpected type when decoding string: %v", typeNum)
}
return d.decodeString(size, newOffset)
}
// This function is used to skip ahead to the next value without decoding
// the one at the offset passed in. The size bits have different meanings for
// different data types
func (d *decoder) nextValueOffset(offset uint, numberToSkip uint) uint {
if numberToSkip == 0 {
return offset
}
typeNum, size, offset := d.decodeCtrlData(offset)
switch typeNum {
case _Pointer:
_, offset = d.decodePointer(size, offset)
case _Map:
numberToSkip += 2 * size
case _Slice:
numberToSkip += size
case _Bool:
default:
offset += size
}
return d.nextValueOffset(offset, numberToSkip-1)
}

View File

@ -1,17 +0,0 @@
package maxminddb
import "fmt"
// InvalidDatabaseError is returned when the database contains invalid data
// and cannot be parsed.
type InvalidDatabaseError struct {
message string
}
func newInvalidDatabaseError(format string, args ...interface{}) InvalidDatabaseError {
return InvalidDatabaseError{fmt.Sprintf(format, args...)}
}
func (e InvalidDatabaseError) Error() string {
return e.message
}

View File

@ -1,7 +0,0 @@
// +build appengine
package maxminddb
func (d *decoder) decodeStructKey(offset uint) (string, uint, error) {
return d.decodeKeyString(offset)
}

View File

@ -1,28 +0,0 @@
// +build !appengine
package maxminddb
import (
"reflect"
"unsafe"
)
// decodeStructKey returns a string which points into the database. Don't keep
// it around.
func (d *decoder) decodeStructKey(offset uint) (string, uint, error) {
typeNum, size, newOffset := d.decodeCtrlData(offset)
switch typeNum {
case _Pointer:
pointer, ptrOffset := d.decodePointer(size, newOffset)
s, _, err := d.decodeStructKey(pointer)
return s, ptrOffset, err
case _String:
var s string
val := (*reflect.StringHeader)(unsafe.Pointer(&s))
val.Data = uintptr(unsafe.Pointer(&d.buffer[newOffset]))
val.Len = int(size)
return s, newOffset + size, nil
default:
return "", 0, newInvalidDatabaseError("unexpected type when decoding struct key: %v", typeNum)
}
}

View File

@ -1,17 +0,0 @@
// +build !windows,!appengine
package maxminddb
import (
"syscall"
"golang.org/x/sys/unix"
)
func mmap(fd int, length int) (data []byte, err error) {
return unix.Mmap(fd, 0, length, syscall.PROT_READ, syscall.MAP_SHARED)
}
func munmap(b []byte) (err error) {
return unix.Munmap(b)
}

View File

@ -1,83 +0,0 @@
package maxminddb
// Windows support largely borrowed from mmap-go.
//
// Copyright 2011 Evan Shaw. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
import (
"errors"
"os"
"reflect"
"sync"
"unsafe"
"golang.org/x/sys/windows"
)
type memoryMap []byte
// Windows
var handleLock sync.Mutex
var handleMap = map[uintptr]windows.Handle{}
func mmap(fd int, length int) (data []byte, err error) {
h, errno := windows.CreateFileMapping(windows.Handle(fd), nil,
uint32(windows.PAGE_READONLY), 0, uint32(length), nil)
if h == 0 {
return nil, os.NewSyscallError("CreateFileMapping", errno)
}
addr, errno := windows.MapViewOfFile(h, uint32(windows.FILE_MAP_READ), 0,
0, uintptr(length))
if addr == 0 {
return nil, os.NewSyscallError("MapViewOfFile", errno)
}
handleLock.Lock()
handleMap[addr] = h
handleLock.Unlock()
m := memoryMap{}
dh := m.header()
dh.Data = addr
dh.Len = length
dh.Cap = dh.Len
return m, nil
}
func (m *memoryMap) header() *reflect.SliceHeader {
return (*reflect.SliceHeader)(unsafe.Pointer(m))
}
func flush(addr, len uintptr) error {
errno := windows.FlushViewOfFile(addr, len)
return os.NewSyscallError("FlushViewOfFile", errno)
}
func munmap(b []byte) (err error) {
m := memoryMap(b)
dh := m.header()
addr := dh.Data
length := uintptr(dh.Len)
flush(addr, length)
err = windows.UnmapViewOfFile(addr)
if err != nil {
return err
}
handleLock.Lock()
defer handleLock.Unlock()
handle, ok := handleMap[addr]
if !ok {
// should be impossible; we would've errored above
return errors.New("unknown base address")
}
delete(handleMap, addr)
e := windows.CloseHandle(windows.Handle(handle))
return os.NewSyscallError("CloseHandle", e)
}

View File

@ -1,221 +0,0 @@
package maxminddb
import (
"bytes"
"errors"
"fmt"
"net"
"reflect"
)
const dataSectionSeparatorSize = 16
var metadataStartMarker = []byte("\xAB\xCD\xEFMaxMind.com")
// Reader holds the data corresponding to the MaxMind DB file. Its only public
// field is Metadata, which contains the metadata from the MaxMind DB file.
type Reader struct {
hasMappedFile bool
buffer []byte
decoder decoder
Metadata Metadata
ipv4Start uint
}
// Metadata holds the metadata decoded from the MaxMind DB file. In particular
// in has the format version, the build time as Unix epoch time, the database
// type and description, the IP version supported, and a slice of the natural
// languages included.
type Metadata struct {
BinaryFormatMajorVersion uint `maxminddb:"binary_format_major_version"`
BinaryFormatMinorVersion uint `maxminddb:"binary_format_minor_version"`
BuildEpoch uint `maxminddb:"build_epoch"`
DatabaseType string `maxminddb:"database_type"`
Description map[string]string `maxminddb:"description"`
IPVersion uint `maxminddb:"ip_version"`
Languages []string `maxminddb:"languages"`
NodeCount uint `maxminddb:"node_count"`
RecordSize uint `maxminddb:"record_size"`
}
// FromBytes takes a byte slice corresponding to a MaxMind DB file and returns
// a Reader structure or an error.
func FromBytes(buffer []byte) (*Reader, error) {
metadataStart := bytes.LastIndex(buffer, metadataStartMarker)
if metadataStart == -1 {
return nil, newInvalidDatabaseError("error opening database: invalid MaxMind DB file")
}
metadataStart += len(metadataStartMarker)
metadataDecoder := decoder{buffer[metadataStart:]}
var metadata Metadata
rvMetdata := reflect.ValueOf(&metadata)
_, err := metadataDecoder.decode(0, rvMetdata)
if err != nil {
return nil, err
}
searchTreeSize := metadata.NodeCount * metadata.RecordSize / 4
dataSectionStart := searchTreeSize + dataSectionSeparatorSize
dataSectionEnd := uint(metadataStart - len(metadataStartMarker))
if dataSectionStart > dataSectionEnd {
return nil, newInvalidDatabaseError("the MaxMind DB contains invalid metadata")
}
d := decoder{
buffer[searchTreeSize+dataSectionSeparatorSize : metadataStart-len(metadataStartMarker)],
}
reader := &Reader{
buffer: buffer,
decoder: d,
Metadata: metadata,
ipv4Start: 0,
}
reader.ipv4Start, err = reader.startNode()
return reader, err
}
func (r *Reader) startNode() (uint, error) {
if r.Metadata.IPVersion != 6 {
return 0, nil
}
nodeCount := r.Metadata.NodeCount
node := uint(0)
var err error
for i := 0; i < 96 && node < nodeCount; i++ {
node, err = r.readNode(node, 0)
if err != nil {
return 0, err
}
}
return node, err
}
// Lookup takes an IP address as a net.IP structure and a pointer to the
// result value to decode into. The result value pointed to must be a data
// value that corresponds to a record in the database. This may include a
// struct representation of the data, a map capable of holding the data or an
// empty interface{} value.
//
// If result is a pointer to a struct, the struct need not include a field
// for every value that may be in the database. If a field is not present in
// the structure, the decoder will not decode that field, reducing the time
// required to decode the record.
//
// Currently the decoder expect most data types to correspond exactly (e.g.,
// a uint64 database type must be decoded into a uint64 Go type). In the
// future, this may be made more flexible.
func (r *Reader) Lookup(ipAddress net.IP, result interface{}) error {
if ipAddress == nil {
return errors.New("ipAddress passed to Lookup cannot be nil")
}
ipV4Address := ipAddress.To4()
if ipV4Address != nil {
ipAddress = ipV4Address
}
if len(ipAddress) == 16 && r.Metadata.IPVersion == 4 {
return fmt.Errorf("error looking up '%s': you attempted to look up an IPv6 address in an IPv4-only database", ipAddress.String())
}
pointer, err := r.findAddressInTree(ipAddress)
if pointer == 0 {
return err
}
return r.retrieveData(pointer, result)
}
func (r *Reader) findAddressInTree(ipAddress net.IP) (uint, error) {
bitCount := uint(len(ipAddress) * 8)
var node uint
if bitCount == 32 {
node = r.ipv4Start
}
nodeCount := r.Metadata.NodeCount
for i := uint(0); i < bitCount && node < nodeCount; i++ {
bit := uint(1) & (uint(ipAddress[i>>3]) >> (7 - (i % 8)))
var err error
node, err = r.readNode(node, bit)
if err != nil {
return 0, err
}
}
if node == nodeCount {
// Record is empty
return 0, nil
} else if node > nodeCount {
return node, nil
}
return 0, newInvalidDatabaseError("invalid node in search tree")
}
func (r *Reader) readNode(nodeNumber uint, index uint) (uint, error) {
RecordSize := r.Metadata.RecordSize
baseOffset := nodeNumber * RecordSize / 4
var nodeBytes []byte
var prefix uint64
switch RecordSize {
case 24:
offset := baseOffset + index*3
nodeBytes = r.buffer[offset : offset+3]
case 28:
prefix = uint64(r.buffer[baseOffset+3])
if index != 0 {
prefix &= 0x0F
} else {
prefix = (0xF0 & prefix) >> 4
}
offset := baseOffset + index*4
nodeBytes = r.buffer[offset : offset+3]
case 32:
offset := baseOffset + index*4
nodeBytes = r.buffer[offset : offset+4]
default:
return 0, newInvalidDatabaseError("unknown record size: %d", RecordSize)
}
return uint(uintFromBytes(prefix, nodeBytes)), nil
}
func (r *Reader) retrieveData(pointer uint, result interface{}) error {
rv := reflect.ValueOf(result)
if rv.Kind() != reflect.Ptr || rv.IsNil() {
return errors.New("result param must be a pointer")
}
offset, err := r.resolveDataPointer(pointer)
if err != nil {
return err
}
_, err = r.decoder.decode(offset, rv)
return err
}
func (r *Reader) resolveDataPointer(pointer uint) (uint, error) {
nodeCount := r.Metadata.NodeCount
resolved := pointer - nodeCount - dataSectionSeparatorSize
if resolved > uint(len(r.buffer)) {
return 0, newInvalidDatabaseError("the MaxMind DB file's search tree is corrupt")
}
return resolved, nil
}

View File

@ -1,26 +0,0 @@
// +build appengine
package maxminddb
import "io/ioutil"
// Open takes a string path to a MaxMind DB file and returns a Reader
// structure or an error. The database file is opened using a memory map,
// except on Google App Engine where mmap is not supported; there the database
// is loaded into memory. Use the Close method on the Reader object to return
// the resources to the system.
func Open(file string) (*Reader, error) {
bytes, err := ioutil.ReadFile(file)
if err != nil {
return nil, err
}
return FromBytes(bytes)
}
// Close unmaps the database file from virtual memory and returns the
// resources to the system. If called on a Reader opened using FromBytes
// or Open on Google App Engine, this method does nothing.
func (r *Reader) Close() error {
return nil
}

View File

@ -1,56 +0,0 @@
// +build !appengine
package maxminddb
import "os"
// Open takes a string path to a MaxMind DB file and returns a Reader
// structure or an error. The database file is opened using a memory map,
// except on Google App Engine where mmap is not supported; there the database
// is loaded into memory. Use the Close method on the Reader object to return
// the resources to the system.
func Open(file string) (*Reader, error) {
mapFile, err := os.Open(file)
if err != nil {
return nil, err
}
defer func() {
if rerr := mapFile.Close(); rerr != nil {
err = rerr
}
}()
stats, err := mapFile.Stat()
if err != nil {
return nil, err
}
fileSize := int(stats.Size())
mmap, err := mmap(int(mapFile.Fd()), fileSize)
if err != nil {
return nil, err
}
reader, err := FromBytes(mmap)
if err != nil {
if err2 := munmap(mmap); err2 != nil {
// failing to unmap the file is probably the more severe error
return nil, err2
}
return nil, err
}
reader.hasMappedFile = true
return reader, err
}
// Close unmaps the database file from virtual memory and returns the
// resources to the system. If called on a Reader opened using FromBytes
// or Open on Google App Engine, this method does nothing.
func (r *Reader) Close() (err error) {
if r.hasMappedFile {
err = munmap(r.buffer)
r.hasMappedFile = false
}
return err
}

View File

@ -1,108 +0,0 @@
package maxminddb
import "net"
// Internal structure used to keep track of nodes we still need to visit.
type netNode struct {
ip net.IP
bit uint
pointer uint
}
// Networks represents a set of subnets that we are iterating over.
type Networks struct {
reader *Reader
nodes []netNode // Nodes we still have to visit.
lastNode netNode
err error
}
// Networks returns an iterator that can be used to traverse all networks in
// the database.
//
// Please note that a MaxMind DB may map IPv4 networks into several locations
// in in an IPv6 database. This iterator will iterate over all of these
// locations separately.
func (r *Reader) Networks() *Networks {
s := 4
if r.Metadata.IPVersion == 6 {
s = 16
}
return &Networks{
reader: r,
nodes: []netNode{
netNode{
ip: make(net.IP, s),
},
},
}
}
// Next prepares the next network for reading with the Network method. It
// returns true if there is another network to be processed and false if there
// are no more networks or if there is an error.
func (n *Networks) Next() bool {
for len(n.nodes) > 0 {
node := n.nodes[len(n.nodes)-1]
n.nodes = n.nodes[:len(n.nodes)-1]
for {
if node.pointer < n.reader.Metadata.NodeCount {
ipRight := make(net.IP, len(node.ip))
copy(ipRight, node.ip)
if len(ipRight) <= int(node.bit>>3) {
n.err = newInvalidDatabaseError(
"invalid search tree at %v/%v", ipRight, node.bit)
return false
}
ipRight[node.bit>>3] |= 1 << uint(7-(node.bit%8))
rightPointer, err := n.reader.readNode(node.pointer, 1)
if err != nil {
n.err = err
return false
}
node.bit++
n.nodes = append(n.nodes, netNode{
pointer: rightPointer,
ip: ipRight,
bit: node.bit,
})
node.pointer, err = n.reader.readNode(node.pointer, 0)
if err != nil {
n.err = err
return false
}
} else if node.pointer > n.reader.Metadata.NodeCount {
n.lastNode = node
return true
} else {
break
}
}
}
return false
}
// Network returns the current network or an error if there is a problem
// decoding the data for the network. It takes a pointer to a result value to
// decode the network's data into.
func (n *Networks) Network(result interface{}) (*net.IPNet, error) {
if err := n.reader.retrieveData(n.lastNode.pointer, result); err != nil {
return nil, err
}
return &net.IPNet{
IP: n.lastNode.ip,
Mask: net.CIDRMask(int(n.lastNode.bit), len(n.lastNode.ip)*8),
}, nil
}
// Err returns an error, if any, that was encountered during iteration.
func (n *Networks) Err() error {
return n.err
}

View File

@ -1,185 +0,0 @@
package maxminddb
import "reflect"
type verifier struct {
reader *Reader
}
// Verify checks that the database is valid. It validates the search tree,
// the data section, and the metadata section. This verifier is stricter than
// the specification and may return errors on databases that are readable.
func (r *Reader) Verify() error {
v := verifier{r}
if err := v.verifyMetadata(); err != nil {
return err
}
return v.verifyDatabase()
}
func (v *verifier) verifyMetadata() error {
metadata := v.reader.Metadata
if metadata.BinaryFormatMajorVersion != 2 {
return testError(
"binary_format_major_version",
2,
metadata.BinaryFormatMajorVersion,
)
}
if metadata.BinaryFormatMinorVersion != 0 {
return testError(
"binary_format_minor_version",
0,
metadata.BinaryFormatMinorVersion,
)
}
if metadata.DatabaseType == "" {
return testError(
"database_type",
"non-empty string",
metadata.DatabaseType,
)
}
if len(metadata.Description) == 0 {
return testError(
"description",
"non-empty slice",
metadata.Description,
)
}
if metadata.IPVersion != 4 && metadata.IPVersion != 6 {
return testError(
"ip_version",
"4 or 6",
metadata.IPVersion,
)
}
if metadata.RecordSize != 24 &&
metadata.RecordSize != 28 &&
metadata.RecordSize != 32 {
return testError(
"record_size",
"24, 28, or 32",
metadata.RecordSize,
)
}
if metadata.NodeCount == 0 {
return testError(
"node_count",
"positive integer",
metadata.NodeCount,
)
}
return nil
}
func (v *verifier) verifyDatabase() error {
offsets, err := v.verifySearchTree()
if err != nil {
return err
}
if err := v.verifyDataSectionSeparator(); err != nil {
return err
}
return v.verifyDataSection(offsets)
}
func (v *verifier) verifySearchTree() (map[uint]bool, error) {
offsets := make(map[uint]bool)
it := v.reader.Networks()
for it.Next() {
offset, err := v.reader.resolveDataPointer(it.lastNode.pointer)
if err != nil {
return nil, err
}
offsets[offset] = true
}
if err := it.Err(); err != nil {
return nil, err
}
return offsets, nil
}
func (v *verifier) verifyDataSectionSeparator() error {
separatorStart := v.reader.Metadata.NodeCount * v.reader.Metadata.RecordSize / 4
separator := v.reader.buffer[separatorStart : separatorStart+dataSectionSeparatorSize]
for _, b := range separator {
if b != 0 {
return newInvalidDatabaseError("unexpected byte in data separator: %v", separator)
}
}
return nil
}
func (v *verifier) verifyDataSection(offsets map[uint]bool) error {
pointerCount := len(offsets)
decoder := v.reader.decoder
var offset uint
bufferLen := uint(len(decoder.buffer))
for offset < bufferLen {
var data interface{}
rv := reflect.ValueOf(&data)
newOffset, err := decoder.decode(offset, rv)
if err != nil {
return newInvalidDatabaseError("received decoding error (%v) at offset of %v", err, offset)
}
if newOffset <= offset {
return newInvalidDatabaseError("data section offset unexpectedly went from %v to %v", offset, newOffset)
}
pointer := offset
if _, ok := offsets[pointer]; ok {
delete(offsets, pointer)
} else {
return newInvalidDatabaseError("found data (%v) at %v that the search tree does not point to", data, pointer)
}
offset = newOffset
}
if offset != bufferLen {
return newInvalidDatabaseError(
"unexpected data at the end of the data section (last offset: %v, end: %v)",
offset,
bufferLen,
)
}
if len(offsets) != 0 {
return newInvalidDatabaseError(
"found %v pointers (of %v) in the search tree that we did not see in the data section",
len(offsets),
pointerCount,
)
}
return nil
}
func testError(
field string,
expected interface{},
actual interface{},
) error {
return newInvalidDatabaseError(
"%v - Expected: %v Actual: %v",
field,
expected,
actual,
)
}

View File

@ -1,140 +0,0 @@
// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package dialer
import (
"net"
"net/http"
"net/url"
"os"
"strings"
"time"
"golang.org/x/net/proxy"
"github.com/syncthing/syncthing/lib/logger"
"github.com/syncthing/syncthing/lib/osutil"
)
var (
l = logger.DefaultLogger.NewFacility("dialer", "Dialing connections")
proxyDialer = getDialer(proxy.Direct)
usingProxy = proxyDialer != proxy.Direct
)
type dialFunc func(network, addr string) (net.Conn, error)
func init() {
l.SetDebug("dialer", strings.Contains(os.Getenv("STTRACE"), "dialer") || os.Getenv("STTRACE") == "all")
if usingProxy {
http.DefaultTransport = &http.Transport{
Dial: Dial,
Proxy: http.ProxyFromEnvironment,
TLSHandshakeTimeout: 10 * time.Second,
}
// Defer this, so that logging gets setup.
go func() {
time.Sleep(500 * time.Millisecond)
l.Infoln("Proxy settings detected")
}()
} else {
go func() {
time.Sleep(500 * time.Millisecond)
l.Debugln("Dialer logging disabled, as no proxy was detected")
}()
}
}
func dialWithFallback(proxyDialFunc dialFunc, fallbackDialFunc dialFunc, network, addr string) (net.Conn, error) {
conn, err := proxyDialFunc(network, addr)
if err == nil {
l.Debugf("Dialing %s address %s via proxy - success, %s -> %s", network, addr, conn.LocalAddr(), conn.RemoteAddr())
if tcpconn, ok := conn.(*net.TCPConn); ok {
osutil.SetTCPOptions(tcpconn)
}
return dialerConn{
conn, newDialerAddr(network, addr),
}, nil
}
l.Debugf("Dialing %s address %s via proxy - error %s", network, addr, err)
conn, err = fallbackDialFunc(network, addr)
if err == nil {
l.Debugf("Dialing %s address %s via fallback - success, %s -> %s", network, addr, conn.LocalAddr(), conn.RemoteAddr())
if tcpconn, ok := conn.(*net.TCPConn); ok {
osutil.SetTCPOptions(tcpconn)
}
} else {
l.Debugf("Dialing %s address %s via fallback - error %s", network, addr, err)
}
return conn, err
}
// This is a rip off of proxy.FromEnvironment with a custom forward dialer
func getDialer(forward proxy.Dialer) proxy.Dialer {
allProxy := os.Getenv("all_proxy")
if len(allProxy) == 0 {
return forward
}
proxyURL, err := url.Parse(allProxy)
if err != nil {
return forward
}
prxy, err := proxy.FromURL(proxyURL, forward)
if err != nil {
return forward
}
noProxy := os.Getenv("no_proxy")
if len(noProxy) == 0 {
return prxy
}
perHost := proxy.NewPerHost(prxy, forward)
perHost.AddFromString(noProxy)
return perHost
}
type timeoutDirectDialer struct {
timeout time.Duration
}
func (d *timeoutDirectDialer) Dial(network, addr string) (net.Conn, error) {
return net.DialTimeout(network, addr, d.timeout)
}
type dialerConn struct {
net.Conn
addr net.Addr
}
func (c dialerConn) RemoteAddr() net.Addr {
return c.addr
}
func newDialerAddr(network, addr string) net.Addr {
netaddr, err := net.ResolveIPAddr(network, addr)
if err == nil {
return netaddr
}
return fallbackAddr{network, addr}
}
type fallbackAddr struct {
network string
addr string
}
func (a fallbackAddr) Network() string {
return a.network
}
func (a fallbackAddr) String() string {
return a.addr
}

View File

@ -1,48 +0,0 @@
// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package dialer
import (
"net"
"time"
)
// Dial tries dialing via proxy if a proxy is configured, and falls back to
// a direct connection if no proxy is defined, or connecting via proxy fails.
func Dial(network, addr string) (net.Conn, error) {
if usingProxy {
return dialWithFallback(proxyDialer.Dial, net.Dial, network, addr)
}
return net.Dial(network, addr)
}
// DialTimeout tries dialing via proxy with a timeout if a proxy is configured,
// and falls back to a direct connection if no proxy is defined, or connecting
// via proxy fails. The timeout can potentially be applied twice, once trying
// to connect via the proxy connection, and second time trying to connect
// directly.
func DialTimeout(network, addr string, timeout time.Duration) (net.Conn, error) {
if usingProxy {
// Because the proxy package is poorly structured, we have to
// construct a struct that matches proxy.Dialer but has a timeout
// and reconstrcut the proxy dialer using that, in order to be able to
// set a timeout.
dd := &timeoutDirectDialer{
timeout: timeout,
}
// Check if the dialer we are getting is not timeoutDirectDialer we just
// created. It could happen that usingProxy is true, but getDialer
// returns timeoutDirectDialer due to env vars changing.
if timeoutProxyDialer := getDialer(dd); timeoutProxyDialer != dd {
directDialFunc := func(inetwork, iaddr string) (net.Conn, error) {
return net.DialTimeout(inetwork, iaddr, timeout)
}
return dialWithFallback(timeoutProxyDialer.Dial, directDialFunc, network, addr)
}
}
return net.DialTimeout(network, addr, timeout)
}

View File

@ -1,19 +0,0 @@
Copyright (C) 2013 Jakob Borg
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:
- The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

View File

@ -1,385 +0,0 @@
// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
// is governed by an MIT-style license that can be found in the LICENSE file.
// Package logger implements a standardized logger with callback functionality
package logger
import (
"fmt"
"io/ioutil"
"log"
"os"
"strings"
"sync"
"time"
)
// This package uses stdlib sync as it may be used to debug syncthing/lib/sync
// and that would cause an implosion of the universe.
type LogLevel int
const (
LevelDebug LogLevel = iota
LevelVerbose
LevelInfo
LevelOK
LevelWarn
LevelFatal
NumLevels
)
// A MessageHandler is called with the log level and message text.
type MessageHandler func(l LogLevel, msg string)
type Logger interface {
AddHandler(level LogLevel, h MessageHandler)
SetFlags(flag int)
SetPrefix(prefix string)
Debugln(vals ...interface{})
Debugf(format string, vals ...interface{})
Verboseln(vals ...interface{})
Verbosef(format string, vals ...interface{})
Infoln(vals ...interface{})
Infof(format string, vals ...interface{})
Okln(vals ...interface{})
Okf(format string, vals ...interface{})
Warnln(vals ...interface{})
Warnf(format string, vals ...interface{})
Fatalln(vals ...interface{})
Fatalf(format string, vals ...interface{})
ShouldDebug(facility string) bool
SetDebug(facility string, enabled bool)
Facilities() map[string]string
FacilityDebugging() []string
NewFacility(facility, description string) Logger
}
type logger struct {
logger *log.Logger
handlers [NumLevels][]MessageHandler
facilities map[string]string // facility name => description
debug map[string]bool // facility name => debugging enabled
mut sync.Mutex
}
// DefaultLogger logs to standard output with a time prefix.
var DefaultLogger = New()
func New() Logger {
if os.Getenv("LOGGER_DISCARD") != "" {
// Hack to completely disable logging, for example when running benchmarks.
return &logger{
logger: log.New(ioutil.Discard, "", 0),
}
}
return &logger{
logger: log.New(os.Stdout, "", log.Ltime),
}
}
// AddHandler registers a new MessageHandler to receive messages with the
// specified log level or above.
func (l *logger) AddHandler(level LogLevel, h MessageHandler) {
l.mut.Lock()
defer l.mut.Unlock()
l.handlers[level] = append(l.handlers[level], h)
}
// See log.SetFlags
func (l *logger) SetFlags(flag int) {
l.logger.SetFlags(flag)
}
// See log.SetPrefix
func (l *logger) SetPrefix(prefix string) {
l.logger.SetPrefix(prefix)
}
func (l *logger) callHandlers(level LogLevel, s string) {
for ll := LevelDebug; ll <= level; ll++ {
for _, h := range l.handlers[ll] {
h(level, strings.TrimSpace(s))
}
}
}
// Debugln logs a line with a DEBUG prefix.
func (l *logger) Debugln(vals ...interface{}) {
l.debugln(3, vals)
}
func (l *logger) debugln(level int, vals ...interface{}) {
l.mut.Lock()
defer l.mut.Unlock()
s := fmt.Sprintln(vals...)
l.logger.Output(level, "DEBUG: "+s)
l.callHandlers(LevelDebug, s)
}
// Debugf logs a formatted line with a DEBUG prefix.
func (l *logger) Debugf(format string, vals ...interface{}) {
l.debugf(3, format, vals...)
}
func (l *logger) debugf(level int, format string, vals ...interface{}) {
l.mut.Lock()
defer l.mut.Unlock()
s := fmt.Sprintf(format, vals...)
l.logger.Output(level, "DEBUG: "+s)
l.callHandlers(LevelDebug, s)
}
// Infoln logs a line with a VERBOSE prefix.
func (l *logger) Verboseln(vals ...interface{}) {
l.mut.Lock()
defer l.mut.Unlock()
s := fmt.Sprintln(vals...)
l.logger.Output(2, "VERBOSE: "+s)
l.callHandlers(LevelVerbose, s)
}
// Infof logs a formatted line with a VERBOSE prefix.
func (l *logger) Verbosef(format string, vals ...interface{}) {
l.mut.Lock()
defer l.mut.Unlock()
s := fmt.Sprintf(format, vals...)
l.logger.Output(2, "VERBOSE: "+s)
l.callHandlers(LevelVerbose, s)
}
// Infoln logs a line with an INFO prefix.
func (l *logger) Infoln(vals ...interface{}) {
l.mut.Lock()
defer l.mut.Unlock()
s := fmt.Sprintln(vals...)
l.logger.Output(2, "INFO: "+s)
l.callHandlers(LevelInfo, s)
}
// Infof logs a formatted line with an INFO prefix.
func (l *logger) Infof(format string, vals ...interface{}) {
l.mut.Lock()
defer l.mut.Unlock()
s := fmt.Sprintf(format, vals...)
l.logger.Output(2, "INFO: "+s)
l.callHandlers(LevelInfo, s)
}
// Okln logs a line with an OK prefix.
func (l *logger) Okln(vals ...interface{}) {
l.mut.Lock()
defer l.mut.Unlock()
s := fmt.Sprintln(vals...)
l.logger.Output(2, "OK: "+s)
l.callHandlers(LevelOK, s)
}
// Okf logs a formatted line with an OK prefix.
func (l *logger) Okf(format string, vals ...interface{}) {
l.mut.Lock()
defer l.mut.Unlock()
s := fmt.Sprintf(format, vals...)
l.logger.Output(2, "OK: "+s)
l.callHandlers(LevelOK, s)
}
// Warnln logs a formatted line with a WARNING prefix.
func (l *logger) Warnln(vals ...interface{}) {
l.mut.Lock()
defer l.mut.Unlock()
s := fmt.Sprintln(vals...)
l.logger.Output(2, "WARNING: "+s)
l.callHandlers(LevelWarn, s)
}
// Warnf logs a formatted line with a WARNING prefix.
func (l *logger) Warnf(format string, vals ...interface{}) {
l.mut.Lock()
defer l.mut.Unlock()
s := fmt.Sprintf(format, vals...)
l.logger.Output(2, "WARNING: "+s)
l.callHandlers(LevelWarn, s)
}
// Fatalln logs a line with a FATAL prefix and exits the process with exit
// code 1.
func (l *logger) Fatalln(vals ...interface{}) {
l.mut.Lock()
defer l.mut.Unlock()
s := fmt.Sprintln(vals...)
l.logger.Output(2, "FATAL: "+s)
l.callHandlers(LevelFatal, s)
os.Exit(1)
}
// Fatalf logs a formatted line with a FATAL prefix and exits the process with
// exit code 1.
func (l *logger) Fatalf(format string, vals ...interface{}) {
l.mut.Lock()
defer l.mut.Unlock()
s := fmt.Sprintf(format, vals...)
l.logger.Output(2, "FATAL: "+s)
l.callHandlers(LevelFatal, s)
os.Exit(1)
}
// ShouldDebug returns true if the given facility has debugging enabled.
func (l *logger) ShouldDebug(facility string) bool {
l.mut.Lock()
res := l.debug[facility]
l.mut.Unlock()
return res
}
// SetDebug enabled or disables debugging for the given facility name.
func (l *logger) SetDebug(facility string, enabled bool) {
l.mut.Lock()
l.debug[facility] = enabled
l.mut.Unlock()
}
// FacilityDebugging returns the set of facilities that have debugging
// enabled.
func (l *logger) FacilityDebugging() []string {
var enabled []string
l.mut.Lock()
for facility, isEnabled := range l.debug {
if isEnabled {
enabled = append(enabled, facility)
}
}
l.mut.Unlock()
return enabled
}
// Facilities returns the currently known set of facilities and their
// descriptions.
func (l *logger) Facilities() map[string]string {
l.mut.Lock()
res := make(map[string]string, len(l.facilities))
for facility, descr := range l.facilities {
res[facility] = descr
}
l.mut.Unlock()
return res
}
// NewFacility returns a new logger bound to the named facility.
func (l *logger) NewFacility(facility, description string) Logger {
l.mut.Lock()
if l.facilities == nil {
l.facilities = make(map[string]string)
}
if description != "" {
l.facilities[facility] = description
}
if l.debug == nil {
l.debug = make(map[string]bool)
}
l.debug[facility] = false
l.mut.Unlock()
return &facilityLogger{
logger: l,
facility: facility,
}
}
// A facilityLogger is a regular logger but bound to a facility name. The
// Debugln and Debugf methods are no-ops unless debugging has been enabled for
// this facility on the parent logger.
type facilityLogger struct {
*logger
facility string
}
// Debugln logs a line with a DEBUG prefix.
func (l *facilityLogger) Debugln(vals ...interface{}) {
if !l.ShouldDebug(l.facility) {
return
}
l.logger.debugln(3, vals...)
}
// Debugf logs a formatted line with a DEBUG prefix.
func (l *facilityLogger) Debugf(format string, vals ...interface{}) {
if !l.ShouldDebug(l.facility) {
return
}
l.logger.debugf(3, format, vals...)
}
// A Recorder keeps a size limited record of log events.
type Recorder struct {
lines []Line
initial int
mut sync.Mutex
}
// A Line represents a single log entry.
type Line struct {
When time.Time `json:"when"`
Message string `json:"message"`
}
func NewRecorder(l Logger, level LogLevel, size, initial int) *Recorder {
r := &Recorder{
lines: make([]Line, 0, size),
initial: initial,
}
l.AddHandler(level, r.append)
return r
}
func (r *Recorder) Since(t time.Time) []Line {
r.mut.Lock()
defer r.mut.Unlock()
res := r.lines
for i := 0; i < len(res) && res[i].When.Before(t); i++ {
// nothing, just incrementing i
}
if len(res) == 0 {
return nil
}
// We must copy the result as r.lines can be mutated as soon as the lock
// is released.
cp := make([]Line, len(res))
copy(cp, res)
return cp
}
func (r *Recorder) Clear() {
r.mut.Lock()
r.lines = r.lines[:0]
r.mut.Unlock()
}
func (r *Recorder) append(l LogLevel, msg string) {
line := Line{
When: time.Now(),
Message: msg,
}
r.mut.Lock()
defer r.mut.Unlock()
if len(r.lines) == cap(r.lines) {
if r.initial > 0 {
// Shift all lines one step to the left, keeping the "initial" first intact.
copy(r.lines[r.initial+1:], r.lines[r.initial+2:])
} else {
copy(r.lines, r.lines[1:])
}
// Add the new one at the end
r.lines[len(r.lines)-1] = line
return
}
r.lines = append(r.lines, line)
if len(r.lines) == r.initial {
r.lines = append(r.lines, Line{time.Now(), "..."})
}
}

View File

@ -1,104 +0,0 @@
// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package osutil
import (
"errors"
"io/ioutil"
"os"
"path/filepath"
"runtime"
)
var (
ErrClosed = errors.New("write to closed writer")
TempPrefix = ".syncthing.tmp."
)
// An AtomicWriter is an *os.File that writes to a temporary file in the same
// directory as the final path. On successfull Close the file is renamed to
// it's final path. Any error on Write or during Close is accumulated and
// returned on Close, so a lazy user can ignore errors until Close.
type AtomicWriter struct {
path string
next *os.File
err error
}
// CreateAtomic is like os.Create with a FileMode, except a temporary file
// name is used instead of the given name.
func CreateAtomic(path string, mode os.FileMode) (*AtomicWriter, error) {
fd, err := ioutil.TempFile(filepath.Dir(path), TempPrefix)
if err != nil {
return nil, err
}
// chmod fails on Android so don't even try
if runtime.GOOS != "android" {
if err := os.Chmod(fd.Name(), mode); err != nil {
fd.Close()
os.Remove(fd.Name())
return nil, err
}
}
w := &AtomicWriter{
path: path,
next: fd,
}
return w, nil
}
// Write is like io.Writer, but is a no-op on an already failed AtomicWriter.
func (w *AtomicWriter) Write(bs []byte) (int, error) {
if w.err != nil {
return 0, w.err
}
n, err := w.next.Write(bs)
if err != nil {
w.err = err
w.next.Close()
}
return n, err
}
// Close closes the temporary file and renames it to the final path. It is
// invalid to call Write() or Close() after Close().
func (w *AtomicWriter) Close() error {
if w.err != nil {
return w.err
}
// Try to not leave temp file around, but ignore error.
defer os.Remove(w.next.Name())
if err := w.next.Close(); err != nil {
w.err = err
return err
}
// Remove the destination file, on Windows only. If it fails, and not due
// to the file not existing, we won't be able to complete the rename
// either. Return this error because it may be more informative. On non-
// Windows we want the atomic rename behavior so we don't attempt remove.
if runtime.GOOS == "windows" {
if err := os.Remove(w.path); err != nil && !os.IsNotExist(err) {
return err
}
}
if err := os.Rename(w.next.Name(), w.path); err != nil {
w.err = err
return err
}
// Set w.err to return appropriately for any future operations.
w.err = ErrClosed
return nil
}

View File

@ -1,17 +0,0 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package osutil
import "golang.org/x/text/unicode/norm"
func NormalizedFilename(s string) string {
return norm.NFC.String(s)
}
func NativeFilename(s string) string {
return norm.NFD.String(s)
}

View File

@ -1,19 +0,0 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
// +build !windows,!darwin
package osutil
import "golang.org/x/text/unicode/norm"
func NormalizedFilename(s string) string {
return norm.NFC.String(s)
}
func NativeFilename(s string) string {
return s
}

View File

@ -1,21 +0,0 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package osutil
import (
"path/filepath"
"golang.org/x/text/unicode/norm"
)
func NormalizedFilename(s string) string {
return norm.NFC.String(filepath.ToSlash(s))
}
func NativeFilename(s string) string {
return filepath.FromSlash(s)
}

View File

@ -1,17 +0,0 @@
// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
// +build !windows
package osutil
import (
"path/filepath"
)
func Glob(pattern string) (matches []string, err error) {
return filepath.Glob(pattern)
}

View File

@ -1,93 +0,0 @@
// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
// +build windows
package osutil
import (
"os"
"path/filepath"
"sort"
"strings"
)
// Glob implements filepath.Glob, but works with Windows long path prefixes.
// Deals with https://github.com/golang/go/issues/10577
func Glob(pattern string) (matches []string, err error) {
if !hasMeta(pattern) {
if _, err = os.Lstat(pattern); err != nil {
return nil, nil
}
return []string{pattern}, nil
}
dir, file := filepath.Split(pattern)
switch dir {
case "":
dir = "."
case string(filepath.Separator):
// nothing
default:
dir = dir[0 : len(dir)-1] // chop off trailing separator
}
if !hasMeta(dir) {
return glob(dir, file, nil)
}
var m []string
m, err = Glob(dir)
if err != nil {
return
}
for _, d := range m {
matches, err = glob(d, file, matches)
if err != nil {
return
}
}
return
}
func hasMeta(path string) bool {
// Strip off Windows long path prefix if it exists.
if strings.HasPrefix(path, "\\\\?\\") {
path = path[4:]
}
// TODO(niemeyer): Should other magic characters be added here?
return strings.IndexAny(path, "*?[") >= 0
}
func glob(dir, pattern string, matches []string) (m []string, e error) {
m = matches
fi, err := os.Stat(dir)
if err != nil {
return
}
if !fi.IsDir() {
return
}
d, err := os.Open(dir)
if err != nil {
return
}
defer d.Close()
names, _ := d.Readdirnames(-1)
sort.Strings(names)
for _, n := range names {
matched, err := filepath.Match(pattern, n)
if err != nil {
return m, err
}
if matched {
m = append(m, filepath.Join(dir, n))
}
}
return
}

View File

@ -1,19 +0,0 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
// +build !windows
package osutil
func HideFile(path string) error {
return nil
}
func ShowFile(path string) error {
return nil
}
func HideConsole() {}

View File

@ -1,52 +0,0 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
// +build windows
package osutil
import "syscall"
func HideFile(path string) error {
p, err := syscall.UTF16PtrFromString(path)
if err != nil {
return err
}
attrs, err := syscall.GetFileAttributes(p)
if err != nil {
return err
}
attrs |= syscall.FILE_ATTRIBUTE_HIDDEN
return syscall.SetFileAttributes(p, attrs)
}
func ShowFile(path string) error {
p, err := syscall.UTF16PtrFromString(path)
if err != nil {
return err
}
attrs, err := syscall.GetFileAttributes(p)
if err != nil {
return err
}
attrs &^= syscall.FILE_ATTRIBUTE_HIDDEN
return syscall.SetFileAttributes(p, attrs)
}
func HideConsole() {
getConsoleWindow := syscall.NewLazyDLL("kernel32.dll").NewProc("GetConsoleWindow")
showWindow := syscall.NewLazyDLL("user32.dll").NewProc("ShowWindow")
if getConsoleWindow.Find() == nil && showWindow.Find() == nil {
hwnd, _, _ := getConsoleWindow.Call()
if hwnd != 0 {
showWindow.Call(hwnd, 0)
}
}
}

View File

@ -1,30 +0,0 @@
// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
// +build !windows
package osutil
import (
"net"
)
func GetLans() ([]*net.IPNet, error) {
addrs, err := net.InterfaceAddrs()
if err != nil {
return nil, err
}
nets := make([]*net.IPNet, 0, len(addrs))
for _, addr := range addrs {
net, ok := addr.(*net.IPNet)
if ok {
nets = append(nets, net)
}
}
return nets, nil
}

View File

@ -1,83 +0,0 @@
// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
// +build windows
package osutil
import (
"net"
"os"
"strings"
"syscall"
"unsafe"
)
// Modified version of:
// http://stackoverflow.com/questions/23529663/how-to-get-all-addresses-and-masks-from-local-interfaces-in-go
// v4 only!
func getAdapterList() (*syscall.IpAdapterInfo, error) {
b := make([]byte, 10240)
l := uint32(len(b))
a := (*syscall.IpAdapterInfo)(unsafe.Pointer(&b[0]))
// TODO(mikio): GetAdaptersInfo returns IP_ADAPTER_INFO that
// contains IPv4 address list only. We should use another API
// for fetching IPv6 stuff from the kernel.
err := syscall.GetAdaptersInfo(a, &l)
if err == syscall.ERROR_BUFFER_OVERFLOW {
b = make([]byte, l)
a = (*syscall.IpAdapterInfo)(unsafe.Pointer(&b[0]))
err = syscall.GetAdaptersInfo(a, &l)
}
if err != nil {
return nil, os.NewSyscallError("GetAdaptersInfo", err)
}
return a, nil
}
func GetLans() ([]*net.IPNet, error) {
ifaces, err := net.Interfaces()
if err != nil {
return nil, err
}
nets := make([]*net.IPNet, 0, len(ifaces))
aList, err := getAdapterList()
if err != nil {
return nil, err
}
for _, ifi := range ifaces {
for ai := aList; ai != nil; ai = ai.Next {
index := ai.Index
if ifi.Index == int(index) {
ipl := &ai.IpAddressList
for ; ipl != nil; ipl = ipl.Next {
ipStr := strings.Trim(string(ipl.IpAddress.String[:]), "\x00")
maskStr := strings.Trim(string(ipl.IpMask.String[:]), "\x00")
ip := net.ParseIP(ipStr)
maskip := net.ParseIP(maskStr)
if ip.IsUnspecified() || maskip.IsUnspecified() {
continue
}
nets = append(nets, &net.IPNet{
IP: ip,
Mask: net.IPv4Mask(
maskip[net.IPv6len-net.IPv4len],
maskip[net.IPv6len-net.IPv4len+1],
maskip[net.IPv6len-net.IPv4len+2],
maskip[net.IPv6len-net.IPv4len+3],
),
})
}
}
}
}
return nets, err
}

View File

@ -1,29 +0,0 @@
// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
// +build linux android
package osutil
import (
"os"
"syscall"
"time"
)
// Lstat is like os.Lstat, except lobotomized for Android. See
// https://forum.syncthing.net/t/2395
func Lstat(name string) (fi os.FileInfo, err error) {
for i := 0; i < 10; i++ { // We have to draw the line somewhere
fi, err = os.Lstat(name)
if err, ok := err.(*os.PathError); ok && err.Err == syscall.EINTR {
time.Sleep(time.Duration(i+1) * time.Millisecond)
continue
}
return
}
return
}

View File

@ -1,15 +0,0 @@
// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
// +build !linux,!android
package osutil
import "os"
func Lstat(name string) (fi os.FileInfo, err error) {
return os.Lstat(name)
}

View File

@ -1,17 +0,0 @@
// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
// +build !windows
package osutil
import (
"os"
)
func MkdirAll(path string, perm os.FileMode) error {
return os.MkdirAll(path, perm)
}

View File

@ -1,65 +0,0 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Modified by Zillode to fix https://github.com/syncthing/syncthing/issues/1822
// Sync with https://github.com/golang/go/blob/master/src/os/path.go
// See https://github.com/golang/go/issues/10900
package osutil
import (
"os"
"path/filepath"
"syscall"
)
// MkdirAll creates a directory named path, along with any necessary parents,
// and returns nil, or else returns an error.
// The permission bits perm are used for all directories that MkdirAll creates.
// If path is already a directory, MkdirAll does nothing and returns nil.
func MkdirAll(path string, perm os.FileMode) error {
// Fast path: if we can tell whether path is a directory or file, stop with success or error.
dir, err := os.Stat(path)
if err == nil {
if dir.IsDir() {
return nil
}
return &os.PathError{"mkdir", path, syscall.ENOTDIR}
}
// Slow path: make sure parent exists and then call Mkdir for path.
i := len(path)
for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator.
i--
}
j := i
for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element.
j--
}
if j > 1 {
// Create parent
parent := path[0 : j-1]
if parent != filepath.VolumeName(parent) {
err = MkdirAll(parent, perm)
if err != nil {
return err
}
}
}
// Parent now exists; invoke Mkdir and use its result.
err = os.Mkdir(path, perm)
if err != nil {
// Handle arguments like "foo/." by
// double-checking that directory doesn't exist.
dir, err1 := os.Lstat(path)
if err1 == nil && dir.IsDir() {
return nil
}
return err
}
return nil
}

View File

@ -1,243 +0,0 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
// Package osutil implements utilities for native OS support.
package osutil
import (
"errors"
"fmt"
"io"
"net"
"os"
"path/filepath"
"runtime"
"strings"
"time"
"github.com/calmh/du"
"github.com/syncthing/syncthing/lib/sync"
)
var ErrNoHome = errors.New("No home directory found - set $HOME (or the platform equivalent).")
// Try to keep this entire operation atomic-like. We shouldn't be doing this
// often enough that there is any contention on this lock.
var renameLock = sync.NewMutex()
// TryRename renames a file, leaving source file intact in case of failure.
// Tries hard to succeed on various systems by temporarily tweaking directory
// permissions and removing the destination file when necessary.
func TryRename(from, to string) error {
renameLock.Lock()
defer renameLock.Unlock()
return withPreparedTarget(from, to, func() error {
return os.Rename(from, to)
})
}
// Rename moves a temporary file to it's final place.
// Will make sure to delete the from file if the operation fails, so use only
// for situations like committing a temp file to it's final location.
// Tries hard to succeed on various systems by temporarily tweaking directory
// permissions and removing the destination file when necessary.
func Rename(from, to string) error {
// Don't leave a dangling temp file in case of rename error
if !(runtime.GOOS == "windows" && strings.EqualFold(from, to)) {
defer os.Remove(from)
}
return TryRename(from, to)
}
// Copy copies the file content from source to destination.
// Tries hard to succeed on various systems by temporarily tweaking directory
// permissions and removing the destination file when necessary.
func Copy(from, to string) (err error) {
return withPreparedTarget(from, to, func() error {
return copyFileContents(from, to)
})
}
// InWritableDir calls fn(path), while making sure that the directory
// containing `path` is writable for the duration of the call.
func InWritableDir(fn func(string) error, path string) error {
dir := filepath.Dir(path)
info, err := os.Stat(dir)
if err != nil {
return err
}
if !info.IsDir() {
return errors.New("Not a directory: " + path)
}
if info.Mode()&0200 == 0 {
// A non-writeable directory (for this user; we assume that's the
// relevant part). Temporarily change the mode so we can delete the
// file or directory inside it.
err = os.Chmod(dir, 0755)
if err == nil {
defer func() {
err = os.Chmod(dir, info.Mode())
if err != nil {
// We managed to change the permission bits like a
// millisecond ago, so it'd be bizarre if we couldn't
// change it back.
panic(err)
}
}()
}
}
return fn(path)
}
// Remove removes the given path. On Windows, removes the read-only attribute
// from the target prior to deletion.
func Remove(path string) error {
if runtime.GOOS == "windows" {
info, err := os.Stat(path)
if err != nil {
return err
}
if info.Mode()&0200 == 0 {
os.Chmod(path, 0700)
}
}
return os.Remove(path)
}
func ExpandTilde(path string) (string, error) {
if path == "~" {
return getHomeDir()
}
path = filepath.FromSlash(path)
if !strings.HasPrefix(path, fmt.Sprintf("~%c", os.PathSeparator)) {
return path, nil
}
home, err := getHomeDir()
if err != nil {
return "", err
}
return filepath.Join(home, path[2:]), nil
}
func getHomeDir() (string, error) {
var home string
switch runtime.GOOS {
case "windows":
home = filepath.Join(os.Getenv("HomeDrive"), os.Getenv("HomePath"))
if home == "" {
home = os.Getenv("UserProfile")
}
default:
home = os.Getenv("HOME")
}
if home == "" {
return "", ErrNoHome
}
return home, nil
}
// Tries hard to succeed on various systems by temporarily tweaking directory
// permissions and removing the destination file when necessary.
func withPreparedTarget(from, to string, f func() error) error {
// Make sure the destination directory is writeable
toDir := filepath.Dir(to)
if info, err := os.Stat(toDir); err == nil && info.IsDir() && info.Mode()&0200 == 0 {
os.Chmod(toDir, 0755)
defer os.Chmod(toDir, info.Mode())
}
// On Windows, make sure the destination file is writeable (or we can't delete it)
if runtime.GOOS == "windows" {
os.Chmod(to, 0666)
if !strings.EqualFold(from, to) {
err := os.Remove(to)
if err != nil && !os.IsNotExist(err) {
return err
}
}
}
return f()
}
// copyFileContents copies the contents of the file named src to the file named
// by dst. The file will be created if it does not already exist. If the
// destination file exists, all it's contents will be replaced by the contents
// of the source file.
func copyFileContents(src, dst string) (err error) {
in, err := os.Open(src)
if err != nil {
return
}
defer in.Close()
out, err := os.Create(dst)
if err != nil {
return
}
defer func() {
cerr := out.Close()
if err == nil {
err = cerr
}
}()
if _, err = io.Copy(out, in); err != nil {
return
}
err = out.Sync()
return
}
var execExts map[string]bool
func init() {
// PATHEXT contains a list of executable file extensions, on Windows
pathext := filepath.SplitList(os.Getenv("PATHEXT"))
// We want the extensions in execExts to be lower case
execExts = make(map[string]bool, len(pathext))
for _, ext := range pathext {
execExts[strings.ToLower(ext)] = true
}
}
// IsWindowsExecutable returns true if the given path has an extension that is
// in the list of executable extensions.
func IsWindowsExecutable(path string) bool {
return execExts[strings.ToLower(filepath.Ext(path))]
}
func DiskFreeBytes(path string) (free int64, err error) {
u, err := du.Get(path)
return u.FreeBytes, err
}
func DiskFreePercentage(path string) (freePct float64, err error) {
u, err := du.Get(path)
return (float64(u.FreeBytes) / float64(u.TotalBytes)) * 100, err
}
// SetTCPOptions sets syncthings default TCP options on a TCP connection
func SetTCPOptions(conn *net.TCPConn) error {
var err error
if err = conn.SetLinger(0); err != nil {
return err
}
if err = conn.SetNoDelay(false); err != nil {
return err
}
if err = conn.SetKeepAlivePeriod(60 * time.Second); err != nil {
return err
}
if err = conn.SetKeepAlive(true); err != nil {
return err
}
return nil
}

View File

@ -1,39 +0,0 @@
// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package osutil
import (
"net"
"net/url"
"time"
)
// TCPPing returns the duration required to establish a TCP connection
// to the given host. ICMP packets require root priviledges, hence why we use
// tcp.
func TCPPing(address string) (time.Duration, error) {
dialer := net.Dialer{
Deadline: time.Now().Add(time.Second),
}
start := time.Now()
conn, err := dialer.Dial("tcp", address)
if conn != nil {
conn.Close()
}
return time.Since(start), err
}
// GetLatencyForURL parses the given URL, tries opening a TCP connection to it
// and returns the time it took to establish the connection.
func GetLatencyForURL(addr string) (time.Duration, error) {
uri, err := url.Parse(addr)
if err != nil {
return 0, err
}
return TCPPing(uri.Host)
}

View File

@ -1,48 +0,0 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package osutil
import (
"bytes"
"io"
)
type ReplacingWriter struct {
Writer io.Writer
From byte
To []byte
}
func (w ReplacingWriter) Write(bs []byte) (int, error) {
var n, written int
var err error
newlineIdx := bytes.IndexByte(bs, w.From)
for newlineIdx >= 0 {
n, err = w.Writer.Write(bs[:newlineIdx])
written += n
if err != nil {
break
}
if len(w.To) > 0 {
n, err := w.Writer.Write(w.To)
if n == len(w.To) {
written++
}
if err != nil {
break
}
}
bs = bs[newlineIdx+1:]
newlineIdx = bytes.IndexByte(bs, w.From)
}
n, err = w.Writer.Write(bs)
written += n
return written, err
}

View File

@ -1,47 +0,0 @@
// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
// +build !windows
package osutil
import "syscall"
// MaximizeOpenFileLimit tries to set the resoure limit RLIMIT_NOFILE (number
// of open file descriptors) to the max (hard limit), if the current (soft
// limit) is below the max. Returns the new (though possibly unchanged) limit,
// or an error if it was could not be changed.
func MaximizeOpenFileLimit() (int, error) {
// Get the current limit on number of open files.
var lim syscall.Rlimit
if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &lim); err != nil {
return 0, err
}
// If we're already at max, there's no need to try to raise the limit.
if lim.Cur >= lim.Max {
return int(lim.Cur), nil
}
// Try to increase the limit to the max.
oldLimit := lim.Cur
lim.Cur = lim.Max
if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &lim); err != nil {
return int(oldLimit), err
}
// If the set succeeded, perform a new get to see what happened. We might
// have gotten a value lower than the one in lim.Max, if lim.Max was
// something that indiciated "unlimited" (i.e. intmax).
if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &lim); err != nil {
// We don't really know the correct value here since Getrlimit
// mysteriously failed after working once... Shouldn't ever happen, I
// think.
return 0, err
}
return int(lim.Cur), nil
}

View File

@ -1,17 +0,0 @@
// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
// +build windows
package osutil
import (
"errors"
)
func MaximizeOpenFileLimit() (int, error) {
return 0, errors.New("not relevant on Windows")
}

View File

@ -1,4 +0,0 @@
# This is the official list of Protocol Authors for copyright purposes.
Audrius Butkevicius <audrius.butkevicius@gmail.com>
Jakob Borg <jakob@nym.se>

View File

@ -1,76 +0,0 @@
## Reporting Bugs
Please file bugs in the [Github Issue
Tracker](https://github.com/syncthing/protocol/issues).
## Contributing Code
Every contribution is welcome. Following the points below will make this
a smoother process.
Individuals making significant and valuable contributions are given
commit-access to the project. If you make a significant contribution and
are not considered for commit-access, please contact any of the
Syncthing core team members.
All nontrivial contributions should go through the pull request
mechanism for internal review. Determining what is "nontrivial" is left
at the discretion of the contributor.
### Authorship
All code authors are listed in the AUTHORS file. Commits must be made
with the same name and email as listed in the AUTHORS file. To
accomplish this, ensure that your git configuration is set correctly
prior to making your first commit;
$ git config --global user.name "Jane Doe"
$ git config --global user.email janedoe@example.com
You must be reachable on the given email address. If you do not wish to
use your real name for whatever reason, using a nickname or pseudonym is
perfectly acceptable.
## Coding Style
- Follow the conventions laid out in [Effective Go](https://golang.org/doc/effective_go.html)
as much as makes sense.
- All text files use Unix line endings.
- Each commit should be `go fmt` clean.
- The commit message subject should be a single short sentence
describing the change, starting with a capital letter.
- Commits that resolve an existing issue must include the issue number
as `(fixes #123)` at the end of the commit message subject.
- Imports are grouped per `goimports` standard; that is, standard
library first, then third party libraries after a blank line.
- A contribution solving a single issue or introducing a single new
feature should probably be a single commit based on the current
`master` branch. You may be asked to "rebase" or "squash" your pull
request to make sure this is the case, especially if there have been
amendments during review.
## Licensing
All contributions are made under the same MIT license as the rest of the
project, except documentation, user interface text and translation
strings which are licensed under the Creative Commons Attribution 4.0
International License. You retain the copyright to code you have
written.
When accepting your first contribution, the maintainer of the project
will ensure that you are added to the AUTHORS file. You are welcome to
add yourself as a separate commit in your first pull request.
## Tests
Yes please!
## License
MIT

View File

@ -1,19 +0,0 @@
Copyright (C) 2014-2015 The Protocol Authors
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:
- The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

View File

@ -1,13 +0,0 @@
The BEPv1 Protocol
==================
[![Latest Build](http://img.shields.io/jenkins/s/http/build.syncthing.net/protocol.svg?style=flat-square)](http://build.syncthing.net/job/protocol/lastBuild/)
[![API Documentation](http://img.shields.io/badge/api-Godoc-blue.svg?style=flat-square)](http://godoc.org/github.com/syncthing/protocol)
[![MIT License](http://img.shields.io/badge/license-MIT-blue.svg?style=flat-square)](http://opensource.org/licenses/MIT)
This is the protocol implementation used by Syncthing.
License
=======
MIT

View File

@ -1,53 +0,0 @@
// Copyright (C) 2015 The Protocol Authors.
package protocol
import "fmt"
type Compression int
const (
CompressMetadata Compression = iota // zero value is the default, default should be "metadata"
CompressNever
CompressAlways
compressionThreshold = 128 // don't bother compressing messages smaller than this many bytes
)
var compressionMarshal = map[Compression]string{
CompressNever: "never",
CompressMetadata: "metadata",
CompressAlways: "always",
}
var compressionUnmarshal = map[string]Compression{
// Legacy
"false": CompressNever,
"true": CompressMetadata,
// Current
"never": CompressNever,
"metadata": CompressMetadata,
"always": CompressAlways,
}
func (c Compression) String() string {
s, ok := compressionMarshal[c]
if !ok {
return fmt.Sprintf("unknown:%d", c)
}
return s
}
func (c Compression) GoString() string {
return fmt.Sprintf("%q", c.String())
}
func (c Compression) MarshalText() ([]byte, error) {
return []byte(compressionMarshal[c]), nil
}
func (c *Compression) UnmarshalText(bs []byte) error {
*c = compressionUnmarshal[string(bs)]
return nil
}

View File

@ -1,62 +0,0 @@
// Copyright (C) 2014 The Protocol Authors.
package protocol
import (
"io"
"sync/atomic"
"time"
)
type countingReader struct {
io.Reader
tot int64 // bytes
last int64 // unix nanos
}
var (
totalIncoming int64
totalOutgoing int64
)
func (c *countingReader) Read(bs []byte) (int, error) {
n, err := c.Reader.Read(bs)
atomic.AddInt64(&c.tot, int64(n))
atomic.AddInt64(&totalIncoming, int64(n))
atomic.StoreInt64(&c.last, time.Now().UnixNano())
return n, err
}
func (c *countingReader) Tot() int64 {
return atomic.LoadInt64(&c.tot)
}
func (c *countingReader) Last() time.Time {
return time.Unix(0, atomic.LoadInt64(&c.last))
}
type countingWriter struct {
io.Writer
tot int64 // bytes
last int64 // unix nanos
}
func (c *countingWriter) Write(bs []byte) (int, error) {
n, err := c.Writer.Write(bs)
atomic.AddInt64(&c.tot, int64(n))
atomic.AddInt64(&totalOutgoing, int64(n))
atomic.StoreInt64(&c.last, time.Now().UnixNano())
return n, err
}
func (c *countingWriter) Tot() int64 {
return atomic.LoadInt64(&c.tot)
}
func (c *countingWriter) Last() time.Time {
return time.Unix(0, atomic.LoadInt64(&c.last))
}
func TotalInOut() (int64, int64) {
return atomic.LoadInt64(&totalIncoming), atomic.LoadInt64(&totalOutgoing)
}

View File

@ -1,22 +0,0 @@
// Copyright (C) 2014 The Protocol Authors.
package protocol
import (
"os"
"strings"
"github.com/syncthing/syncthing/lib/logger"
)
var (
l = logger.DefaultLogger.NewFacility("protocol", "The BEP protocol")
)
func init() {
l.SetDebug("protocol", strings.Contains(os.Getenv("STTRACE"), "protocol") || os.Getenv("STTRACE") == "all")
}
func shouldDebug() bool {
return l.ShouldDebug("protocol")
}

View File

@ -1,163 +0,0 @@
// Copyright (C) 2014 The Protocol Authors.
package protocol
import (
"bytes"
"crypto/sha256"
"encoding/base32"
"encoding/binary"
"errors"
"fmt"
"regexp"
"strings"
"github.com/calmh/luhn"
)
type DeviceID [32]byte
var LocalDeviceID = DeviceID{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}
// NewDeviceID generates a new device ID from the raw bytes of a certificate
func NewDeviceID(rawCert []byte) DeviceID {
var n DeviceID
hf := sha256.New()
hf.Write(rawCert)
hf.Sum(n[:0])
return n
}
func DeviceIDFromString(s string) (DeviceID, error) {
var n DeviceID
err := n.UnmarshalText([]byte(s))
return n, err
}
func DeviceIDFromBytes(bs []byte) DeviceID {
var n DeviceID
if len(bs) != len(n) {
panic("incorrect length of byte slice representing device ID")
}
copy(n[:], bs)
return n
}
// String returns the canonical string representation of the device ID
func (n DeviceID) String() string {
id := base32.StdEncoding.EncodeToString(n[:])
id = strings.Trim(id, "=")
id, err := luhnify(id)
if err != nil {
// Should never happen
panic(err)
}
id = chunkify(id)
return id
}
func (n DeviceID) GoString() string {
return n.String()
}
func (n DeviceID) Compare(other DeviceID) int {
return bytes.Compare(n[:], other[:])
}
func (n DeviceID) Equals(other DeviceID) bool {
return bytes.Compare(n[:], other[:]) == 0
}
// Short returns an integer representing bits 0-63 of the device ID.
func (n DeviceID) Short() uint64 {
return binary.BigEndian.Uint64(n[:])
}
func (n *DeviceID) MarshalText() ([]byte, error) {
return []byte(n.String()), nil
}
func (n *DeviceID) UnmarshalText(bs []byte) error {
id := string(bs)
id = strings.Trim(id, "=")
id = strings.ToUpper(id)
id = untypeoify(id)
id = unchunkify(id)
var err error
switch len(id) {
case 56:
// New style, with check digits
id, err = unluhnify(id)
if err != nil {
return err
}
fallthrough
case 52:
// Old style, no check digits
dec, err := base32.StdEncoding.DecodeString(id + "====")
if err != nil {
return err
}
copy(n[:], dec)
return nil
default:
return errors.New("device ID invalid: incorrect length")
}
}
func luhnify(s string) (string, error) {
if len(s) != 52 {
panic("unsupported string length")
}
res := make([]string, 0, 4)
for i := 0; i < 4; i++ {
p := s[i*13 : (i+1)*13]
l, err := luhn.Base32.Generate(p)
if err != nil {
return "", err
}
res = append(res, fmt.Sprintf("%s%c", p, l))
}
return res[0] + res[1] + res[2] + res[3], nil
}
func unluhnify(s string) (string, error) {
if len(s) != 56 {
return "", fmt.Errorf("unsupported string length %d", len(s))
}
res := make([]string, 0, 4)
for i := 0; i < 4; i++ {
p := s[i*14 : (i+1)*14-1]
l, err := luhn.Base32.Generate(p)
if err != nil {
return "", err
}
if g := fmt.Sprintf("%s%c", p, l); g != s[i*14:(i+1)*14] {
return "", errors.New("check digit incorrect")
}
res = append(res, p)
}
return res[0] + res[1] + res[2] + res[3], nil
}
func chunkify(s string) string {
s = regexp.MustCompile("(.{7})").ReplaceAllString(s, "$1-")
s = strings.Trim(s, "-")
return s
}
func unchunkify(s string) string {
s = strings.Replace(s, "-", "", -1)
s = strings.Replace(s, " ", "", -1)
return s
}
func untypeoify(s string) string {
s = strings.Replace(s, "0", "O", -1)
s = strings.Replace(s, "1", "I", -1)
s = strings.Replace(s, "8", "B", -1)
return s
}

View File

@ -1,4 +0,0 @@
// Copyright (C) 2014 The Protocol Authors.
// Package protocol implements the Block Exchange Protocol.
package protocol

View File

@ -1,51 +0,0 @@
// Copyright (C) 2014 The Protocol Authors.
package protocol
import (
"errors"
)
const (
ecNoError int32 = iota
ecGeneric
ecNoSuchFile
ecInvalid
)
var (
ErrNoError error
ErrGeneric = errors.New("generic error")
ErrNoSuchFile = errors.New("no such file")
ErrInvalid = errors.New("file is invalid")
)
var lookupError = map[int32]error{
ecNoError: ErrNoError,
ecGeneric: ErrGeneric,
ecNoSuchFile: ErrNoSuchFile,
ecInvalid: ErrInvalid,
}
var lookupCode = map[error]int32{
ErrNoError: ecNoError,
ErrGeneric: ecGeneric,
ErrNoSuchFile: ecNoSuchFile,
ErrInvalid: ecInvalid,
}
func codeToError(errcode int32) error {
err, ok := lookupError[errcode]
if !ok {
return ErrGeneric
}
return err
}
func errorToCode(err error) int32 {
code, ok := lookupCode[err]
if !ok {
return ecGeneric
}
return code
}

View File

@ -1,70 +0,0 @@
// Copyright (C) 2015 The Protocol Authors.
// +build gofuzz
package protocol
import (
"bytes"
"encoding/binary"
"encoding/hex"
"fmt"
"reflect"
"sync"
)
func Fuzz(data []byte) int {
// Regenerate the length, or we'll most commonly exit quickly due to an
// unexpected eof which is unintestering.
if len(data) > 8 {
binary.BigEndian.PutUint32(data[4:], uint32(len(data))-8)
}
// Setup a rawConnection we'll use to parse the message.
c := rawConnection{
cr: &countingReader{Reader: bytes.NewReader(data)},
closed: make(chan struct{}),
pool: sync.Pool{
New: func() interface{} {
return make([]byte, BlockSize)
},
},
}
// Attempt to parse the message.
hdr, msg, err := c.readMessage()
if err != nil {
return 0
}
// If parsing worked, attempt to encode it again.
newBs, err := msg.AppendXDR(nil)
if err != nil {
panic("not encodable")
}
// Create an appriate header for the re-encoding.
newMsg := make([]byte, 8)
binary.BigEndian.PutUint32(newMsg, encodeHeader(hdr))
binary.BigEndian.PutUint32(newMsg[4:], uint32(len(newBs)))
newMsg = append(newMsg, newBs...)
// Use the rawConnection to parse the re-encoding.
c.cr = &countingReader{Reader: bytes.NewReader(newMsg)}
hdr2, msg2, err := c.readMessage()
if err != nil {
fmt.Println("Initial:\n" + hex.Dump(data))
fmt.Println("New:\n" + hex.Dump(newMsg))
panic("not parseable after re-encode: " + err.Error())
}
// Make sure the data is the same as it was before.
if hdr != hdr2 {
panic("headers differ")
}
if !reflect.DeepEqual(msg, msg2) {
panic("contents differ")
}
return 1
}

View File

@ -1,43 +0,0 @@
// Copyright (C) 2014 The Protocol Authors.
package protocol
import "github.com/calmh/xdr"
type header struct {
version int
msgID int
msgType int
compression bool
}
func (h header) encodeXDR(xw *xdr.Writer) (int, error) {
u := encodeHeader(h)
return xw.WriteUint32(u)
}
func (h *header) decodeXDR(xr *xdr.Reader) error {
u := xr.ReadUint32()
*h = decodeHeader(u)
return xr.Error()
}
func encodeHeader(h header) uint32 {
var isComp uint32
if h.compression {
isComp = 1 << 0 // the zeroth bit is the compression bit
}
return uint32(h.version&0xf)<<28 +
uint32(h.msgID&0xfff)<<16 +
uint32(h.msgType&0xff)<<8 +
isComp
}
func decodeHeader(u uint32) header {
return header{
version: int(u>>28) & 0xf,
msgID: int(u>>16) & 0xfff,
msgType: int(u>>8) & 0xff,
compression: u&1 == 1,
}
}

View File

@ -1,172 +0,0 @@
// Copyright (C) 2014 The Protocol Authors.
//go:generate -command genxdr go run ../../Godeps/_workspace/src/github.com/calmh/xdr/cmd/genxdr/main.go
//go:generate genxdr -o message_xdr.go message.go
package protocol
import (
"bytes"
"crypto/sha256"
"fmt"
)
var sha256OfEmptyBlock = sha256.Sum256(make([]byte, BlockSize))
type IndexMessage struct {
Folder string // max:256
Files []FileInfo // max:1000000
Flags uint32
Options []Option // max:64
}
type FileInfo struct {
Name string // max:8192
Flags uint32
Modified int64
Version Vector
LocalVersion int64
CachedSize int64 // noencode (cache only)
Blocks []BlockInfo // max:1000000
}
func (f FileInfo) String() string {
return fmt.Sprintf("File{Name:%q, Flags:0%o, Modified:%d, Version:%v, Size:%d, Blocks:%v}",
f.Name, f.Flags, f.Modified, f.Version, f.Size(), f.Blocks)
}
func (f FileInfo) Size() (bytes int64) {
if f.IsDeleted() || f.IsDirectory() {
return 128
}
if f.CachedSize > 0 {
return f.CachedSize
}
for _, b := range f.Blocks {
bytes += int64(b.Size)
}
f.CachedSize = bytes
return
}
func (f FileInfo) IsDeleted() bool {
return f.Flags&FlagDeleted != 0
}
func (f FileInfo) IsInvalid() bool {
return f.Flags&FlagInvalid != 0
}
func (f FileInfo) IsDirectory() bool {
return f.Flags&FlagDirectory != 0
}
func (f FileInfo) IsSymlink() bool {
return f.Flags&FlagSymlink != 0
}
func (f FileInfo) HasPermissionBits() bool {
return f.Flags&FlagNoPermBits == 0
}
// WinsConflict returns true if "f" is the one to choose when it is in
// conflict with "other".
func (f FileInfo) WinsConflict(other FileInfo) bool {
// If a modification is in conflict with a delete, we pick the
// modification.
if !f.IsDeleted() && other.IsDeleted() {
return true
}
if f.IsDeleted() && !other.IsDeleted() {
return false
}
// The one with the newer modification time wins.
if f.Modified > other.Modified {
return true
}
if f.Modified < other.Modified {
return false
}
// The modification times were equal. Use the device ID in the version
// vector as tie breaker.
return f.Version.Compare(other.Version) == ConcurrentGreater
}
type BlockInfo struct {
Offset int64 // noencode (cache only)
Size int32
Hash []byte // max:64
}
func (b BlockInfo) String() string {
return fmt.Sprintf("Block{%d/%d/%x}", b.Offset, b.Size, b.Hash)
}
// IsEmpty returns true if the block is a full block of zeroes.
func (b BlockInfo) IsEmpty() bool {
return b.Size == BlockSize && bytes.Equal(b.Hash, sha256OfEmptyBlock[:])
}
type RequestMessage struct {
Folder string // max:256
Name string // max:8192
Offset int64
Size int32
Hash []byte // max:64
Flags uint32
Options []Option // max:64
}
type ResponseMessage struct {
Data []byte
Code int32
}
type ClusterConfigMessage struct {
DeviceName string // max:64
ClientName string // max:64
ClientVersion string // max:64
Folders []Folder // max:1000000
Options []Option // max:64
}
func (o *ClusterConfigMessage) GetOption(key string) string {
for _, option := range o.Options {
if option.Key == key {
return option.Value
}
}
return ""
}
type Folder struct {
ID string // max:256
Devices []Device // max:1000000
Flags uint32
Options []Option // max:64
}
type Device struct {
ID []byte // max:32
Name string // max:64
Addresses []string // max:64,2083
Compression uint32
CertName string // max:64
MaxLocalVersion int64
Flags uint32
Options []Option // max:64
}
type Option struct {
Key string // max:64
Value string // max:1024
}
type CloseMessage struct {
Reason string // max:1024
Code int32
}
type EmptyMessage struct{}

Some files were not shown because too many files have changed in this diff Show More