mirror of
https://github.com/octoleo/syncthing.git
synced 2024-11-19 03:25:16 +00:00
621 lines
16 KiB
Go
621 lines
16 KiB
Go
package maxminddb
|
|
|
|
import (
|
|
"encoding/binary"
|
|
"math"
|
|
"math/big"
|
|
"reflect"
|
|
"sync"
|
|
)
|
|
|
|
type decoder struct {
|
|
buffer []byte
|
|
}
|
|
|
|
type dataType int
|
|
|
|
const (
|
|
_Extended dataType = iota
|
|
_Pointer
|
|
_String
|
|
_Float64
|
|
_Bytes
|
|
_Uint16
|
|
_Uint32
|
|
_Map
|
|
_Int32
|
|
_Uint64
|
|
_Uint128
|
|
_Slice
|
|
_Container
|
|
_Marker
|
|
_Bool
|
|
_Float32
|
|
)
|
|
|
|
func (d *decoder) decode(offset uint, result reflect.Value) (uint, error) {
|
|
typeNum, size, newOffset := d.decodeCtrlData(offset)
|
|
|
|
if typeNum != _Pointer && result.Kind() == reflect.Uintptr {
|
|
result.Set(reflect.ValueOf(uintptr(offset)))
|
|
return d.nextValueOffset(offset, 1), nil
|
|
}
|
|
return d.decodeFromType(typeNum, size, newOffset, result)
|
|
}
|
|
|
|
func (d *decoder) decodeCtrlData(offset uint) (dataType, uint, uint) {
|
|
newOffset := offset + 1
|
|
ctrlByte := d.buffer[offset]
|
|
|
|
typeNum := dataType(ctrlByte >> 5)
|
|
if typeNum == _Extended {
|
|
typeNum = dataType(d.buffer[newOffset] + 7)
|
|
newOffset++
|
|
}
|
|
|
|
var size uint
|
|
size, newOffset = d.sizeFromCtrlByte(ctrlByte, newOffset, typeNum)
|
|
return typeNum, size, newOffset
|
|
}
|
|
|
|
func (d *decoder) sizeFromCtrlByte(ctrlByte byte, offset uint, typeNum dataType) (uint, uint) {
|
|
size := uint(ctrlByte & 0x1f)
|
|
if typeNum == _Extended {
|
|
return size, offset
|
|
}
|
|
|
|
var bytesToRead uint
|
|
if size > 28 {
|
|
bytesToRead = size - 28
|
|
}
|
|
|
|
newOffset := offset + bytesToRead
|
|
sizeBytes := d.buffer[offset:newOffset]
|
|
|
|
switch {
|
|
case size == 29:
|
|
size = 29 + uint(sizeBytes[0])
|
|
case size == 30:
|
|
size = 285 + uint(uintFromBytes(0, sizeBytes))
|
|
case size > 30:
|
|
size = uint(uintFromBytes(0, sizeBytes)) + 65821
|
|
}
|
|
return size, newOffset
|
|
}
|
|
|
|
func (d *decoder) decodeFromType(dtype dataType, size uint, offset uint, result reflect.Value) (uint, error) {
|
|
result = d.indirect(result)
|
|
|
|
switch dtype {
|
|
case _Bool:
|
|
return d.unmarshalBool(size, offset, result)
|
|
case _Bytes:
|
|
return d.unmarshalBytes(size, offset, result)
|
|
case _Float32:
|
|
return d.unmarshalFloat32(size, offset, result)
|
|
case _Float64:
|
|
return d.unmarshalFloat64(size, offset, result)
|
|
case _Int32:
|
|
return d.unmarshalInt32(size, offset, result)
|
|
case _Map:
|
|
return d.unmarshalMap(size, offset, result)
|
|
case _Pointer:
|
|
return d.unmarshalPointer(size, offset, result)
|
|
case _Slice:
|
|
return d.unmarshalSlice(size, offset, result)
|
|
case _String:
|
|
return d.unmarshalString(size, offset, result)
|
|
case _Uint16:
|
|
return d.unmarshalUint(size, offset, result, 16)
|
|
case _Uint32:
|
|
return d.unmarshalUint(size, offset, result, 32)
|
|
case _Uint64:
|
|
return d.unmarshalUint(size, offset, result, 64)
|
|
case _Uint128:
|
|
return d.unmarshalUint128(size, offset, result)
|
|
default:
|
|
return 0, newInvalidDatabaseError("unknown type: %d", dtype)
|
|
}
|
|
}
|
|
|
|
func (d *decoder) unmarshalBool(size uint, offset uint, result reflect.Value) (uint, error) {
|
|
if size > 1 {
|
|
return 0, newInvalidDatabaseError("the MaxMind DB file's data section contains bad data (bool size of %v)", size)
|
|
}
|
|
value, newOffset, err := d.decodeBool(size, offset)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
switch result.Kind() {
|
|
case reflect.Bool:
|
|
result.SetBool(value)
|
|
return newOffset, nil
|
|
case reflect.Interface:
|
|
if result.NumMethod() == 0 {
|
|
result.Set(reflect.ValueOf(value))
|
|
return newOffset, nil
|
|
}
|
|
}
|
|
return newOffset, newUnmarshalTypeError(value, result.Type())
|
|
}
|
|
|
|
// indirect follows pointers and create values as necessary. This is
|
|
// heavily based on encoding/json as my original version had a subtle
|
|
// bug. This method should be considered to be licensed under
|
|
// https://golang.org/LICENSE
|
|
func (d *decoder) indirect(result reflect.Value) reflect.Value {
|
|
for {
|
|
// Load value from interface, but only if the result will be
|
|
// usefully addressable.
|
|
if result.Kind() == reflect.Interface && !result.IsNil() {
|
|
e := result.Elem()
|
|
if e.Kind() == reflect.Ptr && !e.IsNil() {
|
|
result = e
|
|
continue
|
|
}
|
|
}
|
|
|
|
if result.Kind() != reflect.Ptr {
|
|
break
|
|
}
|
|
|
|
if result.IsNil() {
|
|
result.Set(reflect.New(result.Type().Elem()))
|
|
}
|
|
result = result.Elem()
|
|
}
|
|
return result
|
|
}
|
|
|
|
func (d *decoder) unmarshalBytes(size uint, offset uint, result reflect.Value) (uint, error) {
|
|
value, newOffset, err := d.decodeBytes(size, offset)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
switch result.Kind() {
|
|
case reflect.Slice:
|
|
result.SetBytes(value)
|
|
return newOffset, nil
|
|
case reflect.Interface:
|
|
if result.NumMethod() == 0 {
|
|
result.Set(reflect.ValueOf(value))
|
|
return newOffset, nil
|
|
}
|
|
}
|
|
return newOffset, newUnmarshalTypeError(value, result.Type())
|
|
}
|
|
|
|
func (d *decoder) unmarshalFloat32(size uint, offset uint, result reflect.Value) (uint, error) {
|
|
if size != 4 {
|
|
return 0, newInvalidDatabaseError("the MaxMind DB file's data section contains bad data (float32 size of %v)", size)
|
|
}
|
|
value, newOffset, err := d.decodeFloat32(size, offset)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
switch result.Kind() {
|
|
case reflect.Float32, reflect.Float64:
|
|
result.SetFloat(float64(value))
|
|
return newOffset, nil
|
|
case reflect.Interface:
|
|
if result.NumMethod() == 0 {
|
|
result.Set(reflect.ValueOf(value))
|
|
return newOffset, nil
|
|
}
|
|
}
|
|
return newOffset, newUnmarshalTypeError(value, result.Type())
|
|
}
|
|
|
|
func (d *decoder) unmarshalFloat64(size uint, offset uint, result reflect.Value) (uint, error) {
|
|
|
|
if size != 8 {
|
|
return 0, newInvalidDatabaseError("the MaxMind DB file's data section contains bad data (float 64 size of %v)", size)
|
|
}
|
|
value, newOffset, err := d.decodeFloat64(size, offset)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
switch result.Kind() {
|
|
case reflect.Float32, reflect.Float64:
|
|
if result.OverflowFloat(value) {
|
|
return 0, newUnmarshalTypeError(value, result.Type())
|
|
}
|
|
result.SetFloat(value)
|
|
return newOffset, nil
|
|
case reflect.Interface:
|
|
if result.NumMethod() == 0 {
|
|
result.Set(reflect.ValueOf(value))
|
|
return newOffset, nil
|
|
}
|
|
}
|
|
return newOffset, newUnmarshalTypeError(value, result.Type())
|
|
}
|
|
|
|
func (d *decoder) unmarshalInt32(size uint, offset uint, result reflect.Value) (uint, error) {
|
|
if size > 4 {
|
|
return 0, newInvalidDatabaseError("the MaxMind DB file's data section contains bad data (int32 size of %v)", size)
|
|
}
|
|
value, newOffset, err := d.decodeInt(size, offset)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
switch result.Kind() {
|
|
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
|
|
n := int64(value)
|
|
if !result.OverflowInt(n) {
|
|
result.SetInt(n)
|
|
return newOffset, nil
|
|
}
|
|
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
|
|
n := uint64(value)
|
|
if !result.OverflowUint(n) {
|
|
result.SetUint(n)
|
|
return newOffset, nil
|
|
}
|
|
case reflect.Interface:
|
|
if result.NumMethod() == 0 {
|
|
result.Set(reflect.ValueOf(value))
|
|
return newOffset, nil
|
|
}
|
|
}
|
|
return newOffset, newUnmarshalTypeError(value, result.Type())
|
|
}
|
|
|
|
func (d *decoder) unmarshalMap(size uint, offset uint, result reflect.Value) (uint, error) {
|
|
result = d.indirect(result)
|
|
switch result.Kind() {
|
|
default:
|
|
return 0, newUnmarshalTypeError("map", result.Type())
|
|
case reflect.Struct:
|
|
return d.decodeStruct(size, offset, result)
|
|
case reflect.Map:
|
|
return d.decodeMap(size, offset, result)
|
|
case reflect.Interface:
|
|
if result.NumMethod() == 0 {
|
|
rv := reflect.ValueOf(make(map[string]interface{}, size))
|
|
newOffset, err := d.decodeMap(size, offset, rv)
|
|
result.Set(rv)
|
|
return newOffset, err
|
|
}
|
|
return 0, newUnmarshalTypeError("map", result.Type())
|
|
}
|
|
}
|
|
|
|
func (d *decoder) unmarshalPointer(size uint, offset uint, result reflect.Value) (uint, error) {
|
|
pointer, newOffset := d.decodePointer(size, offset)
|
|
_, err := d.decode(pointer, result)
|
|
return newOffset, err
|
|
}
|
|
|
|
func (d *decoder) unmarshalSlice(size uint, offset uint, result reflect.Value) (uint, error) {
|
|
switch result.Kind() {
|
|
case reflect.Slice:
|
|
return d.decodeSlice(size, offset, result)
|
|
case reflect.Interface:
|
|
if result.NumMethod() == 0 {
|
|
a := []interface{}{}
|
|
rv := reflect.ValueOf(&a).Elem()
|
|
newOffset, err := d.decodeSlice(size, offset, rv)
|
|
result.Set(rv)
|
|
return newOffset, err
|
|
}
|
|
}
|
|
return 0, newUnmarshalTypeError("array", result.Type())
|
|
}
|
|
|
|
func (d *decoder) unmarshalString(size uint, offset uint, result reflect.Value) (uint, error) {
|
|
value, newOffset, err := d.decodeString(size, offset)
|
|
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
switch result.Kind() {
|
|
case reflect.String:
|
|
result.SetString(value)
|
|
return newOffset, nil
|
|
case reflect.Interface:
|
|
if result.NumMethod() == 0 {
|
|
result.Set(reflect.ValueOf(value))
|
|
return newOffset, nil
|
|
}
|
|
}
|
|
return newOffset, newUnmarshalTypeError(value, result.Type())
|
|
|
|
}
|
|
|
|
func (d *decoder) unmarshalUint(size uint, offset uint, result reflect.Value, uintType uint) (uint, error) {
|
|
if size > uintType/8 {
|
|
return 0, newInvalidDatabaseError("the MaxMind DB file's data section contains bad data (uint%v size of %v)", uintType, size)
|
|
}
|
|
|
|
value, newOffset, err := d.decodeUint(size, offset)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
switch result.Kind() {
|
|
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
|
|
n := int64(value)
|
|
if !result.OverflowInt(n) {
|
|
result.SetInt(n)
|
|
return newOffset, nil
|
|
}
|
|
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
|
|
if !result.OverflowUint(value) {
|
|
result.SetUint(value)
|
|
return newOffset, nil
|
|
}
|
|
case reflect.Interface:
|
|
if result.NumMethod() == 0 {
|
|
result.Set(reflect.ValueOf(value))
|
|
return newOffset, nil
|
|
}
|
|
}
|
|
return newOffset, newUnmarshalTypeError(value, result.Type())
|
|
}
|
|
|
|
func (d *decoder) unmarshalUint128(size uint, offset uint, result reflect.Value) (uint, error) {
|
|
if size > 16 {
|
|
return 0, newInvalidDatabaseError("the MaxMind DB file's data section contains bad data (uint128 size of %v)", size)
|
|
}
|
|
value, newOffset, err := d.decodeUint128(size, offset)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
switch result.Kind() {
|
|
case reflect.Struct:
|
|
result.Set(reflect.ValueOf(*value))
|
|
return newOffset, nil
|
|
case reflect.Interface:
|
|
if result.NumMethod() == 0 {
|
|
result.Set(reflect.ValueOf(value))
|
|
return newOffset, nil
|
|
}
|
|
}
|
|
return newOffset, newUnmarshalTypeError(value, result.Type())
|
|
}
|
|
|
|
func (d *decoder) decodeBool(size uint, offset uint) (bool, uint, error) {
|
|
return size != 0, offset, nil
|
|
}
|
|
|
|
func (d *decoder) decodeBytes(size uint, offset uint) ([]byte, uint, error) {
|
|
newOffset := offset + size
|
|
bytes := make([]byte, size)
|
|
copy(bytes, d.buffer[offset:newOffset])
|
|
return bytes, newOffset, nil
|
|
}
|
|
|
|
func (d *decoder) decodeFloat64(size uint, offset uint) (float64, uint, error) {
|
|
newOffset := offset + size
|
|
bits := binary.BigEndian.Uint64(d.buffer[offset:newOffset])
|
|
return math.Float64frombits(bits), newOffset, nil
|
|
}
|
|
|
|
func (d *decoder) decodeFloat32(size uint, offset uint) (float32, uint, error) {
|
|
newOffset := offset + size
|
|
bits := binary.BigEndian.Uint32(d.buffer[offset:newOffset])
|
|
return math.Float32frombits(bits), newOffset, nil
|
|
}
|
|
|
|
func (d *decoder) decodeInt(size uint, offset uint) (int, uint, error) {
|
|
newOffset := offset + size
|
|
var val int32
|
|
for _, b := range d.buffer[offset:newOffset] {
|
|
val = (val << 8) | int32(b)
|
|
}
|
|
return int(val), newOffset, nil
|
|
}
|
|
|
|
func (d *decoder) decodeMap(size uint, offset uint, result reflect.Value) (uint, error) {
|
|
if result.IsNil() {
|
|
result.Set(reflect.MakeMap(result.Type()))
|
|
}
|
|
|
|
for i := uint(0); i < size; i++ {
|
|
var key []byte
|
|
var err error
|
|
key, offset, err = d.decodeKey(offset)
|
|
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
value := reflect.New(result.Type().Elem())
|
|
offset, err = d.decode(offset, value)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
result.SetMapIndex(reflect.ValueOf(string(key)), value.Elem())
|
|
}
|
|
return offset, nil
|
|
}
|
|
|
|
func (d *decoder) decodePointer(size uint, offset uint) (uint, uint) {
|
|
pointerSize := ((size >> 3) & 0x3) + 1
|
|
newOffset := offset + pointerSize
|
|
pointerBytes := d.buffer[offset:newOffset]
|
|
var prefix uint64
|
|
if pointerSize == 4 {
|
|
prefix = 0
|
|
} else {
|
|
prefix = uint64(size & 0x7)
|
|
}
|
|
unpacked := uint(uintFromBytes(prefix, pointerBytes))
|
|
|
|
var pointerValueOffset uint
|
|
switch pointerSize {
|
|
case 1:
|
|
pointerValueOffset = 0
|
|
case 2:
|
|
pointerValueOffset = 2048
|
|
case 3:
|
|
pointerValueOffset = 526336
|
|
case 4:
|
|
pointerValueOffset = 0
|
|
}
|
|
|
|
pointer := unpacked + pointerValueOffset
|
|
|
|
return pointer, newOffset
|
|
}
|
|
|
|
func (d *decoder) decodeSlice(size uint, offset uint, result reflect.Value) (uint, error) {
|
|
result.Set(reflect.MakeSlice(result.Type(), int(size), int(size)))
|
|
for i := 0; i < int(size); i++ {
|
|
var err error
|
|
offset, err = d.decode(offset, result.Index(i))
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
}
|
|
return offset, nil
|
|
}
|
|
|
|
func (d *decoder) decodeString(size uint, offset uint) (string, uint, error) {
|
|
newOffset := offset + size
|
|
return string(d.buffer[offset:newOffset]), newOffset, nil
|
|
}
|
|
|
|
type fieldsType struct {
|
|
namedFields map[string]int
|
|
anonymousFields []int
|
|
}
|
|
|
|
var (
|
|
fieldMap = map[reflect.Type]*fieldsType{}
|
|
fieldMapMu sync.RWMutex
|
|
)
|
|
|
|
func (d *decoder) decodeStruct(size uint, offset uint, result reflect.Value) (uint, error) {
|
|
resultType := result.Type()
|
|
|
|
fieldMapMu.RLock()
|
|
fields, ok := fieldMap[resultType]
|
|
fieldMapMu.RUnlock()
|
|
if !ok {
|
|
numFields := resultType.NumField()
|
|
namedFields := make(map[string]int, numFields)
|
|
var anonymous []int
|
|
for i := 0; i < numFields; i++ {
|
|
field := resultType.Field(i)
|
|
|
|
fieldName := field.Name
|
|
if tag := field.Tag.Get("maxminddb"); tag != "" {
|
|
if tag == "-" {
|
|
continue
|
|
}
|
|
fieldName = tag
|
|
}
|
|
if field.Anonymous {
|
|
anonymous = append(anonymous, i)
|
|
continue
|
|
}
|
|
namedFields[fieldName] = i
|
|
}
|
|
fieldMapMu.Lock()
|
|
fields = &fieldsType{namedFields, anonymous}
|
|
fieldMap[resultType] = fields
|
|
fieldMapMu.Unlock()
|
|
}
|
|
|
|
// This fills in embedded structs
|
|
for i := range fields.anonymousFields {
|
|
_, err := d.unmarshalMap(size, offset, result.Field(i))
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
}
|
|
|
|
// This handles named fields
|
|
for i := uint(0); i < size; i++ {
|
|
var (
|
|
err error
|
|
key []byte
|
|
)
|
|
key, offset, err = d.decodeKey(offset)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
// The string() does not create a copy due to this compiler
|
|
// optimization: https://github.com/golang/go/issues/3512
|
|
j, ok := fields.namedFields[string(key)]
|
|
if !ok {
|
|
offset = d.nextValueOffset(offset, 1)
|
|
continue
|
|
}
|
|
|
|
offset, err = d.decode(offset, result.Field(j))
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
}
|
|
return offset, nil
|
|
}
|
|
|
|
func (d *decoder) decodeUint(size uint, offset uint) (uint64, uint, error) {
|
|
newOffset := offset + size
|
|
val := uintFromBytes(0, d.buffer[offset:newOffset])
|
|
|
|
return val, newOffset, nil
|
|
}
|
|
|
|
func (d *decoder) decodeUint128(size uint, offset uint) (*big.Int, uint, error) {
|
|
newOffset := offset + size
|
|
val := new(big.Int)
|
|
val.SetBytes(d.buffer[offset:newOffset])
|
|
|
|
return val, newOffset, nil
|
|
}
|
|
|
|
func uintFromBytes(prefix uint64, uintBytes []byte) uint64 {
|
|
val := prefix
|
|
for _, b := range uintBytes {
|
|
val = (val << 8) | uint64(b)
|
|
}
|
|
return val
|
|
}
|
|
|
|
// decodeKey decodes a map key into []byte slice. We use a []byte so that we
|
|
// can take advantage of https://github.com/golang/go/issues/3512 to avoid
|
|
// copying the bytes when decoding a struct. Previously, we achieved this by
|
|
// using unsafe.
|
|
func (d *decoder) decodeKey(offset uint) ([]byte, uint, error) {
|
|
typeNum, size, dataOffset := d.decodeCtrlData(offset)
|
|
if typeNum == _Pointer {
|
|
pointer, ptrOffset := d.decodePointer(size, dataOffset)
|
|
key, _, err := d.decodeKey(pointer)
|
|
return key, ptrOffset, err
|
|
}
|
|
if typeNum != _String {
|
|
return nil, 0, newInvalidDatabaseError("unexpected type when decoding string: %v", typeNum)
|
|
}
|
|
newOffset := dataOffset + size
|
|
return d.buffer[dataOffset:newOffset], newOffset, nil
|
|
}
|
|
|
|
// This function is used to skip ahead to the next value without decoding
|
|
// the one at the offset passed in. The size bits have different meanings for
|
|
// different data types
|
|
func (d *decoder) nextValueOffset(offset uint, numberToSkip uint) uint {
|
|
if numberToSkip == 0 {
|
|
return offset
|
|
}
|
|
typeNum, size, offset := d.decodeCtrlData(offset)
|
|
switch typeNum {
|
|
case _Pointer:
|
|
_, offset = d.decodePointer(size, offset)
|
|
case _Map:
|
|
numberToSkip += 2 * size
|
|
case _Slice:
|
|
numberToSkip += size
|
|
case _Bool:
|
|
default:
|
|
offset += size
|
|
}
|
|
return d.nextValueOffset(offset, numberToSkip-1)
|
|
}
|