mirror of
https://github.com/octoleo/syncthing.git
synced 2024-12-22 19:08:58 +00:00
Dependency update, new golang.org/x package names
This commit is contained in:
parent
5dbaf6ceb0
commit
126c4e9a06
40
Godeps/Godeps.json
generated
40
Godeps/Godeps.json
generated
@ -5,26 +5,6 @@
|
||||
"./cmd/..."
|
||||
],
|
||||
"Deps": [
|
||||
{
|
||||
"ImportPath": "code.google.com/p/go.crypto/bcrypt",
|
||||
"Comment": "null-216",
|
||||
"Rev": "41cd4647fccc72b0b79ef1bd1fe6735e718257cd"
|
||||
},
|
||||
{
|
||||
"ImportPath": "code.google.com/p/go.crypto/blowfish",
|
||||
"Comment": "null-216",
|
||||
"Rev": "41cd4647fccc72b0b79ef1bd1fe6735e718257cd"
|
||||
},
|
||||
{
|
||||
"ImportPath": "code.google.com/p/go.text/transform",
|
||||
"Comment": "null-90",
|
||||
"Rev": "d65bffbc88a153d23a6d2a864531e6e7c2cde59b"
|
||||
},
|
||||
{
|
||||
"ImportPath": "code.google.com/p/go.text/unicode/norm",
|
||||
"Comment": "null-90",
|
||||
"Rev": "d65bffbc88a153d23a6d2a864531e6e7c2cde59b"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/AudriusButkevicius/lfu-go",
|
||||
"Rev": "164bcecceb92fd6037f4d18a8d97b495ec6ef669"
|
||||
@ -68,6 +48,26 @@
|
||||
{
|
||||
"ImportPath": "github.com/vitrun/qart/qr",
|
||||
"Rev": "ccb109cf25f0cd24474da73b9fee4e7a3e8a8ce0"
|
||||
},
|
||||
{
|
||||
"ImportPath": "golang.org/x/crypto/bcrypt",
|
||||
"Comment": "null-236",
|
||||
"Rev": "69e2a90ed92d03812364aeb947b7068dc42e561e"
|
||||
},
|
||||
{
|
||||
"ImportPath": "golang.org/x/crypto/blowfish",
|
||||
"Comment": "null-236",
|
||||
"Rev": "69e2a90ed92d03812364aeb947b7068dc42e561e"
|
||||
},
|
||||
{
|
||||
"ImportPath": "golang.org/x/text/transform",
|
||||
"Comment": "null-112",
|
||||
"Rev": "2f707e0ad64637ca1318279be7201f5ed19c4050"
|
||||
},
|
||||
{
|
||||
"ImportPath": "golang.org/x/text/unicode/norm",
|
||||
"Comment": "null-112",
|
||||
"Rev": "2f707e0ad64637ca1318279be7201f5ed19c4050"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
45
Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/maketesttables.go
generated
vendored
45
Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/maketesttables.go
generated
vendored
@ -1,45 +0,0 @@
|
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build ignore
|
||||
|
||||
// Generate test data for trie code.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
)
|
||||
|
||||
func main() {
|
||||
printTestTables()
|
||||
}
|
||||
|
||||
// We take the smallest, largest and an arbitrary value for each
|
||||
// of the UTF-8 sequence lengths.
|
||||
var testRunes = []rune{
|
||||
0x01, 0x0C, 0x7F, // 1-byte sequences
|
||||
0x80, 0x100, 0x7FF, // 2-byte sequences
|
||||
0x800, 0x999, 0xFFFF, // 3-byte sequences
|
||||
0x10000, 0x10101, 0x10FFFF, // 4-byte sequences
|
||||
0x200, 0x201, 0x202, 0x210, 0x215, // five entries in one sparse block
|
||||
}
|
||||
|
||||
const fileHeader = `// Generated by running
|
||||
// maketesttables
|
||||
// DO NOT EDIT
|
||||
|
||||
package norm
|
||||
|
||||
`
|
||||
|
||||
func printTestTables() {
|
||||
fmt.Print(fileHeader)
|
||||
fmt.Printf("var testRunes = %#v\n\n", testRunes)
|
||||
t := newNode()
|
||||
for i, r := range testRunes {
|
||||
t.insert(r, uint16(i))
|
||||
}
|
||||
t.printTables("testdata")
|
||||
}
|
6989
Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/tables.go
generated
vendored
6989
Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/tables.go
generated
vendored
File diff suppressed because it is too large
Load Diff
232
Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/trie.go
generated
vendored
232
Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/trie.go
generated
vendored
@ -1,232 +0,0 @@
|
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package norm
|
||||
|
||||
type valueRange struct {
|
||||
value uint16 // header: value:stride
|
||||
lo, hi byte // header: lo:n
|
||||
}
|
||||
|
||||
type trie struct {
|
||||
index []uint8
|
||||
values []uint16
|
||||
sparse []valueRange
|
||||
sparseOffset []uint16
|
||||
cutoff uint8 // indices >= cutoff are sparse
|
||||
}
|
||||
|
||||
// lookupValue determines the type of block n and looks up the value for b.
|
||||
// For n < t.cutoff, the block is a simple lookup table. Otherwise, the block
|
||||
// is a list of ranges with an accompanying value. Given a matching range r,
|
||||
// the value for b is by r.value + (b - r.lo) * stride.
|
||||
func (t *trie) lookupValue(n uint8, b byte) uint16 {
|
||||
if n < t.cutoff {
|
||||
return t.values[uint16(n)<<6+uint16(b)]
|
||||
}
|
||||
offset := t.sparseOffset[n-t.cutoff]
|
||||
header := t.sparse[offset]
|
||||
lo := offset + 1
|
||||
hi := lo + uint16(header.lo)
|
||||
for lo < hi {
|
||||
m := lo + (hi-lo)/2
|
||||
r := t.sparse[m]
|
||||
if r.lo <= b && b <= r.hi {
|
||||
return r.value + uint16(b-r.lo)*header.value
|
||||
}
|
||||
if b < r.lo {
|
||||
hi = m
|
||||
} else {
|
||||
lo = m + 1
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
const (
|
||||
t1 = 0x00 // 0000 0000
|
||||
tx = 0x80 // 1000 0000
|
||||
t2 = 0xC0 // 1100 0000
|
||||
t3 = 0xE0 // 1110 0000
|
||||
t4 = 0xF0 // 1111 0000
|
||||
t5 = 0xF8 // 1111 1000
|
||||
t6 = 0xFC // 1111 1100
|
||||
te = 0xFE // 1111 1110
|
||||
)
|
||||
|
||||
// lookup returns the trie value for the first UTF-8 encoding in s and
|
||||
// the width in bytes of this encoding. The size will be 0 if s does not
|
||||
// hold enough bytes to complete the encoding. len(s) must be greater than 0.
|
||||
func (t *trie) lookup(s []byte) (v uint16, sz int) {
|
||||
c0 := s[0]
|
||||
switch {
|
||||
case c0 < tx:
|
||||
return t.values[c0], 1
|
||||
case c0 < t2:
|
||||
return 0, 1
|
||||
case c0 < t3:
|
||||
if len(s) < 2 {
|
||||
return 0, 0
|
||||
}
|
||||
i := t.index[c0]
|
||||
c1 := s[1]
|
||||
if c1 < tx || t2 <= c1 {
|
||||
return 0, 1
|
||||
}
|
||||
return t.lookupValue(i, c1), 2
|
||||
case c0 < t4:
|
||||
if len(s) < 3 {
|
||||
return 0, 0
|
||||
}
|
||||
i := t.index[c0]
|
||||
c1 := s[1]
|
||||
if c1 < tx || t2 <= c1 {
|
||||
return 0, 1
|
||||
}
|
||||
o := uint16(i)<<6 + uint16(c1)
|
||||
i = t.index[o]
|
||||
c2 := s[2]
|
||||
if c2 < tx || t2 <= c2 {
|
||||
return 0, 2
|
||||
}
|
||||
return t.lookupValue(i, c2), 3
|
||||
case c0 < t5:
|
||||
if len(s) < 4 {
|
||||
return 0, 0
|
||||
}
|
||||
i := t.index[c0]
|
||||
c1 := s[1]
|
||||
if c1 < tx || t2 <= c1 {
|
||||
return 0, 1
|
||||
}
|
||||
o := uint16(i)<<6 + uint16(c1)
|
||||
i = t.index[o]
|
||||
c2 := s[2]
|
||||
if c2 < tx || t2 <= c2 {
|
||||
return 0, 2
|
||||
}
|
||||
o = uint16(i)<<6 + uint16(c2)
|
||||
i = t.index[o]
|
||||
c3 := s[3]
|
||||
if c3 < tx || t2 <= c3 {
|
||||
return 0, 3
|
||||
}
|
||||
return t.lookupValue(i, c3), 4
|
||||
}
|
||||
// Illegal rune
|
||||
return 0, 1
|
||||
}
|
||||
|
||||
// lookupString returns the trie value for the first UTF-8 encoding in s and
|
||||
// the width in bytes of this encoding. The size will be 0 if s does not
|
||||
// hold enough bytes to complete the encoding. len(s) must be greater than 0.
|
||||
func (t *trie) lookupString(s string) (v uint16, sz int) {
|
||||
c0 := s[0]
|
||||
switch {
|
||||
case c0 < tx:
|
||||
return t.values[c0], 1
|
||||
case c0 < t2:
|
||||
return 0, 1
|
||||
case c0 < t3:
|
||||
if len(s) < 2 {
|
||||
return 0, 0
|
||||
}
|
||||
i := t.index[c0]
|
||||
c1 := s[1]
|
||||
if c1 < tx || t2 <= c1 {
|
||||
return 0, 1
|
||||
}
|
||||
return t.lookupValue(i, c1), 2
|
||||
case c0 < t4:
|
||||
if len(s) < 3 {
|
||||
return 0, 0
|
||||
}
|
||||
i := t.index[c0]
|
||||
c1 := s[1]
|
||||
if c1 < tx || t2 <= c1 {
|
||||
return 0, 1
|
||||
}
|
||||
o := uint16(i)<<6 + uint16(c1)
|
||||
i = t.index[o]
|
||||
c2 := s[2]
|
||||
if c2 < tx || t2 <= c2 {
|
||||
return 0, 2
|
||||
}
|
||||
return t.lookupValue(i, c2), 3
|
||||
case c0 < t5:
|
||||
if len(s) < 4 {
|
||||
return 0, 0
|
||||
}
|
||||
i := t.index[c0]
|
||||
c1 := s[1]
|
||||
if c1 < tx || t2 <= c1 {
|
||||
return 0, 1
|
||||
}
|
||||
o := uint16(i)<<6 + uint16(c1)
|
||||
i = t.index[o]
|
||||
c2 := s[2]
|
||||
if c2 < tx || t2 <= c2 {
|
||||
return 0, 2
|
||||
}
|
||||
o = uint16(i)<<6 + uint16(c2)
|
||||
i = t.index[o]
|
||||
c3 := s[3]
|
||||
if c3 < tx || t2 <= c3 {
|
||||
return 0, 3
|
||||
}
|
||||
return t.lookupValue(i, c3), 4
|
||||
}
|
||||
// Illegal rune
|
||||
return 0, 1
|
||||
}
|
||||
|
||||
// lookupUnsafe returns the trie value for the first UTF-8 encoding in s.
|
||||
// s must hold a full encoding.
|
||||
func (t *trie) lookupUnsafe(s []byte) uint16 {
|
||||
c0 := s[0]
|
||||
if c0 < tx {
|
||||
return t.values[c0]
|
||||
}
|
||||
if c0 < t2 {
|
||||
return 0
|
||||
}
|
||||
i := t.index[c0]
|
||||
if c0 < t3 {
|
||||
return t.lookupValue(i, s[1])
|
||||
}
|
||||
i = t.index[uint16(i)<<6+uint16(s[1])]
|
||||
if c0 < t4 {
|
||||
return t.lookupValue(i, s[2])
|
||||
}
|
||||
i = t.index[uint16(i)<<6+uint16(s[2])]
|
||||
if c0 < t5 {
|
||||
return t.lookupValue(i, s[3])
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// lookupStringUnsafe returns the trie value for the first UTF-8 encoding in s.
|
||||
// s must hold a full encoding.
|
||||
func (t *trie) lookupStringUnsafe(s string) uint16 {
|
||||
c0 := s[0]
|
||||
if c0 < tx {
|
||||
return t.values[c0]
|
||||
}
|
||||
if c0 < t2 {
|
||||
return 0
|
||||
}
|
||||
i := t.index[c0]
|
||||
if c0 < t3 {
|
||||
return t.lookupValue(i, s[1])
|
||||
}
|
||||
i = t.index[uint16(i)<<6+uint16(s[1])]
|
||||
if c0 < t4 {
|
||||
return t.lookupValue(i, s[2])
|
||||
}
|
||||
i = t.index[uint16(i)<<6+uint16(s[2])]
|
||||
if c0 < t5 {
|
||||
return t.lookupValue(i, s[3])
|
||||
}
|
||||
return 0
|
||||
}
|
152
Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/trie_test.go
generated
vendored
152
Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/trie_test.go
generated
vendored
@ -1,152 +0,0 @@
|
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package norm
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// Test data is located in triedata_test.go; generated by maketesttables.
|
||||
var testdata = testdataTrie
|
||||
|
||||
type rangeTest struct {
|
||||
block uint8
|
||||
lookup byte
|
||||
result uint16
|
||||
table []valueRange
|
||||
offsets []uint16
|
||||
}
|
||||
|
||||
var range1Off = []uint16{0, 2}
|
||||
var range1 = []valueRange{
|
||||
{0, 1, 0},
|
||||
{1, 0x80, 0x80},
|
||||
{0, 2, 0},
|
||||
{1, 0x80, 0x80},
|
||||
{9, 0xff, 0xff},
|
||||
}
|
||||
|
||||
var rangeTests = []rangeTest{
|
||||
{10, 0x80, 1, range1, range1Off},
|
||||
{10, 0x00, 0, range1, range1Off},
|
||||
{11, 0x80, 1, range1, range1Off},
|
||||
{11, 0xff, 9, range1, range1Off},
|
||||
{11, 0x00, 0, range1, range1Off},
|
||||
}
|
||||
|
||||
func TestLookupSparse(t *testing.T) {
|
||||
for i, test := range rangeTests {
|
||||
n := trie{sparse: test.table, sparseOffset: test.offsets, cutoff: 10}
|
||||
v := n.lookupValue(test.block, test.lookup)
|
||||
if v != test.result {
|
||||
t.Errorf("LookupSparse:%d: found %X; want %X", i, v, test.result)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test cases for illegal runes.
|
||||
type trietest struct {
|
||||
size int
|
||||
bytes []byte
|
||||
}
|
||||
|
||||
var tests = []trietest{
|
||||
// illegal runes
|
||||
{1, []byte{0x80}},
|
||||
{1, []byte{0xFF}},
|
||||
{1, []byte{t2, tx - 1}},
|
||||
{1, []byte{t2, t2}},
|
||||
{2, []byte{t3, tx, tx - 1}},
|
||||
{2, []byte{t3, tx, t2}},
|
||||
{1, []byte{t3, tx - 1, tx}},
|
||||
{3, []byte{t4, tx, tx, tx - 1}},
|
||||
{3, []byte{t4, tx, tx, t2}},
|
||||
{1, []byte{t4, t2, tx, tx - 1}},
|
||||
{2, []byte{t4, tx, t2, tx - 1}},
|
||||
|
||||
// short runes
|
||||
{0, []byte{t2}},
|
||||
{0, []byte{t3, tx}},
|
||||
{0, []byte{t4, tx, tx}},
|
||||
|
||||
// we only support UTF-8 up to utf8.UTFMax bytes (4 bytes)
|
||||
{1, []byte{t5, tx, tx, tx, tx}},
|
||||
{1, []byte{t6, tx, tx, tx, tx, tx}},
|
||||
}
|
||||
|
||||
func mkUTF8(r rune) ([]byte, int) {
|
||||
var b [utf8.UTFMax]byte
|
||||
sz := utf8.EncodeRune(b[:], r)
|
||||
return b[:sz], sz
|
||||
}
|
||||
|
||||
func TestLookup(t *testing.T) {
|
||||
for i, tt := range testRunes {
|
||||
b, szg := mkUTF8(tt)
|
||||
v, szt := testdata.lookup(b)
|
||||
if int(v) != i {
|
||||
t.Errorf("lookup(%U): found value %#x, expected %#x", tt, v, i)
|
||||
}
|
||||
if szt != szg {
|
||||
t.Errorf("lookup(%U): found size %d, expected %d", tt, szt, szg)
|
||||
}
|
||||
}
|
||||
for i, tt := range tests {
|
||||
v, sz := testdata.lookup(tt.bytes)
|
||||
if v != 0 {
|
||||
t.Errorf("lookup of illegal rune, case %d: found value %#x, expected 0", i, v)
|
||||
}
|
||||
if sz != tt.size {
|
||||
t.Errorf("lookup of illegal rune, case %d: found size %d, expected %d", i, sz, tt.size)
|
||||
}
|
||||
}
|
||||
// Verify defaults.
|
||||
if v, _ := testdata.lookup([]byte{0xC1, 0x8C}); v != 0 {
|
||||
t.Errorf("lookup of non-existing rune should be 0; found %X", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLookupUnsafe(t *testing.T) {
|
||||
for i, tt := range testRunes {
|
||||
b, _ := mkUTF8(tt)
|
||||
v := testdata.lookupUnsafe(b)
|
||||
if int(v) != i {
|
||||
t.Errorf("lookupUnsafe(%U): found value %#x, expected %#x", i, v, i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLookupString(t *testing.T) {
|
||||
for i, tt := range testRunes {
|
||||
b, szg := mkUTF8(tt)
|
||||
v, szt := testdata.lookupString(string(b))
|
||||
if int(v) != i {
|
||||
t.Errorf("lookup(%U): found value %#x, expected %#x", i, v, i)
|
||||
}
|
||||
if szt != szg {
|
||||
t.Errorf("lookup(%U): found size %d, expected %d", i, szt, szg)
|
||||
}
|
||||
}
|
||||
for i, tt := range tests {
|
||||
v, sz := testdata.lookupString(string(tt.bytes))
|
||||
if int(v) != 0 {
|
||||
t.Errorf("lookup of illegal rune, case %d: found value %#x, expected 0", i, v)
|
||||
}
|
||||
if sz != tt.size {
|
||||
t.Errorf("lookup of illegal rune, case %d: found size %d, expected %d", i, sz, tt.size)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLookupStringUnsafe(t *testing.T) {
|
||||
for i, tt := range testRunes {
|
||||
b, _ := mkUTF8(tt)
|
||||
v := testdata.lookupStringUnsafe(string(b))
|
||||
if int(v) != i {
|
||||
t.Errorf("lookupUnsafe(%U): found value %#x, expected %#x", i, v, i)
|
||||
}
|
||||
}
|
||||
}
|
85
Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/triedata_test.go
generated
vendored
85
Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/triedata_test.go
generated
vendored
@ -1,85 +0,0 @@
|
||||
// Generated by running
|
||||
// maketesttables
|
||||
// DO NOT EDIT
|
||||
|
||||
package norm
|
||||
|
||||
var testRunes = []int32{1, 12, 127, 128, 256, 2047, 2048, 2457, 65535, 65536, 65793, 1114111, 512, 513, 514, 528, 533}
|
||||
|
||||
// testdataValues: 192 entries, 384 bytes
|
||||
// Block 2 is the null block.
|
||||
var testdataValues = [192]uint16{
|
||||
// Block 0x0, offset 0x0
|
||||
0x000c: 0x0001,
|
||||
// Block 0x1, offset 0x40
|
||||
0x007f: 0x0002,
|
||||
// Block 0x2, offset 0x80
|
||||
}
|
||||
|
||||
// testdataSparseOffset: 10 entries, 20 bytes
|
||||
var testdataSparseOffset = []uint16{0x0, 0x2, 0x4, 0x8, 0xa, 0xc, 0xe, 0x10, 0x12, 0x14}
|
||||
|
||||
// testdataSparseValues: 22 entries, 88 bytes
|
||||
var testdataSparseValues = [22]valueRange{
|
||||
// Block 0x0, offset 0x1
|
||||
{value: 0x0000, lo: 0x01},
|
||||
{value: 0x0003, lo: 0x80, hi: 0x80},
|
||||
// Block 0x1, offset 0x2
|
||||
{value: 0x0000, lo: 0x01},
|
||||
{value: 0x0004, lo: 0x80, hi: 0x80},
|
||||
// Block 0x2, offset 0x3
|
||||
{value: 0x0001, lo: 0x03},
|
||||
{value: 0x000c, lo: 0x80, hi: 0x82},
|
||||
{value: 0x000f, lo: 0x90, hi: 0x90},
|
||||
{value: 0x0010, lo: 0x95, hi: 0x95},
|
||||
// Block 0x3, offset 0x4
|
||||
{value: 0x0000, lo: 0x01},
|
||||
{value: 0x0005, lo: 0xbf, hi: 0xbf},
|
||||
// Block 0x4, offset 0x5
|
||||
{value: 0x0000, lo: 0x01},
|
||||
{value: 0x0006, lo: 0x80, hi: 0x80},
|
||||
// Block 0x5, offset 0x6
|
||||
{value: 0x0000, lo: 0x01},
|
||||
{value: 0x0007, lo: 0x99, hi: 0x99},
|
||||
// Block 0x6, offset 0x7
|
||||
{value: 0x0000, lo: 0x01},
|
||||
{value: 0x0008, lo: 0xbf, hi: 0xbf},
|
||||
// Block 0x7, offset 0x8
|
||||
{value: 0x0000, lo: 0x01},
|
||||
{value: 0x0009, lo: 0x80, hi: 0x80},
|
||||
// Block 0x8, offset 0x9
|
||||
{value: 0x0000, lo: 0x01},
|
||||
{value: 0x000a, lo: 0x81, hi: 0x81},
|
||||
// Block 0x9, offset 0xa
|
||||
{value: 0x0000, lo: 0x01},
|
||||
{value: 0x000b, lo: 0xbf, hi: 0xbf},
|
||||
}
|
||||
|
||||
// testdataLookup: 640 bytes
|
||||
// Block 0 is the null block.
|
||||
var testdataLookup = [640]uint8{
|
||||
// Block 0x0, offset 0x0
|
||||
// Block 0x1, offset 0x40
|
||||
// Block 0x2, offset 0x80
|
||||
// Block 0x3, offset 0xc0
|
||||
0x0c2: 0x01, 0x0c4: 0x02,
|
||||
0x0c8: 0x03,
|
||||
0x0df: 0x04,
|
||||
0x0e0: 0x02,
|
||||
0x0ef: 0x03,
|
||||
0x0f0: 0x05, 0x0f4: 0x07,
|
||||
// Block 0x4, offset 0x100
|
||||
0x120: 0x05, 0x126: 0x06,
|
||||
// Block 0x5, offset 0x140
|
||||
0x17f: 0x07,
|
||||
// Block 0x6, offset 0x180
|
||||
0x180: 0x08, 0x184: 0x09,
|
||||
// Block 0x7, offset 0x1c0
|
||||
0x1d0: 0x04,
|
||||
// Block 0x8, offset 0x200
|
||||
0x23f: 0x0a,
|
||||
// Block 0x9, offset 0x240
|
||||
0x24f: 0x06,
|
||||
}
|
||||
|
||||
var testdataTrie = trie{testdataLookup[:], testdataValues[:], testdataSparseValues[:], testdataSparseOffset[:], 1}
|
317
Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/triegen.go
generated
vendored
317
Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/triegen.go
generated
vendored
@ -1,317 +0,0 @@
|
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build ignore
|
||||
|
||||
// Trie table generator.
|
||||
// Used by make*tables tools to generate a go file with trie data structures
|
||||
// for mapping UTF-8 to a 16-bit value. All but the last byte in a UTF-8 byte
|
||||
// sequence are used to lookup offsets in the index table to be used for the
|
||||
// next byte. The last byte is used to index into a table with 16-bit values.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
"log"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
const (
|
||||
blockSize = 64
|
||||
blockOffset = 2 // Subtract two blocks to compensate for the 0x80 added to continuation bytes.
|
||||
maxSparseEntries = 16
|
||||
)
|
||||
|
||||
// Intermediate trie structure
|
||||
type trieNode struct {
|
||||
table [256]*trieNode
|
||||
value int
|
||||
b byte
|
||||
leaf bool
|
||||
}
|
||||
|
||||
func newNode() *trieNode {
|
||||
return new(trieNode)
|
||||
}
|
||||
|
||||
func (n trieNode) String() string {
|
||||
s := fmt.Sprint("trieNode{table: { non-nil at index: ")
|
||||
for i, v := range n.table {
|
||||
if v != nil {
|
||||
s += fmt.Sprintf("%d, ", i)
|
||||
}
|
||||
}
|
||||
s += fmt.Sprintf("}, value:%#x, b:%#x leaf:%v}", n.value, n.b, n.leaf)
|
||||
return s
|
||||
}
|
||||
|
||||
func (n trieNode) isInternal() bool {
|
||||
internal := true
|
||||
for i := 0; i < 256; i++ {
|
||||
if nn := n.table[i]; nn != nil {
|
||||
if !internal && !nn.leaf {
|
||||
log.Fatalf("triegen: isInternal: node contains both leaf and non-leaf children (%v)", n)
|
||||
}
|
||||
internal = internal && !nn.leaf
|
||||
}
|
||||
}
|
||||
return internal
|
||||
}
|
||||
|
||||
func (n trieNode) mostFrequentStride() int {
|
||||
counts := make(map[int]int)
|
||||
v := 0
|
||||
for _, t := range n.table[0x80 : 0x80+blockSize] {
|
||||
if t != nil {
|
||||
if stride := t.value - v; v != 0 && stride >= 0 {
|
||||
counts[stride]++
|
||||
}
|
||||
v = t.value
|
||||
} else {
|
||||
v = 0
|
||||
}
|
||||
}
|
||||
var maxs, maxc int
|
||||
for stride, cnt := range counts {
|
||||
if cnt > maxc || (cnt == maxc && stride < maxs) {
|
||||
maxs, maxc = stride, cnt
|
||||
}
|
||||
}
|
||||
return maxs
|
||||
}
|
||||
|
||||
func (n trieNode) countSparseEntries() int {
|
||||
stride := n.mostFrequentStride()
|
||||
var count, v int
|
||||
for _, t := range n.table[0x80 : 0x80+blockSize] {
|
||||
tv := 0
|
||||
if t != nil {
|
||||
tv = t.value
|
||||
}
|
||||
if tv-v != stride {
|
||||
if tv != 0 {
|
||||
count++
|
||||
}
|
||||
}
|
||||
v = tv
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func (n *trieNode) insert(r rune, value uint16) {
|
||||
var p [utf8.UTFMax]byte
|
||||
sz := utf8.EncodeRune(p[:], r)
|
||||
|
||||
for i := 0; i < sz; i++ {
|
||||
if n.leaf {
|
||||
log.Fatalf("triegen: insert: node (%#v) should not be a leaf", n)
|
||||
}
|
||||
nn := n.table[p[i]]
|
||||
if nn == nil {
|
||||
nn = newNode()
|
||||
nn.b = p[i]
|
||||
n.table[p[i]] = nn
|
||||
}
|
||||
n = nn
|
||||
}
|
||||
n.value = int(value)
|
||||
n.leaf = true
|
||||
}
|
||||
|
||||
type nodeIndex struct {
|
||||
lookupBlocks []*trieNode
|
||||
valueBlocks []*trieNode
|
||||
sparseBlocks []*trieNode
|
||||
sparseOffset []uint16
|
||||
sparseCount int
|
||||
|
||||
lookupBlockIdx map[uint32]int
|
||||
valueBlockIdx map[uint32]int
|
||||
}
|
||||
|
||||
func newIndex() *nodeIndex {
|
||||
index := &nodeIndex{}
|
||||
index.lookupBlocks = make([]*trieNode, 0)
|
||||
index.valueBlocks = make([]*trieNode, 0)
|
||||
index.sparseBlocks = make([]*trieNode, 0)
|
||||
index.sparseOffset = make([]uint16, 1)
|
||||
index.lookupBlockIdx = make(map[uint32]int)
|
||||
index.valueBlockIdx = make(map[uint32]int)
|
||||
return index
|
||||
}
|
||||
|
||||
func computeOffsets(index *nodeIndex, n *trieNode) int {
|
||||
if n.leaf {
|
||||
return n.value
|
||||
}
|
||||
hasher := crc32.New(crc32.MakeTable(crc32.IEEE))
|
||||
// We only index continuation bytes.
|
||||
for i := 0; i < blockSize; i++ {
|
||||
v := 0
|
||||
if nn := n.table[0x80+i]; nn != nil {
|
||||
v = computeOffsets(index, nn)
|
||||
}
|
||||
hasher.Write([]byte{uint8(v >> 8), uint8(v)})
|
||||
}
|
||||
h := hasher.Sum32()
|
||||
if n.isInternal() {
|
||||
v, ok := index.lookupBlockIdx[h]
|
||||
if !ok {
|
||||
v = len(index.lookupBlocks) - blockOffset
|
||||
index.lookupBlocks = append(index.lookupBlocks, n)
|
||||
index.lookupBlockIdx[h] = v
|
||||
}
|
||||
n.value = v
|
||||
} else {
|
||||
v, ok := index.valueBlockIdx[h]
|
||||
if !ok {
|
||||
if c := n.countSparseEntries(); c > maxSparseEntries {
|
||||
v = len(index.valueBlocks) - blockOffset
|
||||
index.valueBlocks = append(index.valueBlocks, n)
|
||||
index.valueBlockIdx[h] = v
|
||||
} else {
|
||||
v = -len(index.sparseOffset)
|
||||
index.sparseBlocks = append(index.sparseBlocks, n)
|
||||
index.sparseOffset = append(index.sparseOffset, uint16(index.sparseCount))
|
||||
index.sparseCount += c + 1
|
||||
index.valueBlockIdx[h] = v
|
||||
}
|
||||
}
|
||||
n.value = v
|
||||
}
|
||||
return n.value
|
||||
}
|
||||
|
||||
func printValueBlock(nr int, n *trieNode, offset int) {
|
||||
boff := nr * blockSize
|
||||
fmt.Printf("\n// Block %#x, offset %#x", nr, boff)
|
||||
var printnewline bool
|
||||
for i := 0; i < blockSize; i++ {
|
||||
if i%6 == 0 {
|
||||
printnewline = true
|
||||
}
|
||||
v := 0
|
||||
if nn := n.table[i+offset]; nn != nil {
|
||||
v = nn.value
|
||||
}
|
||||
if v != 0 {
|
||||
if printnewline {
|
||||
fmt.Printf("\n")
|
||||
printnewline = false
|
||||
}
|
||||
fmt.Printf("%#04x:%#04x, ", boff+i, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func printSparseBlock(nr int, n *trieNode) {
|
||||
boff := -n.value
|
||||
fmt.Printf("\n// Block %#x, offset %#x", nr, boff)
|
||||
v := 0
|
||||
//stride := f(n)
|
||||
stride := n.mostFrequentStride()
|
||||
c := n.countSparseEntries()
|
||||
fmt.Printf("\n{value:%#04x,lo:%#02x},", stride, uint8(c))
|
||||
for i, nn := range n.table[0x80 : 0x80+blockSize] {
|
||||
nv := 0
|
||||
if nn != nil {
|
||||
nv = nn.value
|
||||
}
|
||||
if nv-v != stride {
|
||||
if v != 0 {
|
||||
fmt.Printf(",hi:%#02x},", 0x80+i-1)
|
||||
}
|
||||
if nv != 0 {
|
||||
fmt.Printf("\n{value:%#04x,lo:%#02x", nv, nn.b)
|
||||
}
|
||||
}
|
||||
v = nv
|
||||
}
|
||||
if v != 0 {
|
||||
fmt.Printf(",hi:%#02x},", 0x80+blockSize-1)
|
||||
}
|
||||
}
|
||||
|
||||
func printLookupBlock(nr int, n *trieNode, offset, cutoff int) {
|
||||
boff := nr * blockSize
|
||||
fmt.Printf("\n// Block %#x, offset %#x", nr, boff)
|
||||
var printnewline bool
|
||||
for i := 0; i < blockSize; i++ {
|
||||
if i%8 == 0 {
|
||||
printnewline = true
|
||||
}
|
||||
v := 0
|
||||
if nn := n.table[i+offset]; nn != nil {
|
||||
v = nn.value
|
||||
}
|
||||
if v != 0 {
|
||||
if v < 0 {
|
||||
v = -v - 1 + cutoff
|
||||
}
|
||||
if printnewline {
|
||||
fmt.Printf("\n")
|
||||
printnewline = false
|
||||
}
|
||||
fmt.Printf("%#03x:%#02x, ", boff+i, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// printTables returns the size in bytes of the generated tables.
|
||||
func (t *trieNode) printTables(name string) int {
|
||||
index := newIndex()
|
||||
// Values for 7-bit ASCII are stored in first two block, followed by nil block.
|
||||
index.valueBlocks = append(index.valueBlocks, nil, nil, nil)
|
||||
// First byte of multi-byte UTF-8 codepoints are indexed in 4th block.
|
||||
index.lookupBlocks = append(index.lookupBlocks, nil, nil, nil, nil)
|
||||
// Index starter bytes of multi-byte UTF-8.
|
||||
for i := 0xC0; i < 0x100; i++ {
|
||||
if t.table[i] != nil {
|
||||
computeOffsets(index, t.table[i])
|
||||
}
|
||||
}
|
||||
|
||||
nv := len(index.valueBlocks) * blockSize
|
||||
fmt.Printf("// %sValues: %d entries, %d bytes\n", name, nv, nv*2)
|
||||
fmt.Printf("// Block 2 is the null block.\n")
|
||||
fmt.Printf("var %sValues = [%d]uint16 {", name, nv)
|
||||
printValueBlock(0, t, 0)
|
||||
printValueBlock(1, t, 64)
|
||||
printValueBlock(2, newNode(), 0)
|
||||
for i := 3; i < len(index.valueBlocks); i++ {
|
||||
printValueBlock(i, index.valueBlocks[i], 0x80)
|
||||
}
|
||||
fmt.Print("\n}\n\n")
|
||||
|
||||
ls := len(index.sparseBlocks)
|
||||
fmt.Printf("// %sSparseOffset: %d entries, %d bytes\n", name, ls, ls*2)
|
||||
fmt.Printf("var %sSparseOffset = %#v\n\n", name, index.sparseOffset[1:])
|
||||
|
||||
ns := index.sparseCount
|
||||
fmt.Printf("// %sSparseValues: %d entries, %d bytes\n", name, ns, ns*4)
|
||||
fmt.Printf("var %sSparseValues = [%d]valueRange {", name, ns)
|
||||
for i, n := range index.sparseBlocks {
|
||||
printSparseBlock(i, n)
|
||||
}
|
||||
fmt.Print("\n}\n\n")
|
||||
|
||||
cutoff := len(index.valueBlocks) - blockOffset
|
||||
ni := len(index.lookupBlocks) * blockSize
|
||||
fmt.Printf("// %sLookup: %d bytes\n", name, ni)
|
||||
fmt.Printf("// Block 0 is the null block.\n")
|
||||
fmt.Printf("var %sLookup = [%d]uint8 {", name, ni)
|
||||
printLookupBlock(0, newNode(), 0, cutoff)
|
||||
printLookupBlock(1, newNode(), 0, cutoff)
|
||||
printLookupBlock(2, newNode(), 0, cutoff)
|
||||
printLookupBlock(3, t, 0xC0, cutoff)
|
||||
for i := 4; i < len(index.lookupBlocks); i++ {
|
||||
printLookupBlock(i, index.lookupBlocks[i], 0x80, cutoff)
|
||||
}
|
||||
fmt.Print("\n}\n\n")
|
||||
fmt.Printf("var %sTrie = trie{ %sLookup[:], %sValues[:], %sSparseValues[:], %sSparseOffset[:], %d}\n\n",
|
||||
name, name, name, name, name, cutoff)
|
||||
return nv*2 + ns*4 + ni + ls*2
|
||||
}
|
@ -8,11 +8,11 @@ package bcrypt
|
||||
|
||||
// The code is a port of Provos and Mazières's C implementation.
|
||||
import (
|
||||
"code.google.com/p/go.crypto/blowfish"
|
||||
"crypto/rand"
|
||||
"crypto/subtle"
|
||||
"errors"
|
||||
"fmt"
|
||||
"golang.org/x/crypto/blowfish"
|
||||
"io"
|
||||
"strconv"
|
||||
)
|
@ -8,8 +8,8 @@ import (
|
||||
"fmt"
|
||||
"unicode"
|
||||
|
||||
"code.google.com/p/go.text/transform"
|
||||
"code.google.com/p/go.text/unicode/norm"
|
||||
"golang.org/x/text/transform"
|
||||
"golang.org/x/text/unicode/norm"
|
||||
)
|
||||
|
||||
func ExampleRemoveFunc() {
|
@ -55,10 +55,17 @@ type Transformer interface {
|
||||
// either error may be returned. Other than the error conditions listed
|
||||
// here, implementations are free to report other errors that arise.
|
||||
Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error)
|
||||
|
||||
// Reset resets the state and allows a Transformer to be reused.
|
||||
Reset()
|
||||
}
|
||||
|
||||
// TODO: Do we require that a Transformer be reusable if it returns a nil error
|
||||
// or do we always require a reset after use? Is Reset mandatory or optional?
|
||||
// NopResetter can be embedded by implementations of Transformer to add a nop
|
||||
// Reset method.
|
||||
type NopResetter struct{}
|
||||
|
||||
// Reset implements the Reset method of the Transformer interface.
|
||||
func (NopResetter) Reset() {}
|
||||
|
||||
// Reader wraps another io.Reader by transforming the bytes read.
|
||||
type Reader struct {
|
||||
@ -84,8 +91,9 @@ type Reader struct {
|
||||
const defaultBufSize = 4096
|
||||
|
||||
// NewReader returns a new Reader that wraps r by transforming the bytes read
|
||||
// via t.
|
||||
// via t. It calls Reset on t.
|
||||
func NewReader(r io.Reader, t Transformer) *Reader {
|
||||
t.Reset()
|
||||
return &Reader{
|
||||
r: r,
|
||||
t: t,
|
||||
@ -170,8 +178,9 @@ type Writer struct {
|
||||
}
|
||||
|
||||
// NewWriter returns a new Writer that wraps w by transforming the bytes written
|
||||
// via t.
|
||||
// via t. It calls Reset on t.
|
||||
func NewWriter(w io.Writer, t Transformer) *Writer {
|
||||
t.Reset()
|
||||
return &Writer{
|
||||
w: w,
|
||||
t: t,
|
||||
@ -247,7 +256,7 @@ func (w *Writer) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
type nop struct{}
|
||||
type nop struct{ NopResetter }
|
||||
|
||||
func (nop) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
|
||||
n := copy(dst, src)
|
||||
@ -257,7 +266,7 @@ func (nop) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
|
||||
return n, n, err
|
||||
}
|
||||
|
||||
type discard struct{}
|
||||
type discard struct{ NopResetter }
|
||||
|
||||
func (discard) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
|
||||
return 0, len(src), nil
|
||||
@ -327,6 +336,16 @@ func Chain(t ...Transformer) Transformer {
|
||||
return c
|
||||
}
|
||||
|
||||
// Reset resets the state of Chain. It calls Reset on all the Transformers.
|
||||
func (c *chain) Reset() {
|
||||
for i, l := range c.link {
|
||||
if l.t != nil {
|
||||
l.t.Reset()
|
||||
}
|
||||
c.link[i].p, c.link[i].n = 0, 0
|
||||
}
|
||||
}
|
||||
|
||||
// Transform applies the transformers of c in sequence.
|
||||
func (c *chain) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
|
||||
// Set up src and dst in the chain.
|
||||
@ -425,6 +444,8 @@ func RemoveFunc(f func(r rune) bool) Transformer {
|
||||
|
||||
type removeF func(r rune) bool
|
||||
|
||||
func (removeF) Reset() {}
|
||||
|
||||
// Transform implements the Transformer interface.
|
||||
func (t removeF) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
|
||||
for r, sz := rune(0), 0; len(src) > 0; src = src[sz:] {
|
||||
@ -436,7 +457,7 @@ func (t removeF) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err err
|
||||
|
||||
if sz == 1 {
|
||||
// Invalid rune.
|
||||
if !atEOF && !utf8.FullRune(src[nSrc:]) {
|
||||
if !atEOF && !utf8.FullRune(src) {
|
||||
err = ErrShortSrc
|
||||
break
|
||||
}
|
||||
@ -485,12 +506,14 @@ func grow(b []byte, n int) []byte {
|
||||
const initialBufSize = 128
|
||||
|
||||
// String returns a string with the result of converting s[:n] using t, where
|
||||
// n <= len(s). If err == nil, n will be len(s).
|
||||
// n <= len(s). If err == nil, n will be len(s). It calls Reset on t.
|
||||
func String(t Transformer, s string) (result string, n int, err error) {
|
||||
if s == "" {
|
||||
return "", 0, nil
|
||||
}
|
||||
|
||||
t.Reset()
|
||||
|
||||
// Allocate only once. Note that both dst and src escape when passed to
|
||||
// Transform.
|
||||
buf := [2 * initialBufSize]byte{}
|
||||
@ -571,8 +594,9 @@ func String(t Transformer, s string) (result string, n int, err error) {
|
||||
}
|
||||
|
||||
// Bytes returns a new byte slice with the result of converting b[:n] using t,
|
||||
// where n <= len(b). If err == nil, n will be len(b).
|
||||
// where n <= len(b). If err == nil, n will be len(b). It calls Reset on t.
|
||||
func Bytes(t Transformer, b []byte) (result []byte, n int, err error) {
|
||||
t.Reset()
|
||||
dst := make([]byte, len(b))
|
||||
pDst, pSrc := 0, 0
|
||||
for {
|
@ -16,7 +16,7 @@ import (
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
type lowerCaseASCII struct{}
|
||||
type lowerCaseASCII struct{ transform.NopResetter }
|
||||
|
||||
func (lowerCaseASCII) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
|
||||
n := len(src)
|
||||
@ -34,7 +34,7 @@ func (lowerCaseASCII) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, er
|
||||
|
||||
var errYouMentionedX = errors.New("you mentioned X")
|
||||
|
||||
type dontMentionX struct{}
|
||||
type dontMentionX struct{ transform.NopResetter }
|
||||
|
||||
func (dontMentionX) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
|
||||
n := len(src)
|
||||
@ -52,7 +52,7 @@ func (dontMentionX) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err
|
||||
|
||||
// doublerAtEOF is a strange Transformer that transforms "this" to "tthhiiss",
|
||||
// but only if atEOF is true.
|
||||
type doublerAtEOF struct{}
|
||||
type doublerAtEOF struct{ transform.NopResetter }
|
||||
|
||||
func (doublerAtEOF) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
|
||||
if !atEOF {
|
||||
@ -71,7 +71,7 @@ func (doublerAtEOF) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err
|
||||
// rleDecode and rleEncode implement a toy run-length encoding: "aabbbbbbbbbb"
|
||||
// is encoded as "2a10b". The decoding is assumed to not contain any numbers.
|
||||
|
||||
type rleDecode struct{}
|
||||
type rleDecode struct{ transform.NopResetter }
|
||||
|
||||
func (rleDecode) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
|
||||
loop:
|
||||
@ -104,6 +104,8 @@ loop:
|
||||
}
|
||||
|
||||
type rleEncode struct {
|
||||
transform.NopResetter
|
||||
|
||||
// allowStutter means that "xxxxxxxx" can be encoded as "5x3x"
|
||||
// instead of always as "8x".
|
||||
allowStutter bool
|
||||
@ -136,6 +138,10 @@ func (e rleEncode) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err e
|
||||
// trickler consumes all input bytes, but writes a single byte at a time to dst.
|
||||
type trickler []byte
|
||||
|
||||
func (t *trickler) Reset() {
|
||||
*t = nil
|
||||
}
|
||||
|
||||
func (t *trickler) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
|
||||
*t = append(*t, src...)
|
||||
if len(*t) == 0 {
|
||||
@ -157,6 +163,9 @@ func (t *trickler) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err e
|
||||
// to have some tolerance as long as progress can be detected.
|
||||
type delayedTrickler []byte
|
||||
|
||||
func (t *delayedTrickler) Reset() {
|
||||
*t = nil
|
||||
}
|
||||
func (t *delayedTrickler) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
|
||||
if len(*t) > 0 && len(dst) > 0 {
|
||||
dst[0] = (*t)[0]
|
||||
@ -447,7 +456,6 @@ var testCases = []testCase{
|
||||
|
||||
func TestReader(t *testing.T) {
|
||||
for _, tc := range testCases {
|
||||
reset(tc.t)
|
||||
r := NewReader(strings.NewReader(tc.src), tc.t)
|
||||
// Differently sized dst and src buffers are not part of the
|
||||
// exported API. We override them manually.
|
||||
@ -461,13 +469,6 @@ func TestReader(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func reset(t Transformer) {
|
||||
var dst [128]byte
|
||||
for err := ErrShortDst; err != nil; {
|
||||
_, _, err = t.Transform(dst[:], nil, true)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriter(t *testing.T) {
|
||||
tests := append(testCases, chainTests()...)
|
||||
for _, tc := range tests {
|
||||
@ -477,7 +478,6 @@ func TestWriter(t *testing.T) {
|
||||
}
|
||||
for _, sz := range sizes {
|
||||
bb := &bytes.Buffer{}
|
||||
reset(tc.t)
|
||||
w := NewWriter(bb, tc.t)
|
||||
// Differently sized dst and src buffers are not part of the
|
||||
// exported API. We override them manually.
|
||||
@ -735,7 +735,7 @@ func chainTests() []testCase {
|
||||
}
|
||||
|
||||
func doTransform(tc testCase) (res string, iter int, err error) {
|
||||
reset(tc.t)
|
||||
tc.t.Reset()
|
||||
dst := make([]byte, tc.dstSize)
|
||||
out, in := make([]byte, 0, 2*len(tc.src)), []byte(tc.src)
|
||||
for {
|
||||
@ -884,6 +884,14 @@ func TestRemoveFunc(t *testing.T) {
|
||||
wantIter: 2,
|
||||
},
|
||||
|
||||
{
|
||||
// Test a long buffer greater than the internal buffer size
|
||||
src: "hello\xcc\xcc\xccworld",
|
||||
srcSize: 13,
|
||||
wantStr: "hello\uFFFD\uFFFD\uFFFDworld",
|
||||
wantIter: 1,
|
||||
},
|
||||
|
||||
{
|
||||
src: "\u2345",
|
||||
dstSize: 2,
|
||||
@ -947,7 +955,6 @@ func testString(t *testing.T, f func(Transformer, string) (string, int, error))
|
||||
// that depend on a specific buffer size being set.
|
||||
continue
|
||||
}
|
||||
reset(tt.t)
|
||||
if tt.wantErr == ErrShortDst || tt.wantErr == ErrShortSrc {
|
||||
// The result string will be different.
|
||||
continue
|
@ -5,9 +5,6 @@
|
||||
maketables: maketables.go triegen.go
|
||||
go build $^
|
||||
|
||||
maketesttables: maketesttables.go triegen.go
|
||||
go build $^
|
||||
|
||||
normregtest: normregtest.go
|
||||
go build $^
|
||||
|
||||
@ -15,10 +12,6 @@ tables: maketables
|
||||
./maketables > tables.go
|
||||
gofmt -w tables.go
|
||||
|
||||
trietesttables: maketesttables
|
||||
./maketesttables > triedata_test.go
|
||||
gofmt -w triedata_test.go
|
||||
|
||||
# Downloads from www.unicode.org, so not part
|
||||
# of standard test scripts.
|
||||
test: testtables regtest
|
@ -9,7 +9,7 @@ import (
|
||||
"fmt"
|
||||
"unicode/utf8"
|
||||
|
||||
"code.google.com/p/go.text/unicode/norm"
|
||||
"golang.org/x/text/unicode/norm"
|
||||
)
|
||||
|
||||
// EqualSimple uses a norm.Iter to compare two non-normalized
|
@ -201,17 +201,17 @@ func lookupInfoNFKC(b input, i int) Properties {
|
||||
// Properties returns properties for the first rune in s.
|
||||
func (f Form) Properties(s []byte) Properties {
|
||||
if f == NFC || f == NFD {
|
||||
return compInfo(nfcTrie.lookup(s))
|
||||
return compInfo(nfcData.lookup(s))
|
||||
}
|
||||
return compInfo(nfkcTrie.lookup(s))
|
||||
return compInfo(nfkcData.lookup(s))
|
||||
}
|
||||
|
||||
// PropertiesString returns properties for the first rune in s.
|
||||
func (f Form) PropertiesString(s string) Properties {
|
||||
if f == NFC || f == NFD {
|
||||
return compInfo(nfcTrie.lookupString(s))
|
||||
return compInfo(nfcData.lookupString(s))
|
||||
}
|
||||
return compInfo(nfkcTrie.lookupString(s))
|
||||
return compInfo(nfkcData.lookupString(s))
|
||||
}
|
||||
|
||||
// compInfo converts the information contained in v and sz
|
@ -77,16 +77,16 @@ func (in *input) copySlice(buf []byte, b, e int) int {
|
||||
|
||||
func (in *input) charinfoNFC(p int) (uint16, int) {
|
||||
if in.bytes == nil {
|
||||
return nfcTrie.lookupString(in.str[p:])
|
||||
return nfcData.lookupString(in.str[p:])
|
||||
}
|
||||
return nfcTrie.lookup(in.bytes[p:])
|
||||
return nfcData.lookup(in.bytes[p:])
|
||||
}
|
||||
|
||||
func (in *input) charinfoNFKC(p int) (uint16, int) {
|
||||
if in.bytes == nil {
|
||||
return nfkcTrie.lookupString(in.str[p:])
|
||||
return nfkcData.lookupString(in.str[p:])
|
||||
}
|
||||
return nfkcTrie.lookup(in.bytes[p:])
|
||||
return nfkcData.lookup(in.bytes[p:])
|
||||
}
|
||||
|
||||
func (in *input) hangul(p int) (r rune) {
|
@ -9,6 +9,8 @@ import (
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// MaxSegmentSize is the maximum size of a byte buffer needed to consider any
|
||||
// sequence of starter and non-starter runes for the purpose of normalization.
|
||||
const MaxSegmentSize = maxByteBufferSize
|
||||
|
||||
// An Iter iterates over a string or byte slice, while normalizing it
|
@ -24,7 +24,8 @@ import (
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"code.google.com/p/go.text/internal/ucd"
|
||||
"golang.org/x/text/internal/triegen"
|
||||
"golang.org/x/text/internal/ucd"
|
||||
)
|
||||
|
||||
func main() {
|
||||
@ -714,13 +715,14 @@ func printCharInfoTables() int {
|
||||
|
||||
varnames := []string{"nfc", "nfkc"}
|
||||
for i := 0; i < FNumberOfFormTypes; i++ {
|
||||
trie := newNode()
|
||||
trie := triegen.NewTrie(varnames[i])
|
||||
|
||||
for r, c := range chars {
|
||||
f := c.forms[i]
|
||||
d := f.expandedDecomp
|
||||
if len(d) != 0 {
|
||||
_, key := mkstr(c.codePoint, &f)
|
||||
trie.insert(rune(r), positionMap[key])
|
||||
trie.Insert(rune(r), uint64(positionMap[key]))
|
||||
if c.ccc != ccc(d[0]) {
|
||||
// We assume the lead ccc of a decomposition !=0 in this case.
|
||||
if ccc(d[0]) == 0 {
|
||||
@ -730,12 +732,16 @@ func printCharInfoTables() int {
|
||||
} else if c.nLeadingNonStarters > 0 && len(f.expandedDecomp) == 0 && c.ccc == 0 && !f.combinesBackward {
|
||||
// Handle cases where it can't be detected that the nLead should be equal
|
||||
// to nTrail.
|
||||
trie.insert(c.codePoint, positionMap[nLeadStr])
|
||||
trie.Insert(c.codePoint, uint64(positionMap[nLeadStr]))
|
||||
} else if v := makeEntry(&f, &c)<<8 | uint16(c.ccc); v != 0 {
|
||||
trie.insert(c.codePoint, 0x8000|v)
|
||||
trie.Insert(c.codePoint, uint64(0x8000|v))
|
||||
}
|
||||
}
|
||||
size += trie.printTables(varnames[i])
|
||||
sz, err := trie.Gen(os.Stdout, triegen.Compact(&normCompacter{name: varnames[i]}))
|
||||
if err != nil {
|
||||
logger.Fatal(err)
|
||||
}
|
||||
size += sz
|
||||
}
|
||||
return size
|
||||
}
|
@ -23,7 +23,7 @@ import (
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
"code.google.com/p/go.text/unicode/norm"
|
||||
"golang.org/x/text/unicode/norm"
|
||||
)
|
||||
|
||||
func main() {
|
7549
Godeps/_workspace/src/golang.org/x/text/unicode/norm/tables.go
generated
vendored
Normal file
7549
Godeps/_workspace/src/golang.org/x/text/unicode/norm/tables.go
generated
vendored
Normal file
File diff suppressed because it is too large
Load Diff
@ -7,13 +7,16 @@ package norm
|
||||
import (
|
||||
"unicode/utf8"
|
||||
|
||||
"code.google.com/p/go.text/transform"
|
||||
"golang.org/x/text/transform"
|
||||
)
|
||||
|
||||
// Transform implements the transform.Transformer interface. It may need to
|
||||
// write segments of up to MaxSegmentSize at once. Users should either catch
|
||||
// ErrShortDst and allow dst to grow or have dst be at least of size
|
||||
// MaxTransformChunkSize to be guaranteed of progress.
|
||||
// Reset implements the Reset method of the transform.Transformer interface.
|
||||
func (Form) Reset() {}
|
||||
|
||||
// Transform implements the Transform method of the transform.Transformer
|
||||
// interface. It may need to write segments of up to MaxSegmentSize at once.
|
||||
// Users should either catch ErrShortDst and allow dst to grow or have dst be at
|
||||
// least of size MaxTransformChunkSize to be guaranteed of progress.
|
||||
func (f Form) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
|
||||
n := 0
|
||||
// Cap the maximum number of src bytes to check.
|
@ -8,7 +8,7 @@ import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"code.google.com/p/go.text/transform"
|
||||
"golang.org/x/text/transform"
|
||||
)
|
||||
|
||||
func TestTransform(t *testing.T) {
|
54
Godeps/_workspace/src/golang.org/x/text/unicode/norm/trie.go
generated
vendored
Normal file
54
Godeps/_workspace/src/golang.org/x/text/unicode/norm/trie.go
generated
vendored
Normal file
@ -0,0 +1,54 @@
|
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package norm
|
||||
|
||||
type valueRange struct {
|
||||
value uint16 // header: value:stride
|
||||
lo, hi byte // header: lo:n
|
||||
}
|
||||
|
||||
type sparseBlocks struct {
|
||||
values []valueRange
|
||||
offset []uint16
|
||||
}
|
||||
|
||||
var nfcSparse = sparseBlocks{
|
||||
values: nfcSparseValues[:],
|
||||
offset: nfcSparseOffset[:],
|
||||
}
|
||||
|
||||
var nfkcSparse = sparseBlocks{
|
||||
values: nfkcSparseValues[:],
|
||||
offset: nfkcSparseOffset[:],
|
||||
}
|
||||
|
||||
var (
|
||||
nfcData = newNfcTrie(0)
|
||||
nfkcData = newNfkcTrie(0)
|
||||
)
|
||||
|
||||
// lookupValue determines the type of block n and looks up the value for b.
|
||||
// For n < t.cutoff, the block is a simple lookup table. Otherwise, the block
|
||||
// is a list of ranges with an accompanying value. Given a matching range r,
|
||||
// the value for b is by r.value + (b - r.lo) * stride.
|
||||
func (t *sparseBlocks) lookup(n uint32, b byte) uint16 {
|
||||
offset := t.offset[n]
|
||||
header := t.values[offset]
|
||||
lo := offset + 1
|
||||
hi := lo + uint16(header.lo)
|
||||
for lo < hi {
|
||||
m := lo + (hi-lo)/2
|
||||
r := t.values[m]
|
||||
if r.lo <= b && b <= r.hi {
|
||||
return r.value + uint16(b-r.lo)*header.value
|
||||
}
|
||||
if b < r.lo {
|
||||
hi = m
|
||||
} else {
|
||||
lo = m + 1
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
117
Godeps/_workspace/src/golang.org/x/text/unicode/norm/triegen.go
generated
vendored
Normal file
117
Godeps/_workspace/src/golang.org/x/text/unicode/norm/triegen.go
generated
vendored
Normal file
@ -0,0 +1,117 @@
|
||||
// Copyright 2011 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build ignore
|
||||
|
||||
// Trie table generator.
|
||||
// Used by make*tables tools to generate a go file with trie data structures
|
||||
// for mapping UTF-8 to a 16-bit value. All but the last byte in a UTF-8 byte
|
||||
// sequence are used to lookup offsets in the index table to be used for the
|
||||
// next byte. The last byte is used to index into a table with 16-bit values.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
)
|
||||
|
||||
const maxSparseEntries = 16
|
||||
|
||||
type normCompacter struct {
|
||||
sparseBlocks [][]uint64
|
||||
sparseOffset []uint16
|
||||
sparseCount int
|
||||
name string
|
||||
}
|
||||
|
||||
func mostFrequentStride(a []uint64) int {
|
||||
counts := make(map[int]int)
|
||||
var v int
|
||||
for _, x := range a {
|
||||
if stride := int(x) - v; v != 0 && stride >= 0 {
|
||||
counts[stride]++
|
||||
}
|
||||
v = int(x)
|
||||
}
|
||||
var maxs, maxc int
|
||||
for stride, cnt := range counts {
|
||||
if cnt > maxc || (cnt == maxc && stride < maxs) {
|
||||
maxs, maxc = stride, cnt
|
||||
}
|
||||
}
|
||||
return maxs
|
||||
}
|
||||
|
||||
func countSparseEntries(a []uint64) int {
|
||||
stride := mostFrequentStride(a)
|
||||
var v, count int
|
||||
for _, tv := range a {
|
||||
if int(tv)-v != stride {
|
||||
if tv != 0 {
|
||||
count++
|
||||
}
|
||||
}
|
||||
v = int(tv)
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func (c *normCompacter) Size(v []uint64) (sz int, ok bool) {
|
||||
if n := countSparseEntries(v); n <= maxSparseEntries {
|
||||
return (n+1)*4 + 2, true
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func (c *normCompacter) Store(v []uint64) uint32 {
|
||||
h := uint32(len(c.sparseOffset))
|
||||
c.sparseBlocks = append(c.sparseBlocks, v)
|
||||
c.sparseOffset = append(c.sparseOffset, uint16(c.sparseCount))
|
||||
c.sparseCount += countSparseEntries(v) + 1
|
||||
return h
|
||||
}
|
||||
|
||||
func (c *normCompacter) Handler() string {
|
||||
return c.name + "Sparse.lookup"
|
||||
}
|
||||
|
||||
func (c *normCompacter) Print(w io.Writer) (retErr error) {
|
||||
p := func(f string, x ...interface{}) {
|
||||
if _, err := fmt.Fprintf(w, f, x...); retErr == nil && err != nil {
|
||||
retErr = err
|
||||
}
|
||||
}
|
||||
|
||||
ls := len(c.sparseBlocks)
|
||||
p("// %sSparseOffset: %d entries, %d bytes\n", c.name, ls, ls*2)
|
||||
p("var %sSparseOffset = %#v\n\n", c.name, c.sparseOffset)
|
||||
|
||||
ns := c.sparseCount
|
||||
p("// %sSparseValues: %d entries, %d bytes\n", c.name, ns, ns*4)
|
||||
p("var %sSparseValues = [%d]valueRange {", c.name, ns)
|
||||
for i, b := range c.sparseBlocks {
|
||||
p("\n// Block %#x, offset %#x", i, c.sparseOffset[i])
|
||||
var v int
|
||||
stride := mostFrequentStride(b)
|
||||
n := countSparseEntries(b)
|
||||
p("\n{value:%#04x,lo:%#02x},", stride, uint8(n))
|
||||
for i, nv := range b {
|
||||
if int(nv)-v != stride {
|
||||
if v != 0 {
|
||||
p(",hi:%#02x},", 0x80+i-1)
|
||||
}
|
||||
if nv != 0 {
|
||||
p("\n{value:%#04x,lo:%#02x", nv, 0x80+i)
|
||||
}
|
||||
}
|
||||
v = int(nv)
|
||||
}
|
||||
if v != 0 {
|
||||
p(",hi:%#02x},", 0x80+len(b)-1)
|
||||
}
|
||||
}
|
||||
p("\n}\n\n")
|
||||
return
|
||||
}
|
6
build.go
6
build.go
@ -167,9 +167,9 @@ func checkRequiredGoVersion() {
|
||||
}
|
||||
|
||||
func setup() {
|
||||
runPrint("go", "get", "-v", "code.google.com/p/go.tools/cmd/cover")
|
||||
runPrint("go", "get", "-v", "code.google.com/p/go.tools/cmd/vet")
|
||||
runPrint("go", "get", "-v", "code.google.com/p/go.net/html")
|
||||
runPrint("go", "get", "-v", "golang.org/x/tools/cmd/cover")
|
||||
runPrint("go", "get", "-v", "golang.org/x/tools/cmd/vet")
|
||||
runPrint("go", "get", "-v", "golang.org/x/net/html")
|
||||
runPrint("go", "get", "-v", "github.com/tools/godep")
|
||||
}
|
||||
|
||||
|
@ -31,7 +31,6 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"code.google.com/p/go.crypto/bcrypt"
|
||||
"github.com/calmh/logger"
|
||||
"github.com/syncthing/syncthing/internal/auto"
|
||||
"github.com/syncthing/syncthing/internal/config"
|
||||
@ -42,6 +41,7 @@ import (
|
||||
"github.com/syncthing/syncthing/internal/protocol"
|
||||
"github.com/syncthing/syncthing/internal/upgrade"
|
||||
"github.com/vitrun/qart/qr"
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
)
|
||||
|
||||
type guiError struct {
|
||||
|
@ -24,8 +24,8 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"code.google.com/p/go.crypto/bcrypt"
|
||||
"github.com/syncthing/syncthing/internal/config"
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
)
|
||||
|
||||
var (
|
||||
|
@ -36,7 +36,6 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"code.google.com/p/go.crypto/bcrypt"
|
||||
"github.com/calmh/logger"
|
||||
"github.com/juju/ratelimit"
|
||||
"github.com/syncthing/syncthing/internal/config"
|
||||
@ -50,6 +49,7 @@ import (
|
||||
"github.com/syncthing/syncthing/internal/upnp"
|
||||
"github.com/syndtr/goleveldb/leveldb"
|
||||
"github.com/syndtr/goleveldb/leveldb/opt"
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
)
|
||||
|
||||
var (
|
||||
|
@ -24,7 +24,7 @@ import (
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"code.google.com/p/go.net/html"
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
var trans = make(map[string]string)
|
||||
|
@ -42,6 +42,7 @@
|
||||
"Folder ID": "Folder ID",
|
||||
"Folder Master": "Folder Master",
|
||||
"Folder Path": "Folder Path",
|
||||
"Folders": "Folders",
|
||||
"GUI Authentication Password": "GUI Authentication Password",
|
||||
"GUI Authentication User": "GUI Authentication User",
|
||||
"GUI Listen Addresses": "GUI Listen Addresses",
|
||||
@ -131,6 +132,7 @@
|
||||
"The rescan interval must be at least 5 seconds.": "The rescan interval must be at least 5 seconds.",
|
||||
"Unknown": "Unknown",
|
||||
"Unshared": "Unshared",
|
||||
"Unused": "Unused",
|
||||
"Up to Date": "Up to Date",
|
||||
"Upgrade To {%version%}": "Upgrade To {{version}}",
|
||||
"Upgrading": "Upgrading",
|
||||
|
@ -26,10 +26,10 @@ import (
|
||||
"sort"
|
||||
"strconv"
|
||||
|
||||
"code.google.com/p/go.crypto/bcrypt"
|
||||
"github.com/calmh/logger"
|
||||
"github.com/syncthing/syncthing/internal/osutil"
|
||||
"github.com/syncthing/syncthing/internal/protocol"
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
)
|
||||
|
||||
var l = logger.DefaultLogger
|
||||
|
@ -15,7 +15,7 @@
|
||||
|
||||
package osutil
|
||||
|
||||
import "code.google.com/p/go.text/unicode/norm"
|
||||
import "golang.org/x/text/unicode/norm"
|
||||
|
||||
func NormalizedFilename(s string) string {
|
||||
return norm.NFC.String(s)
|
||||
|
@ -17,7 +17,7 @@
|
||||
|
||||
package osutil
|
||||
|
||||
import "code.google.com/p/go.text/unicode/norm"
|
||||
import "golang.org/x/text/unicode/norm"
|
||||
|
||||
func NormalizedFilename(s string) string {
|
||||
return norm.NFC.String(s)
|
||||
|
@ -18,7 +18,7 @@ package osutil
|
||||
import (
|
||||
"path/filepath"
|
||||
|
||||
"code.google.com/p/go.text/unicode/norm"
|
||||
"golang.org/x/text/unicode/norm"
|
||||
)
|
||||
|
||||
func NormalizedFilename(s string) string {
|
||||
|
@ -19,7 +19,7 @@ package protocol
|
||||
|
||||
// Darwin uses NFD normalization
|
||||
|
||||
import "code.google.com/p/go.text/unicode/norm"
|
||||
import "golang.org/x/text/unicode/norm"
|
||||
|
||||
type nativeModel struct {
|
||||
next Model
|
||||
|
@ -18,7 +18,7 @@ package protocol
|
||||
import (
|
||||
"path/filepath"
|
||||
|
||||
"code.google.com/p/go.text/unicode/norm"
|
||||
"golang.org/x/text/unicode/norm"
|
||||
)
|
||||
|
||||
type wireFormatConnection struct {
|
||||
|
@ -22,12 +22,11 @@ import (
|
||||
"runtime"
|
||||
"strings"
|
||||
|
||||
"code.google.com/p/go.text/unicode/norm"
|
||||
|
||||
"github.com/syncthing/syncthing/internal/ignore"
|
||||
"github.com/syncthing/syncthing/internal/lamport"
|
||||
"github.com/syncthing/syncthing/internal/protocol"
|
||||
"github.com/syncthing/syncthing/internal/symlinks"
|
||||
"golang.org/x/text/unicode/norm"
|
||||
)
|
||||
|
||||
type Walker struct {
|
||||
|
Loading…
Reference in New Issue
Block a user