mirror of
https://github.com/Llewellynvdm/fzf.git
synced 2024-11-19 03:25:11 +00:00
Junegunn Choi
parent
9d545f9578
commit
45793d75c2
@ -6,6 +6,8 @@ CHANGELOG
|
||||
- Added `--height HEIGHT[%]` option
|
||||
- Preview window will truncate long lines by default. Line wrap can be enabled
|
||||
by `:wrap` flag in `--preview-window`.
|
||||
- Added `--normalize` option to normalize latin script letters before
|
||||
matching. e.g. `sodanco` can match `Só Danço Samba`.
|
||||
|
||||
0.15.9
|
||||
------
|
||||
|
||||
@ -48,6 +48,10 @@ Case-insensitive match (default: smart-case match)
|
||||
.B "+i"
|
||||
Case-sensitive match
|
||||
.TP
|
||||
.B "--normalize"
|
||||
Normalize latin script letters before matching. This is not enabled by default
|
||||
to avoid performance overhead.
|
||||
.TP
|
||||
.BI "--algo=" TYPE
|
||||
Fuzzy matching algorithm (default: v2)
|
||||
|
||||
|
||||
113
src/algo/algo.go
113
src/algo/algo.go
@ -234,9 +234,36 @@ func bonusAt(input util.Chars, idx int) int16 {
|
||||
return bonusFor(charClassOf(input.Get(idx-1)), charClassOf(input.Get(idx)))
|
||||
}
|
||||
|
||||
type Algo func(caseSensitive bool, forward bool, input util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int)
|
||||
func normalizeRune(r rune) rune {
|
||||
if r < 0x00C0 || r > 0x2184 {
|
||||
return r
|
||||
}
|
||||
|
||||
func FuzzyMatchV2(caseSensitive bool, forward bool, input util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) {
|
||||
n := normalized[r]
|
||||
if n > 0 {
|
||||
return n
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func normalizeRunes(runes []rune) []rune {
|
||||
ret := make([]rune, len(runes))
|
||||
copy(ret, runes)
|
||||
for idx, r := range runes {
|
||||
if r < 0x00C0 || r > 0x2184 {
|
||||
continue
|
||||
}
|
||||
n := normalized[r]
|
||||
if n > 0 {
|
||||
ret[idx] = normalized[r]
|
||||
}
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
type Algo func(caseSensitive bool, normalize bool, forward bool, input util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int)
|
||||
|
||||
func FuzzyMatchV2(caseSensitive bool, normalize bool, forward bool, input util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) {
|
||||
// Assume that pattern is given in lowercase if case-insensitive.
|
||||
// First check if there's a match and calculate bonus for each position.
|
||||
// If the input string is too long, consider finding the matching chars in
|
||||
@ -247,13 +274,17 @@ func FuzzyMatchV2(caseSensitive bool, forward bool, input util.Chars, pattern []
|
||||
case 0:
|
||||
return Result{0, 0, 0}, posArray(withPos, M)
|
||||
case 1:
|
||||
return ExactMatchNaive(caseSensitive, forward, input, pattern[0:1], withPos, slab)
|
||||
return ExactMatchNaive(caseSensitive, normalize, forward, input, pattern[0:1], withPos, slab)
|
||||
}
|
||||
|
||||
// Since O(nm) algorithm can be prohibitively expensive for large input,
|
||||
// we fall back to the greedy algorithm.
|
||||
if slab != nil && N*M > cap(slab.I16) {
|
||||
return FuzzyMatchV1(caseSensitive, forward, input, pattern, withPos, slab)
|
||||
return FuzzyMatchV1(caseSensitive, normalize, forward, input, pattern, withPos, slab)
|
||||
}
|
||||
|
||||
if normalize {
|
||||
pattern = normalizeRunes(pattern)
|
||||
}
|
||||
|
||||
// Reuse pre-allocated integer slice to avoid unnecessary sweeping of garbages
|
||||
@ -285,6 +316,10 @@ func FuzzyMatchV2(caseSensitive bool, forward bool, input util.Chars, pattern []
|
||||
}
|
||||
}
|
||||
|
||||
if normalize {
|
||||
char = normalizeRune(char)
|
||||
}
|
||||
|
||||
T[idx] = char
|
||||
B[idx] = bonusFor(prevClass, class)
|
||||
prevClass = class
|
||||
@ -432,7 +467,7 @@ func FuzzyMatchV2(caseSensitive bool, forward bool, input util.Chars, pattern []
|
||||
}
|
||||
|
||||
// Implement the same sorting criteria as V2
|
||||
func calculateScore(caseSensitive bool, text util.Chars, pattern []rune, sidx int, eidx int, withPos bool) (int, *[]int) {
|
||||
func calculateScore(caseSensitive bool, normalize bool, text util.Chars, pattern []rune, sidx int, eidx int, withPos bool) (int, *[]int) {
|
||||
pidx, score, inGap, consecutive, firstBonus := 0, 0, false, 0, int16(0)
|
||||
pos := posArray(withPos, len(pattern))
|
||||
prevClass := charNonWord
|
||||
@ -449,6 +484,10 @@ func calculateScore(caseSensitive bool, text util.Chars, pattern []rune, sidx in
|
||||
char = unicode.To(unicode.LowerCase, char)
|
||||
}
|
||||
}
|
||||
// pattern is already normalized
|
||||
if normalize {
|
||||
char = normalizeRune(char)
|
||||
}
|
||||
if char == pattern[pidx] {
|
||||
if withPos {
|
||||
*pos = append(*pos, idx)
|
||||
@ -488,7 +527,7 @@ func calculateScore(caseSensitive bool, text util.Chars, pattern []rune, sidx in
|
||||
}
|
||||
|
||||
// FuzzyMatchV1 performs fuzzy-match
|
||||
func FuzzyMatchV1(caseSensitive bool, forward bool, text util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) {
|
||||
func FuzzyMatchV1(caseSensitive bool, normalize bool, forward bool, text util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) {
|
||||
if len(pattern) == 0 {
|
||||
return Result{0, 0, 0}, nil
|
||||
}
|
||||
@ -500,6 +539,10 @@ func FuzzyMatchV1(caseSensitive bool, forward bool, text util.Chars, pattern []r
|
||||
lenRunes := text.Length()
|
||||
lenPattern := len(pattern)
|
||||
|
||||
if normalize {
|
||||
pattern = normalizeRunes(pattern)
|
||||
}
|
||||
|
||||
for index := 0; index < lenRunes; index++ {
|
||||
char := text.Get(indexAt(index, lenRunes, forward))
|
||||
// This is considerably faster than blindly applying strings.ToLower to the
|
||||
@ -514,6 +557,9 @@ func FuzzyMatchV1(caseSensitive bool, forward bool, text util.Chars, pattern []r
|
||||
char = unicode.To(unicode.LowerCase, char)
|
||||
}
|
||||
}
|
||||
if normalize {
|
||||
char = normalizeRune(char)
|
||||
}
|
||||
pchar := pattern[indexAt(pidx, lenPattern, forward)]
|
||||
if char == pchar {
|
||||
if sidx < 0 {
|
||||
@ -553,7 +599,7 @@ func FuzzyMatchV1(caseSensitive bool, forward bool, text util.Chars, pattern []r
|
||||
sidx, eidx = lenRunes-eidx, lenRunes-sidx
|
||||
}
|
||||
|
||||
score, pos := calculateScore(caseSensitive, text, pattern, sidx, eidx, withPos)
|
||||
score, pos := calculateScore(caseSensitive, normalize, text, pattern, sidx, eidx, withPos)
|
||||
return Result{sidx, eidx, score}, pos
|
||||
}
|
||||
return Result{-1, -1, 0}, nil
|
||||
@ -568,7 +614,7 @@ func FuzzyMatchV1(caseSensitive bool, forward bool, text util.Chars, pattern []r
|
||||
// bonus point, instead of stopping immediately after finding the first match.
|
||||
// The solution is much cheaper since there is only one possible alignment of
|
||||
// the pattern.
|
||||
func ExactMatchNaive(caseSensitive bool, forward bool, text util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) {
|
||||
func ExactMatchNaive(caseSensitive bool, normalize bool, forward bool, text util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) {
|
||||
if len(pattern) == 0 {
|
||||
return Result{0, 0, 0}, nil
|
||||
}
|
||||
@ -580,6 +626,10 @@ func ExactMatchNaive(caseSensitive bool, forward bool, text util.Chars, pattern
|
||||
return Result{-1, -1, 0}, nil
|
||||
}
|
||||
|
||||
if normalize {
|
||||
pattern = normalizeRunes(pattern)
|
||||
}
|
||||
|
||||
// For simplicity, only look at the bonus at the first character position
|
||||
pidx := 0
|
||||
bestPos, bonus, bestBonus := -1, int16(0), int16(-1)
|
||||
@ -593,6 +643,9 @@ func ExactMatchNaive(caseSensitive bool, forward bool, text util.Chars, pattern
|
||||
char = unicode.To(unicode.LowerCase, char)
|
||||
}
|
||||
}
|
||||
if normalize {
|
||||
char = normalizeRune(char)
|
||||
}
|
||||
pidx_ := indexAt(pidx, lenPattern, forward)
|
||||
pchar := pattern[pidx_]
|
||||
if pchar == char {
|
||||
@ -624,14 +677,14 @@ func ExactMatchNaive(caseSensitive bool, forward bool, text util.Chars, pattern
|
||||
sidx = lenRunes - (bestPos + 1)
|
||||
eidx = lenRunes - (bestPos - lenPattern + 1)
|
||||
}
|
||||
score, _ := calculateScore(caseSensitive, text, pattern, sidx, eidx, false)
|
||||
score, _ := calculateScore(caseSensitive, normalize, text, pattern, sidx, eidx, false)
|
||||
return Result{sidx, eidx, score}, nil
|
||||
}
|
||||
return Result{-1, -1, 0}, nil
|
||||
}
|
||||
|
||||
// PrefixMatch performs prefix-match
|
||||
func PrefixMatch(caseSensitive bool, forward bool, text util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) {
|
||||
func PrefixMatch(caseSensitive bool, normalize bool, forward bool, text util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) {
|
||||
if len(pattern) == 0 {
|
||||
return Result{0, 0, 0}, nil
|
||||
}
|
||||
@ -640,22 +693,29 @@ func PrefixMatch(caseSensitive bool, forward bool, text util.Chars, pattern []ru
|
||||
return Result{-1, -1, 0}, nil
|
||||
}
|
||||
|
||||
if normalize {
|
||||
pattern = normalizeRunes(pattern)
|
||||
}
|
||||
|
||||
for index, r := range pattern {
|
||||
char := text.Get(index)
|
||||
if !caseSensitive {
|
||||
char = unicode.ToLower(char)
|
||||
}
|
||||
if normalize {
|
||||
char = normalizeRune(char)
|
||||
}
|
||||
if char != r {
|
||||
return Result{-1, -1, 0}, nil
|
||||
}
|
||||
}
|
||||
lenPattern := len(pattern)
|
||||
score, _ := calculateScore(caseSensitive, text, pattern, 0, lenPattern, false)
|
||||
score, _ := calculateScore(caseSensitive, normalize, text, pattern, 0, lenPattern, false)
|
||||
return Result{0, lenPattern, score}, nil
|
||||
}
|
||||
|
||||
// SuffixMatch performs suffix-match
|
||||
func SuffixMatch(caseSensitive bool, forward bool, text util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) {
|
||||
func SuffixMatch(caseSensitive bool, normalize bool, forward bool, text util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) {
|
||||
lenRunes := text.Length()
|
||||
trimmedLen := lenRunes - text.TrailingWhitespaces()
|
||||
if len(pattern) == 0 {
|
||||
@ -666,11 +726,18 @@ func SuffixMatch(caseSensitive bool, forward bool, text util.Chars, pattern []ru
|
||||
return Result{-1, -1, 0}, nil
|
||||
}
|
||||
|
||||
if normalize {
|
||||
pattern = normalizeRunes(pattern)
|
||||
}
|
||||
|
||||
for index, r := range pattern {
|
||||
char := text.Get(index + diff)
|
||||
if !caseSensitive {
|
||||
char = unicode.ToLower(char)
|
||||
}
|
||||
if normalize {
|
||||
char = normalizeRune(char)
|
||||
}
|
||||
if char != r {
|
||||
return Result{-1, -1, 0}, nil
|
||||
}
|
||||
@ -678,21 +745,37 @@ func SuffixMatch(caseSensitive bool, forward bool, text util.Chars, pattern []ru
|
||||
lenPattern := len(pattern)
|
||||
sidx := trimmedLen - lenPattern
|
||||
eidx := trimmedLen
|
||||
score, _ := calculateScore(caseSensitive, text, pattern, sidx, eidx, false)
|
||||
score, _ := calculateScore(caseSensitive, normalize, text, pattern, sidx, eidx, false)
|
||||
return Result{sidx, eidx, score}, nil
|
||||
}
|
||||
|
||||
// EqualMatch performs equal-match
|
||||
func EqualMatch(caseSensitive bool, forward bool, text util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) {
|
||||
func EqualMatch(caseSensitive bool, normalize bool, forward bool, text util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) {
|
||||
lenPattern := len(pattern)
|
||||
if text.Length() != lenPattern {
|
||||
return Result{-1, -1, 0}, nil
|
||||
}
|
||||
match := true
|
||||
if normalize {
|
||||
runes := text.ToRunes()
|
||||
for idx, pchar := range pattern {
|
||||
char := runes[idx]
|
||||
if !caseSensitive {
|
||||
char = unicode.To(unicode.LowerCase, char)
|
||||
}
|
||||
if normalizeRune(pchar) != normalizeRune(char) {
|
||||
match = false
|
||||
break
|
||||
}
|
||||
}
|
||||
} else {
|
||||
runesStr := text.ToString()
|
||||
if !caseSensitive {
|
||||
runesStr = strings.ToLower(runesStr)
|
||||
}
|
||||
if runesStr == string(pattern) {
|
||||
match = runesStr == string(pattern)
|
||||
}
|
||||
if match {
|
||||
return Result{0, lenPattern, (scoreMatch+bonusBoundary)*lenPattern +
|
||||
(bonusFirstCharMultiplier-1)*bonusBoundary}, nil
|
||||
}
|
||||
|
||||
@ -10,10 +10,14 @@ import (
|
||||
)
|
||||
|
||||
func assertMatch(t *testing.T, fun Algo, caseSensitive, forward bool, input, pattern string, sidx int, eidx int, score int) {
|
||||
assertMatch2(t, fun, caseSensitive, false, forward, input, pattern, sidx, eidx, score)
|
||||
}
|
||||
|
||||
func assertMatch2(t *testing.T, fun Algo, caseSensitive, normalize, forward bool, input, pattern string, sidx int, eidx int, score int) {
|
||||
if !caseSensitive {
|
||||
pattern = strings.ToLower(pattern)
|
||||
}
|
||||
res, pos := fun(caseSensitive, forward, util.RunesToChars([]rune(input)), []rune(pattern), true, nil)
|
||||
res, pos := fun(caseSensitive, normalize, forward, util.RunesToChars([]rune(input)), []rune(pattern), true, nil)
|
||||
var start, end int
|
||||
if pos == nil || len(*pos) == 0 {
|
||||
start = res.Start
|
||||
@ -156,6 +160,21 @@ func TestEmptyPattern(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalize(t *testing.T) {
|
||||
caseSensitive := false
|
||||
normalize := true
|
||||
forward := true
|
||||
test := func(input, pattern string, sidx, eidx, score int, funs ...Algo) {
|
||||
for _, fun := range funs {
|
||||
assertMatch2(t, fun, caseSensitive, normalize, forward,
|
||||
input, pattern, sidx, eidx, score)
|
||||
}
|
||||
}
|
||||
test("Só Danço Samba", "So", 0, 2, 56, FuzzyMatchV1, FuzzyMatchV2, PrefixMatch, ExactMatchNaive)
|
||||
test("Só Danço Samba", "sodc", 0, 7, 89, FuzzyMatchV1, FuzzyMatchV2)
|
||||
test("Danço", "danco", 0, 5, 128, FuzzyMatchV1, FuzzyMatchV2, PrefixMatch, SuffixMatch, ExactMatchNaive, EqualMatch)
|
||||
}
|
||||
|
||||
func TestLongString(t *testing.T) {
|
||||
bytes := make([]byte, math.MaxUint16*2)
|
||||
for i := range bytes {
|
||||
|
||||
408
src/algo/normalize.go
Normal file
408
src/algo/normalize.go
Normal file
@ -0,0 +1,408 @@
|
||||
// Normalization of latin script letters
|
||||
// Reference: http://www.unicode.org/Public/UCD/latest/ucd/Index.txt
|
||||
|
||||
package algo
|
||||
|
||||
var normalized map[rune]rune = map[rune]rune{
|
||||
0x00E1: 'a', // WITH ACUTE, LATIN SMALL LETTER
|
||||
0x0103: 'a', // WITH BREVE, LATIN SMALL LETTER
|
||||
0x01CE: 'a', // WITH CARON, LATIN SMALL LETTER
|
||||
0x00E2: 'a', // WITH CIRCUMFLEX, LATIN SMALL LETTER
|
||||
0x00E4: 'a', // WITH DIAERESIS, LATIN SMALL LETTER
|
||||
0x0227: 'a', // WITH DOT ABOVE, LATIN SMALL LETTER
|
||||
0x1EA1: 'a', // WITH DOT BELOW, LATIN SMALL LETTER
|
||||
0x0201: 'a', // WITH DOUBLE GRAVE, LATIN SMALL LETTER
|
||||
0x00E0: 'a', // WITH GRAVE, LATIN SMALL LETTER
|
||||
0x1EA3: 'a', // WITH HOOK ABOVE, LATIN SMALL LETTER
|
||||
0x0203: 'a', // WITH INVERTED BREVE, LATIN SMALL LETTER
|
||||
0x0101: 'a', // WITH MACRON, LATIN SMALL LETTER
|
||||
0x0105: 'a', // WITH OGONEK, LATIN SMALL LETTER
|
||||
0x1E9A: 'a', // WITH RIGHT HALF RING, LATIN SMALL LETTER
|
||||
0x00E5: 'a', // WITH RING ABOVE, LATIN SMALL LETTER
|
||||
0x1E01: 'a', // WITH RING BELOW, LATIN SMALL LETTER
|
||||
0x00E3: 'a', // WITH TILDE, LATIN SMALL LETTER
|
||||
0x0363: 'a', // , COMBINING LATIN SMALL LETTER
|
||||
0x0250: 'a', // , LATIN SMALL LETTER TURNED
|
||||
0x1E03: 'b', // WITH DOT ABOVE, LATIN SMALL LETTER
|
||||
0x1E05: 'b', // WITH DOT BELOW, LATIN SMALL LETTER
|
||||
0x0253: 'b', // WITH HOOK, LATIN SMALL LETTER
|
||||
0x1E07: 'b', // WITH LINE BELOW, LATIN SMALL LETTER
|
||||
0x0180: 'b', // WITH STROKE, LATIN SMALL LETTER
|
||||
0x0183: 'b', // WITH TOPBAR, LATIN SMALL LETTER
|
||||
0x0107: 'c', // WITH ACUTE, LATIN SMALL LETTER
|
||||
0x010D: 'c', // WITH CARON, LATIN SMALL LETTER
|
||||
0x00E7: 'c', // WITH CEDILLA, LATIN SMALL LETTER
|
||||
0x0109: 'c', // WITH CIRCUMFLEX, LATIN SMALL LETTER
|
||||
0x0255: 'c', // WITH CURL, LATIN SMALL LETTER
|
||||
0x010B: 'c', // WITH DOT ABOVE, LATIN SMALL LETTER
|
||||
0x0188: 'c', // WITH HOOK, LATIN SMALL LETTER
|
||||
0x023C: 'c', // WITH STROKE, LATIN SMALL LETTER
|
||||
0x0368: 'c', // , COMBINING LATIN SMALL LETTER
|
||||
0x0297: 'c', // , LATIN LETTER STRETCHED
|
||||
0x2184: 'c', // , LATIN SMALL LETTER REVERSED
|
||||
0x010F: 'd', // WITH CARON, LATIN SMALL LETTER
|
||||
0x1E11: 'd', // WITH CEDILLA, LATIN SMALL LETTER
|
||||
0x1E13: 'd', // WITH CIRCUMFLEX BELOW, LATIN SMALL LETTER
|
||||
0x0221: 'd', // WITH CURL, LATIN SMALL LETTER
|
||||
0x1E0B: 'd', // WITH DOT ABOVE, LATIN SMALL LETTER
|
||||
0x1E0D: 'd', // WITH DOT BELOW, LATIN SMALL LETTER
|
||||
0x0257: 'd', // WITH HOOK, LATIN SMALL LETTER
|
||||
0x1E0F: 'd', // WITH LINE BELOW, LATIN SMALL LETTER
|
||||
0x0111: 'd', // WITH STROKE, LATIN SMALL LETTER
|
||||
0x0256: 'd', // WITH TAIL, LATIN SMALL LETTER
|
||||
0x018C: 'd', // WITH TOPBAR, LATIN SMALL LETTER
|
||||
0x0369: 'd', // , COMBINING LATIN SMALL LETTER
|
||||
0x00E9: 'e', // WITH ACUTE, LATIN SMALL LETTER
|
||||
0x0115: 'e', // WITH BREVE, LATIN SMALL LETTER
|
||||
0x011B: 'e', // WITH CARON, LATIN SMALL LETTER
|
||||
0x0229: 'e', // WITH CEDILLA, LATIN SMALL LETTER
|
||||
0x1E19: 'e', // WITH CIRCUMFLEX BELOW, LATIN SMALL LETTER
|
||||
0x00EA: 'e', // WITH CIRCUMFLEX, LATIN SMALL LETTER
|
||||
0x00EB: 'e', // WITH DIAERESIS, LATIN SMALL LETTER
|
||||
0x0117: 'e', // WITH DOT ABOVE, LATIN SMALL LETTER
|
||||
0x1EB9: 'e', // WITH DOT BELOW, LATIN SMALL LETTER
|
||||
0x0205: 'e', // WITH DOUBLE GRAVE, LATIN SMALL LETTER
|
||||
0x00E8: 'e', // WITH GRAVE, LATIN SMALL LETTER
|
||||
0x1EBB: 'e', // WITH HOOK ABOVE, LATIN SMALL LETTER
|
||||
0x025D: 'e', // WITH HOOK, LATIN SMALL LETTER REVERSED OPEN
|
||||
0x0207: 'e', // WITH INVERTED BREVE, LATIN SMALL LETTER
|
||||
0x0113: 'e', // WITH MACRON, LATIN SMALL LETTER
|
||||
0x0119: 'e', // WITH OGONEK, LATIN SMALL LETTER
|
||||
0x0247: 'e', // WITH STROKE, LATIN SMALL LETTER
|
||||
0x1E1B: 'e', // WITH TILDE BELOW, LATIN SMALL LETTER
|
||||
0x1EBD: 'e', // WITH TILDE, LATIN SMALL LETTER
|
||||
0x0364: 'e', // , COMBINING LATIN SMALL LETTER
|
||||
0x029A: 'e', // , LATIN SMALL LETTER CLOSED OPEN
|
||||
0x025E: 'e', // , LATIN SMALL LETTER CLOSED REVERSED OPEN
|
||||
0x025B: 'e', // , LATIN SMALL LETTER OPEN
|
||||
0x0258: 'e', // , LATIN SMALL LETTER REVERSED
|
||||
0x025C: 'e', // , LATIN SMALL LETTER REVERSED OPEN
|
||||
0x01DD: 'e', // , LATIN SMALL LETTER TURNED
|
||||
0x1D08: 'e', // , LATIN SMALL LETTER TURNED OPEN
|
||||
0x1E1F: 'f', // WITH DOT ABOVE, LATIN SMALL LETTER
|
||||
0x0192: 'f', // WITH HOOK, LATIN SMALL LETTER
|
||||
0x01F5: 'g', // WITH ACUTE, LATIN SMALL LETTER
|
||||
0x011F: 'g', // WITH BREVE, LATIN SMALL LETTER
|
||||
0x01E7: 'g', // WITH CARON, LATIN SMALL LETTER
|
||||
0x0123: 'g', // WITH CEDILLA, LATIN SMALL LETTER
|
||||
0x011D: 'g', // WITH CIRCUMFLEX, LATIN SMALL LETTER
|
||||
0x0121: 'g', // WITH DOT ABOVE, LATIN SMALL LETTER
|
||||
0x0260: 'g', // WITH HOOK, LATIN SMALL LETTER
|
||||
0x1E21: 'g', // WITH MACRON, LATIN SMALL LETTER
|
||||
0x01E5: 'g', // WITH STROKE, LATIN SMALL LETTER
|
||||
0x0261: 'g', // , LATIN SMALL LETTER SCRIPT
|
||||
0x1E2B: 'h', // WITH BREVE BELOW, LATIN SMALL LETTER
|
||||
0x021F: 'h', // WITH CARON, LATIN SMALL LETTER
|
||||
0x1E29: 'h', // WITH CEDILLA, LATIN SMALL LETTER
|
||||
0x0125: 'h', // WITH CIRCUMFLEX, LATIN SMALL LETTER
|
||||
0x1E27: 'h', // WITH DIAERESIS, LATIN SMALL LETTER
|
||||
0x1E23: 'h', // WITH DOT ABOVE, LATIN SMALL LETTER
|
||||
0x1E25: 'h', // WITH DOT BELOW, LATIN SMALL LETTER
|
||||
0x02AE: 'h', // WITH FISHHOOK, LATIN SMALL LETTER TURNED
|
||||
0x0266: 'h', // WITH HOOK, LATIN SMALL LETTER
|
||||
0x1E96: 'h', // WITH LINE BELOW, LATIN SMALL LETTER
|
||||
0x0127: 'h', // WITH STROKE, LATIN SMALL LETTER
|
||||
0x036A: 'h', // , COMBINING LATIN SMALL LETTER
|
||||
0x0265: 'h', // , LATIN SMALL LETTER TURNED
|
||||
0x2095: 'h', // , LATIN SUBSCRIPT SMALL LETTER
|
||||
0x00ED: 'i', // WITH ACUTE, LATIN SMALL LETTER
|
||||
0x012D: 'i', // WITH BREVE, LATIN SMALL LETTER
|
||||
0x01D0: 'i', // WITH CARON, LATIN SMALL LETTER
|
||||
0x00EE: 'i', // WITH CIRCUMFLEX, LATIN SMALL LETTER
|
||||
0x00EF: 'i', // WITH DIAERESIS, LATIN SMALL LETTER
|
||||
0x1ECB: 'i', // WITH DOT BELOW, LATIN SMALL LETTER
|
||||
0x0209: 'i', // WITH DOUBLE GRAVE, LATIN SMALL LETTER
|
||||
0x00EC: 'i', // WITH GRAVE, LATIN SMALL LETTER
|
||||
0x1EC9: 'i', // WITH HOOK ABOVE, LATIN SMALL LETTER
|
||||
0x020B: 'i', // WITH INVERTED BREVE, LATIN SMALL LETTER
|
||||
0x012B: 'i', // WITH MACRON, LATIN SMALL LETTER
|
||||
0x012F: 'i', // WITH OGONEK, LATIN SMALL LETTER
|
||||
0x0268: 'i', // WITH STROKE, LATIN SMALL LETTER
|
||||
0x1E2D: 'i', // WITH TILDE BELOW, LATIN SMALL LETTER
|
||||
0x0129: 'i', // WITH TILDE, LATIN SMALL LETTER
|
||||
0x0365: 'i', // , COMBINING LATIN SMALL LETTER
|
||||
0x0131: 'i', // , LATIN SMALL LETTER DOTLESS
|
||||
0x1D09: 'i', // , LATIN SMALL LETTER TURNED
|
||||
0x1D62: 'i', // , LATIN SUBSCRIPT SMALL LETTER
|
||||
0x2071: 'i', // , SUPERSCRIPT LATIN SMALL LETTER
|
||||
0x01F0: 'j', // WITH CARON, LATIN SMALL LETTER
|
||||
0x0135: 'j', // WITH CIRCUMFLEX, LATIN SMALL LETTER
|
||||
0x029D: 'j', // WITH CROSSED-TAIL, LATIN SMALL LETTER
|
||||
0x0249: 'j', // WITH STROKE, LATIN SMALL LETTER
|
||||
0x025F: 'j', // WITH STROKE, LATIN SMALL LETTER DOTLESS
|
||||
0x0237: 'j', // , LATIN SMALL LETTER DOTLESS
|
||||
0x1E31: 'k', // WITH ACUTE, LATIN SMALL LETTER
|
||||
0x01E9: 'k', // WITH CARON, LATIN SMALL LETTER
|
||||
0x0137: 'k', // WITH CEDILLA, LATIN SMALL LETTER
|
||||
0x1E33: 'k', // WITH DOT BELOW, LATIN SMALL LETTER
|
||||
0x0199: 'k', // WITH HOOK, LATIN SMALL LETTER
|
||||
0x1E35: 'k', // WITH LINE BELOW, LATIN SMALL LETTER
|
||||
0x029E: 'k', // , LATIN SMALL LETTER TURNED
|
||||
0x2096: 'k', // , LATIN SUBSCRIPT SMALL LETTER
|
||||
0x013A: 'l', // WITH ACUTE, LATIN SMALL LETTER
|
||||
0x019A: 'l', // WITH BAR, LATIN SMALL LETTER
|
||||
0x026C: 'l', // WITH BELT, LATIN SMALL LETTER
|
||||
0x013E: 'l', // WITH CARON, LATIN SMALL LETTER
|
||||
0x013C: 'l', // WITH CEDILLA, LATIN SMALL LETTER
|
||||
0x1E3D: 'l', // WITH CIRCUMFLEX BELOW, LATIN SMALL LETTER
|
||||
0x0234: 'l', // WITH CURL, LATIN SMALL LETTER
|
||||
0x1E37: 'l', // WITH DOT BELOW, LATIN SMALL LETTER
|
||||
0x1E3B: 'l', // WITH LINE BELOW, LATIN SMALL LETTER
|
||||
0x0140: 'l', // WITH MIDDLE DOT, LATIN SMALL LETTER
|
||||
0x026B: 'l', // WITH MIDDLE TILDE, LATIN SMALL LETTER
|
||||
0x026D: 'l', // WITH RETROFLEX HOOK, LATIN SMALL LETTER
|
||||
0x0142: 'l', // WITH STROKE, LATIN SMALL LETTER
|
||||
0x2097: 'l', // , LATIN SUBSCRIPT SMALL LETTER
|
||||
0x1E3F: 'm', // WITH ACUTE, LATIN SMALL LETTER
|
||||
0x1E41: 'm', // WITH DOT ABOVE, LATIN SMALL LETTER
|
||||
0x1E43: 'm', // WITH DOT BELOW, LATIN SMALL LETTER
|
||||
0x0271: 'm', // WITH HOOK, LATIN SMALL LETTER
|
||||
0x0270: 'm', // WITH LONG LEG, LATIN SMALL LETTER TURNED
|
||||
0x036B: 'm', // , COMBINING LATIN SMALL LETTER
|
||||
0x1D1F: 'm', // , LATIN SMALL LETTER SIDEWAYS TURNED
|
||||
0x026F: 'm', // , LATIN SMALL LETTER TURNED
|
||||
0x2098: 'm', // , LATIN SUBSCRIPT SMALL LETTER
|
||||
0x0144: 'n', // WITH ACUTE, LATIN SMALL LETTER
|
||||
0x0148: 'n', // WITH CARON, LATIN SMALL LETTER
|
||||
0x0146: 'n', // WITH CEDILLA, LATIN SMALL LETTER
|
||||
0x1E4B: 'n', // WITH CIRCUMFLEX BELOW, LATIN SMALL LETTER
|
||||
0x0235: 'n', // WITH CURL, LATIN SMALL LETTER
|
||||
0x1E45: 'n', // WITH DOT ABOVE, LATIN SMALL LETTER
|
||||
0x1E47: 'n', // WITH DOT BELOW, LATIN SMALL LETTER
|
||||
0x01F9: 'n', // WITH GRAVE, LATIN SMALL LETTER
|
||||
0x0272: 'n', // WITH LEFT HOOK, LATIN SMALL LETTER
|
||||
0x1E49: 'n', // WITH LINE BELOW, LATIN SMALL LETTER
|
||||
0x019E: 'n', // WITH LONG RIGHT LEG, LATIN SMALL LETTER
|
||||
0x0273: 'n', // WITH RETROFLEX HOOK, LATIN SMALL LETTER
|
||||
0x00F1: 'n', // WITH TILDE, LATIN SMALL LETTER
|
||||
0x2099: 'n', // , LATIN SUBSCRIPT SMALL LETTER
|
||||
0x00F3: 'o', // WITH ACUTE, LATIN SMALL LETTER
|
||||
0x014F: 'o', // WITH BREVE, LATIN SMALL LETTER
|
||||
0x01D2: 'o', // WITH CARON, LATIN SMALL LETTER
|
||||
0x00F4: 'o', // WITH CIRCUMFLEX, LATIN SMALL LETTER
|
||||
0x00F6: 'o', // WITH DIAERESIS, LATIN SMALL LETTER
|
||||
0x022F: 'o', // WITH DOT ABOVE, LATIN SMALL LETTER
|
||||
0x1ECD: 'o', // WITH DOT BELOW, LATIN SMALL LETTER
|
||||
0x0151: 'o', // WITH DOUBLE ACUTE, LATIN SMALL LETTER
|
||||
0x020D: 'o', // WITH DOUBLE GRAVE, LATIN SMALL LETTER
|
||||
0x00F2: 'o', // WITH GRAVE, LATIN SMALL LETTER
|
||||
0x1ECF: 'o', // WITH HOOK ABOVE, LATIN SMALL LETTER
|
||||
0x01A1: 'o', // WITH HORN, LATIN SMALL LETTER
|
||||
0x020F: 'o', // WITH INVERTED BREVE, LATIN SMALL LETTER
|
||||
0x014D: 'o', // WITH MACRON, LATIN SMALL LETTER
|
||||
0x01EB: 'o', // WITH OGONEK, LATIN SMALL LETTER
|
||||
0x00F8: 'o', // WITH STROKE, LATIN SMALL LETTER
|
||||
0x1D13: 'o', // WITH STROKE, LATIN SMALL LETTER SIDEWAYS
|
||||
0x00F5: 'o', // WITH TILDE, LATIN SMALL LETTER
|
||||
0x0366: 'o', // , COMBINING LATIN SMALL LETTER
|
||||
0x0275: 'o', // , LATIN SMALL LETTER BARRED
|
||||
0x1D17: 'o', // , LATIN SMALL LETTER BOTTOM HALF
|
||||
0x0254: 'o', // , LATIN SMALL LETTER OPEN
|
||||
0x1D11: 'o', // , LATIN SMALL LETTER SIDEWAYS
|
||||
0x1D12: 'o', // , LATIN SMALL LETTER SIDEWAYS OPEN
|
||||
0x1D16: 'o', // , LATIN SMALL LETTER TOP HALF
|
||||
0x1E55: 'p', // WITH ACUTE, LATIN SMALL LETTER
|
||||
0x1E57: 'p', // WITH DOT ABOVE, LATIN SMALL LETTER
|
||||
0x01A5: 'p', // WITH HOOK, LATIN SMALL LETTER
|
||||
0x209A: 'p', // , LATIN SUBSCRIPT SMALL LETTER
|
||||
0x024B: 'q', // WITH HOOK TAIL, LATIN SMALL LETTER
|
||||
0x02A0: 'q', // WITH HOOK, LATIN SMALL LETTER
|
||||
0x0155: 'r', // WITH ACUTE, LATIN SMALL LETTER
|
||||
0x0159: 'r', // WITH CARON, LATIN SMALL LETTER
|
||||
0x0157: 'r', // WITH CEDILLA, LATIN SMALL LETTER
|
||||
0x1E59: 'r', // WITH DOT ABOVE, LATIN SMALL LETTER
|
||||
0x1E5B: 'r', // WITH DOT BELOW, LATIN SMALL LETTER
|
||||
0x0211: 'r', // WITH DOUBLE GRAVE, LATIN SMALL LETTER
|
||||
0x027E: 'r', // WITH FISHHOOK, LATIN SMALL LETTER
|
||||
0x027F: 'r', // WITH FISHHOOK, LATIN SMALL LETTER REVERSED
|
||||
0x027B: 'r', // WITH HOOK, LATIN SMALL LETTER TURNED
|
||||
0x0213: 'r', // WITH INVERTED BREVE, LATIN SMALL LETTER
|
||||
0x1E5F: 'r', // WITH LINE BELOW, LATIN SMALL LETTER
|
||||
0x027C: 'r', // WITH LONG LEG, LATIN SMALL LETTER
|
||||
0x027A: 'r', // WITH LONG LEG, LATIN SMALL LETTER TURNED
|
||||
0x024D: 'r', // WITH STROKE, LATIN SMALL LETTER
|
||||
0x027D: 'r', // WITH TAIL, LATIN SMALL LETTER
|
||||
0x036C: 'r', // , COMBINING LATIN SMALL LETTER
|
||||
0x0279: 'r', // , LATIN SMALL LETTER TURNED
|
||||
0x1D63: 'r', // , LATIN SUBSCRIPT SMALL LETTER
|
||||
0x015B: 's', // WITH ACUTE, LATIN SMALL LETTER
|
||||
0x0161: 's', // WITH CARON, LATIN SMALL LETTER
|
||||
0x015F: 's', // WITH CEDILLA, LATIN SMALL LETTER
|
||||
0x015D: 's', // WITH CIRCUMFLEX, LATIN SMALL LETTER
|
||||
0x0219: 's', // WITH COMMA BELOW, LATIN SMALL LETTER
|
||||
0x1E61: 's', // WITH DOT ABOVE, LATIN SMALL LETTER
|
||||
0x1E9B: 's', // WITH DOT ABOVE, LATIN SMALL LETTER LONG
|
||||
0x1E63: 's', // WITH DOT BELOW, LATIN SMALL LETTER
|
||||
0x0282: 's', // WITH HOOK, LATIN SMALL LETTER
|
||||
0x023F: 's', // WITH SWASH TAIL, LATIN SMALL LETTER
|
||||
0x017F: 's', // , LATIN SMALL LETTER LONG
|
||||
0x00DF: 's', // , LATIN SMALL LETTER SHARP
|
||||
0x209B: 's', // , LATIN SUBSCRIPT SMALL LETTER
|
||||
0x0165: 't', // WITH CARON, LATIN SMALL LETTER
|
||||
0x0163: 't', // WITH CEDILLA, LATIN SMALL LETTER
|
||||
0x1E71: 't', // WITH CIRCUMFLEX BELOW, LATIN SMALL LETTER
|
||||
0x021B: 't', // WITH COMMA BELOW, LATIN SMALL LETTER
|
||||
0x0236: 't', // WITH CURL, LATIN SMALL LETTER
|
||||
0x1E97: 't', // WITH DIAERESIS, LATIN SMALL LETTER
|
||||
0x1E6B: 't', // WITH DOT ABOVE, LATIN SMALL LETTER
|
||||
0x1E6D: 't', // WITH DOT BELOW, LATIN SMALL LETTER
|
||||
0x01AD: 't', // WITH HOOK, LATIN SMALL LETTER
|
||||
0x1E6F: 't', // WITH LINE BELOW, LATIN SMALL LETTER
|
||||
0x01AB: 't', // WITH PALATAL HOOK, LATIN SMALL LETTER
|
||||
0x0288: 't', // WITH RETROFLEX HOOK, LATIN SMALL LETTER
|
||||
0x0167: 't', // WITH STROKE, LATIN SMALL LETTER
|
||||
0x036D: 't', // , COMBINING LATIN SMALL LETTER
|
||||
0x0287: 't', // , LATIN SMALL LETTER TURNED
|
||||
0x209C: 't', // , LATIN SUBSCRIPT SMALL LETTER
|
||||
0x0289: 'u', // BAR, LATIN SMALL LETTER
|
||||
0x00FA: 'u', // WITH ACUTE, LATIN SMALL LETTER
|
||||
0x016D: 'u', // WITH BREVE, LATIN SMALL LETTER
|
||||
0x01D4: 'u', // WITH CARON, LATIN SMALL LETTER
|
||||
0x1E77: 'u', // WITH CIRCUMFLEX BELOW, LATIN SMALL LETTER
|
||||
0x00FB: 'u', // WITH CIRCUMFLEX, LATIN SMALL LETTER
|
||||
0x1E73: 'u', // WITH DIAERESIS BELOW, LATIN SMALL LETTER
|
||||
0x00FC: 'u', // WITH DIAERESIS, LATIN SMALL LETTER
|
||||
0x1EE5: 'u', // WITH DOT BELOW, LATIN SMALL LETTER
|
||||
0x0171: 'u', // WITH DOUBLE ACUTE, LATIN SMALL LETTER
|
||||
0x0215: 'u', // WITH DOUBLE GRAVE, LATIN SMALL LETTER
|
||||
0x00F9: 'u', // WITH GRAVE, LATIN SMALL LETTER
|
||||
0x1EE7: 'u', // WITH HOOK ABOVE, LATIN SMALL LETTER
|
||||
0x01B0: 'u', // WITH HORN, LATIN SMALL LETTER
|
||||
0x0217: 'u', // WITH INVERTED BREVE, LATIN SMALL LETTER
|
||||
0x016B: 'u', // WITH MACRON, LATIN SMALL LETTER
|
||||
0x0173: 'u', // WITH OGONEK, LATIN SMALL LETTER
|
||||
0x016F: 'u', // WITH RING ABOVE, LATIN SMALL LETTER
|
||||
0x1E75: 'u', // WITH TILDE BELOW, LATIN SMALL LETTER
|
||||
0x0169: 'u', // WITH TILDE, LATIN SMALL LETTER
|
||||
0x0367: 'u', // , COMBINING LATIN SMALL LETTER
|
||||
0x1D1D: 'u', // , LATIN SMALL LETTER SIDEWAYS
|
||||
0x1D1E: 'u', // , LATIN SMALL LETTER SIDEWAYS DIAERESIZED
|
||||
0x1D64: 'u', // , LATIN SUBSCRIPT SMALL LETTER
|
||||
0x1E7F: 'v', // WITH DOT BELOW, LATIN SMALL LETTER
|
||||
0x028B: 'v', // WITH HOOK, LATIN SMALL LETTER
|
||||
0x1E7D: 'v', // WITH TILDE, LATIN SMALL LETTER
|
||||
0x036E: 'v', // , COMBINING LATIN SMALL LETTER
|
||||
0x028C: 'v', // , LATIN SMALL LETTER TURNED
|
||||
0x1D65: 'v', // , LATIN SUBSCRIPT SMALL LETTER
|
||||
0x1E83: 'w', // WITH ACUTE, LATIN SMALL LETTER
|
||||
0x0175: 'w', // WITH CIRCUMFLEX, LATIN SMALL LETTER
|
||||
0x1E85: 'w', // WITH DIAERESIS, LATIN SMALL LETTER
|
||||
0x1E87: 'w', // WITH DOT ABOVE, LATIN SMALL LETTER
|
||||
0x1E89: 'w', // WITH DOT BELOW, LATIN SMALL LETTER
|
||||
0x1E81: 'w', // WITH GRAVE, LATIN SMALL LETTER
|
||||
0x1E98: 'w', // WITH RING ABOVE, LATIN SMALL LETTER
|
||||
0x028D: 'w', // , LATIN SMALL LETTER TURNED
|
||||
0x1E8D: 'x', // WITH DIAERESIS, LATIN SMALL LETTER
|
||||
0x1E8B: 'x', // WITH DOT ABOVE, LATIN SMALL LETTER
|
||||
0x036F: 'x', // , COMBINING LATIN SMALL LETTER
|
||||
0x00FD: 'y', // WITH ACUTE, LATIN SMALL LETTER
|
||||
0x0177: 'y', // WITH CIRCUMFLEX, LATIN SMALL LETTER
|
||||
0x00FF: 'y', // WITH DIAERESIS, LATIN SMALL LETTER
|
||||
0x1E8F: 'y', // WITH DOT ABOVE, LATIN SMALL LETTER
|
||||
0x1EF5: 'y', // WITH DOT BELOW, LATIN SMALL LETTER
|
||||
0x1EF3: 'y', // WITH GRAVE, LATIN SMALL LETTER
|
||||
0x1EF7: 'y', // WITH HOOK ABOVE, LATIN SMALL LETTER
|
||||
0x01B4: 'y', // WITH HOOK, LATIN SMALL LETTER
|
||||
0x0233: 'y', // WITH MACRON, LATIN SMALL LETTER
|
||||
0x1E99: 'y', // WITH RING ABOVE, LATIN SMALL LETTER
|
||||
0x024F: 'y', // WITH STROKE, LATIN SMALL LETTER
|
||||
0x1EF9: 'y', // WITH TILDE, LATIN SMALL LETTER
|
||||
0x028E: 'y', // , LATIN SMALL LETTER TURNED
|
||||
0x017A: 'z', // WITH ACUTE, LATIN SMALL LETTER
|
||||
0x017E: 'z', // WITH CARON, LATIN SMALL LETTER
|
||||
0x1E91: 'z', // WITH CIRCUMFLEX, LATIN SMALL LETTER
|
||||
0x0291: 'z', // WITH CURL, LATIN SMALL LETTER
|
||||
0x017C: 'z', // WITH DOT ABOVE, LATIN SMALL LETTER
|
||||
0x1E93: 'z', // WITH DOT BELOW, LATIN SMALL LETTER
|
||||
0x0225: 'z', // WITH HOOK, LATIN SMALL LETTER
|
||||
0x1E95: 'z', // WITH LINE BELOW, LATIN SMALL LETTER
|
||||
0x0290: 'z', // WITH RETROFLEX HOOK, LATIN SMALL LETTER
|
||||
0x01B6: 'z', // WITH STROKE, LATIN SMALL LETTER
|
||||
0x0240: 'z', // WITH SWASH TAIL, LATIN SMALL LETTER
|
||||
0x0251: 'a', // , latin small letter script
|
||||
0x00C1: 'A', // WITH ACUTE, LATIN CAPITAL LETTER
|
||||
0x00C2: 'A', // WITH CIRCUMFLEX, LATIN CAPITAL LETTER
|
||||
0x00C4: 'A', // WITH DIAERESIS, LATIN CAPITAL LETTER
|
||||
0x00C0: 'A', // WITH GRAVE, LATIN CAPITAL LETTER
|
||||
0x00C5: 'A', // WITH RING ABOVE, LATIN CAPITAL LETTER
|
||||
0x023A: 'A', // WITH STROKE, LATIN CAPITAL LETTER
|
||||
0x00C3: 'A', // WITH TILDE, LATIN CAPITAL LETTER
|
||||
0x1D00: 'A', // , LATIN LETTER SMALL CAPITAL
|
||||
0x0181: 'B', // WITH HOOK, LATIN CAPITAL LETTER
|
||||
0x0243: 'B', // WITH STROKE, LATIN CAPITAL LETTER
|
||||
0x0299: 'B', // , LATIN LETTER SMALL CAPITAL
|
||||
0x1D03: 'B', // , LATIN LETTER SMALL CAPITAL BARRED
|
||||
0x00C7: 'C', // WITH CEDILLA, LATIN CAPITAL LETTER
|
||||
0x023B: 'C', // WITH STROKE, LATIN CAPITAL LETTER
|
||||
0x1D04: 'C', // , LATIN LETTER SMALL CAPITAL
|
||||
0x018A: 'D', // WITH HOOK, LATIN CAPITAL LETTER
|
||||
0x0189: 'D', // , LATIN CAPITAL LETTER AFRICAN
|
||||
0x1D05: 'D', // , LATIN LETTER SMALL CAPITAL
|
||||
0x00C9: 'E', // WITH ACUTE, LATIN CAPITAL LETTER
|
||||
0x00CA: 'E', // WITH CIRCUMFLEX, LATIN CAPITAL LETTER
|
||||
0x00CB: 'E', // WITH DIAERESIS, LATIN CAPITAL LETTER
|
||||
0x00C8: 'E', // WITH GRAVE, LATIN CAPITAL LETTER
|
||||
0x0246: 'E', // WITH STROKE, LATIN CAPITAL LETTER
|
||||
0x0190: 'E', // , LATIN CAPITAL LETTER OPEN
|
||||
0x018E: 'E', // , LATIN CAPITAL LETTER REVERSED
|
||||
0x1D07: 'E', // , LATIN LETTER SMALL CAPITAL
|
||||
0x0193: 'G', // WITH HOOK, LATIN CAPITAL LETTER
|
||||
0x029B: 'G', // WITH HOOK, LATIN LETTER SMALL CAPITAL
|
||||
0x0262: 'G', // , LATIN LETTER SMALL CAPITAL
|
||||
0x029C: 'H', // , LATIN LETTER SMALL CAPITAL
|
||||
0x00CD: 'I', // WITH ACUTE, LATIN CAPITAL LETTER
|
||||
0x00CE: 'I', // WITH CIRCUMFLEX, LATIN CAPITAL LETTER
|
||||
0x00CF: 'I', // WITH DIAERESIS, LATIN CAPITAL LETTER
|
||||
0x0130: 'I', // WITH DOT ABOVE, LATIN CAPITAL LETTER
|
||||
0x00CC: 'I', // WITH GRAVE, LATIN CAPITAL LETTER
|
||||
0x0197: 'I', // WITH STROKE, LATIN CAPITAL LETTER
|
||||
0x026A: 'I', // , LATIN LETTER SMALL CAPITAL
|
||||
0x0248: 'J', // WITH STROKE, LATIN CAPITAL LETTER
|
||||
0x1D0A: 'J', // , LATIN LETTER SMALL CAPITAL
|
||||
0x1D0B: 'K', // , LATIN LETTER SMALL CAPITAL
|
||||
0x023D: 'L', // WITH BAR, LATIN CAPITAL LETTER
|
||||
0x1D0C: 'L', // WITH STROKE, LATIN LETTER SMALL CAPITAL
|
||||
0x029F: 'L', // , LATIN LETTER SMALL CAPITAL
|
||||
0x019C: 'M', // , LATIN CAPITAL LETTER TURNED
|
||||
0x1D0D: 'M', // , LATIN LETTER SMALL CAPITAL
|
||||
0x019D: 'N', // WITH LEFT HOOK, LATIN CAPITAL LETTER
|
||||
0x0220: 'N', // WITH LONG RIGHT LEG, LATIN CAPITAL LETTER
|
||||
0x00D1: 'N', // WITH TILDE, LATIN CAPITAL LETTER
|
||||
0x0274: 'N', // , LATIN LETTER SMALL CAPITAL
|
||||
0x1D0E: 'N', // , LATIN LETTER SMALL CAPITAL REVERSED
|
||||
0x00D3: 'O', // WITH ACUTE, LATIN CAPITAL LETTER
|
||||
0x00D4: 'O', // WITH CIRCUMFLEX, LATIN CAPITAL LETTER
|
||||
0x00D6: 'O', // WITH DIAERESIS, LATIN CAPITAL LETTER
|
||||
0x00D2: 'O', // WITH GRAVE, LATIN CAPITAL LETTER
|
||||
0x019F: 'O', // WITH MIDDLE TILDE, LATIN CAPITAL LETTER
|
||||
0x00D8: 'O', // WITH STROKE, LATIN CAPITAL LETTER
|
||||
0x00D5: 'O', // WITH TILDE, LATIN CAPITAL LETTER
|
||||
0x0186: 'O', // , LATIN CAPITAL LETTER OPEN
|
||||
0x1D0F: 'O', // , LATIN LETTER SMALL CAPITAL
|
||||
0x1D10: 'O', // , LATIN LETTER SMALL CAPITAL OPEN
|
||||
0x1D18: 'P', // , LATIN LETTER SMALL CAPITAL
|
||||
0x024A: 'Q', // WITH HOOK TAIL, LATIN CAPITAL LETTER SMALL
|
||||
0x024C: 'R', // WITH STROKE, LATIN CAPITAL LETTER
|
||||
0x0280: 'R', // , LATIN LETTER SMALL CAPITAL
|
||||
0x0281: 'R', // , LATIN LETTER SMALL CAPITAL INVERTED
|
||||
0x1D19: 'R', // , LATIN LETTER SMALL CAPITAL REVERSED
|
||||
0x1D1A: 'R', // , LATIN LETTER SMALL CAPITAL TURNED
|
||||
0x023E: 'T', // WITH DIAGONAL STROKE, LATIN CAPITAL LETTER
|
||||
0x01AE: 'T', // WITH RETROFLEX HOOK, LATIN CAPITAL LETTER
|
||||
0x1D1B: 'T', // , LATIN LETTER SMALL CAPITAL
|
||||
0x0244: 'U', // BAR, LATIN CAPITAL LETTER
|
||||
0x00DA: 'U', // WITH ACUTE, LATIN CAPITAL LETTER
|
||||
0x00DB: 'U', // WITH CIRCUMFLEX, LATIN CAPITAL LETTER
|
||||
0x00DC: 'U', // WITH DIAERESIS, LATIN CAPITAL LETTER
|
||||
0x00D9: 'U', // WITH GRAVE, LATIN CAPITAL LETTER
|
||||
0x1D1C: 'U', // , LATIN LETTER SMALL CAPITAL
|
||||
0x01B2: 'V', // WITH HOOK, LATIN CAPITAL LETTER
|
||||
0x0245: 'V', // , LATIN CAPITAL LETTER TURNED
|
||||
0x1D20: 'V', // , LATIN LETTER SMALL CAPITAL
|
||||
0x1D21: 'W', // , LATIN LETTER SMALL CAPITAL
|
||||
0x00DD: 'Y', // WITH ACUTE, LATIN CAPITAL LETTER
|
||||
0x0178: 'Y', // WITH DIAERESIS, LATIN CAPITAL LETTER
|
||||
0x024E: 'Y', // WITH STROKE, LATIN CAPITAL LETTER
|
||||
0x028F: 'Y', // , LATIN LETTER SMALL CAPITAL
|
||||
0x1D22: 'Z', // , LATIN LETTER SMALL CAPITAL
|
||||
}
|
||||
@ -143,7 +143,7 @@ func Run(opts *Options) {
|
||||
}
|
||||
patternBuilder := func(runes []rune) *Pattern {
|
||||
return BuildPattern(
|
||||
opts.Fuzzy, opts.FuzzyAlgo, opts.Extended, opts.Case, forward,
|
||||
opts.Fuzzy, opts.FuzzyAlgo, opts.Extended, opts.Case, opts.Normalize, forward,
|
||||
opts.Filter == nil, opts.Nth, opts.Delimiter, runes)
|
||||
}
|
||||
matcher := NewMatcher(patternBuilder, sort, opts.Tac, eventBox)
|
||||
|
||||
@ -24,6 +24,7 @@ const usage = `usage: fzf [options]
|
||||
--algo=TYPE Fuzzy matching algorithm: [v1|v2] (default: v2)
|
||||
-i Case-insensitive match (default: smart-case match)
|
||||
+i Case-sensitive match
|
||||
--normalize Normalize latin script letters before matching
|
||||
-n, --nth=N[,..] Comma-separated list of field index expressions
|
||||
for limiting search scope. Each can be a non-zero
|
||||
integer or a range expression ([BEGIN]..[END]).
|
||||
@ -138,6 +139,7 @@ type Options struct {
|
||||
FuzzyAlgo algo.Algo
|
||||
Extended bool
|
||||
Case Case
|
||||
Normalize bool
|
||||
Nth []Range
|
||||
WithNth []Range
|
||||
Delimiter Delimiter
|
||||
@ -185,6 +187,7 @@ func defaultOptions() *Options {
|
||||
FuzzyAlgo: algo.FuzzyMatchV2,
|
||||
Extended: true,
|
||||
Case: CaseSmart,
|
||||
Normalize: false,
|
||||
Nth: make([]Range, 0),
|
||||
WithNth: make([]Range, 0),
|
||||
Delimiter: Delimiter{},
|
||||
@ -887,6 +890,10 @@ func parseOptions(opts *Options, allArgs []string) {
|
||||
case "-f", "--filter":
|
||||
filter := nextString(allArgs, &i, "query string required")
|
||||
opts.Filter = &filter
|
||||
case "--normalize":
|
||||
opts.Normalize = true
|
||||
case "--no-normalize":
|
||||
opts.Normalize = false
|
||||
case "--algo":
|
||||
opts.FuzzyAlgo = parseAlgo(nextString(allArgs, &i, "algorithm required (v1|v2)"))
|
||||
case "--expect":
|
||||
|
||||
@ -43,6 +43,7 @@ type Pattern struct {
|
||||
fuzzyAlgo algo.Algo
|
||||
extended bool
|
||||
caseSensitive bool
|
||||
normalize bool
|
||||
forward bool
|
||||
text []rune
|
||||
termSets []termSet
|
||||
@ -75,7 +76,7 @@ func clearChunkCache() {
|
||||
}
|
||||
|
||||
// BuildPattern builds Pattern object from the given arguments
|
||||
func BuildPattern(fuzzy bool, fuzzyAlgo algo.Algo, extended bool, caseMode Case, forward bool,
|
||||
func BuildPattern(fuzzy bool, fuzzyAlgo algo.Algo, extended bool, caseMode Case, normalize bool, forward bool,
|
||||
cacheable bool, nth []Range, delimiter Delimiter, runes []rune) *Pattern {
|
||||
|
||||
var asString string
|
||||
@ -120,6 +121,7 @@ func BuildPattern(fuzzy bool, fuzzyAlgo algo.Algo, extended bool, caseMode Case,
|
||||
fuzzyAlgo: fuzzyAlgo,
|
||||
extended: extended,
|
||||
caseSensitive: caseSensitive,
|
||||
normalize: normalize,
|
||||
forward: forward,
|
||||
text: []rune(asString),
|
||||
termSets: termSets,
|
||||
@ -309,9 +311,9 @@ func (p *Pattern) MatchItem(item *Item, withPos bool, slab *util.Slab) (*Result,
|
||||
func (p *Pattern) basicMatch(item *Item, withPos bool, slab *util.Slab) (Offset, int, int, *[]int) {
|
||||
input := p.prepareInput(item)
|
||||
if p.fuzzy {
|
||||
return p.iter(p.fuzzyAlgo, input, p.caseSensitive, p.forward, p.text, withPos, slab)
|
||||
return p.iter(p.fuzzyAlgo, input, p.caseSensitive, p.normalize, p.forward, p.text, withPos, slab)
|
||||
}
|
||||
return p.iter(algo.ExactMatchNaive, input, p.caseSensitive, p.forward, p.text, withPos, slab)
|
||||
return p.iter(algo.ExactMatchNaive, input, p.caseSensitive, p.normalize, p.forward, p.text, withPos, slab)
|
||||
}
|
||||
|
||||
func (p *Pattern) extendedMatch(item *Item, withPos bool, slab *util.Slab) ([]Offset, int, int, *[]int) {
|
||||
@ -330,7 +332,7 @@ func (p *Pattern) extendedMatch(item *Item, withPos bool, slab *util.Slab) ([]Of
|
||||
matched := false
|
||||
for _, term := range termSet {
|
||||
pfun := p.procFun[term.typ]
|
||||
off, score, tLen, pos := p.iter(pfun, input, term.caseSensitive, p.forward, term.text, withPos, slab)
|
||||
off, score, tLen, pos := p.iter(pfun, input, term.caseSensitive, p.normalize, p.forward, term.text, withPos, slab)
|
||||
if sidx := off[0]; sidx >= 0 {
|
||||
if term.inv {
|
||||
continue
|
||||
@ -378,9 +380,9 @@ func (p *Pattern) prepareInput(item *Item) []Token {
|
||||
return ret
|
||||
}
|
||||
|
||||
func (p *Pattern) iter(pfun algo.Algo, tokens []Token, caseSensitive bool, forward bool, pattern []rune, withPos bool, slab *util.Slab) (Offset, int, int, *[]int) {
|
||||
func (p *Pattern) iter(pfun algo.Algo, tokens []Token, caseSensitive bool, normalize bool, forward bool, pattern []rune, withPos bool, slab *util.Slab) (Offset, int, int, *[]int) {
|
||||
for _, part := range tokens {
|
||||
if res, pos := pfun(caseSensitive, forward, *part.text, pattern, withPos, slab); res.Start >= 0 {
|
||||
if res, pos := pfun(caseSensitive, normalize, forward, *part.text, pattern, withPos, slab); res.Start >= 0 {
|
||||
sidx := int32(res.Start) + part.prefixLength
|
||||
eidx := int32(res.End) + part.prefixLength
|
||||
if pos != nil {
|
||||
|
||||
@ -75,10 +75,10 @@ func TestParseTermsEmpty(t *testing.T) {
|
||||
func TestExact(t *testing.T) {
|
||||
defer clearPatternCache()
|
||||
clearPatternCache()
|
||||
pattern := BuildPattern(true, algo.FuzzyMatchV2, true, CaseSmart, true, true,
|
||||
pattern := BuildPattern(true, algo.FuzzyMatchV2, true, CaseSmart, false, true, true,
|
||||
[]Range{}, Delimiter{}, []rune("'abc"))
|
||||
res, pos := algo.ExactMatchNaive(
|
||||
pattern.caseSensitive, pattern.forward, util.RunesToChars([]rune("aabbcc abc")), pattern.termSets[0][0].text, true, nil)
|
||||
pattern.caseSensitive, pattern.normalize, pattern.forward, util.RunesToChars([]rune("aabbcc abc")), pattern.termSets[0][0].text, true, nil)
|
||||
if res.Start != 7 || res.End != 10 {
|
||||
t.Errorf("%s / %d / %d", pattern.termSets, res.Start, res.End)
|
||||
}
|
||||
@ -90,11 +90,11 @@ func TestExact(t *testing.T) {
|
||||
func TestEqual(t *testing.T) {
|
||||
defer clearPatternCache()
|
||||
clearPatternCache()
|
||||
pattern := BuildPattern(true, algo.FuzzyMatchV2, true, CaseSmart, true, true, []Range{}, Delimiter{}, []rune("^AbC$"))
|
||||
pattern := BuildPattern(true, algo.FuzzyMatchV2, true, CaseSmart, false, true, true, []Range{}, Delimiter{}, []rune("^AbC$"))
|
||||
|
||||
match := func(str string, sidxExpected int, eidxExpected int) {
|
||||
res, pos := algo.EqualMatch(
|
||||
pattern.caseSensitive, pattern.forward, util.RunesToChars([]rune(str)), pattern.termSets[0][0].text, true, nil)
|
||||
pattern.caseSensitive, pattern.normalize, pattern.forward, util.RunesToChars([]rune(str)), pattern.termSets[0][0].text, true, nil)
|
||||
if res.Start != sidxExpected || res.End != eidxExpected {
|
||||
t.Errorf("%s / %d / %d", pattern.termSets, res.Start, res.End)
|
||||
}
|
||||
@ -109,17 +109,17 @@ func TestEqual(t *testing.T) {
|
||||
func TestCaseSensitivity(t *testing.T) {
|
||||
defer clearPatternCache()
|
||||
clearPatternCache()
|
||||
pat1 := BuildPattern(true, algo.FuzzyMatchV2, false, CaseSmart, true, true, []Range{}, Delimiter{}, []rune("abc"))
|
||||
pat1 := BuildPattern(true, algo.FuzzyMatchV2, false, CaseSmart, false, true, true, []Range{}, Delimiter{}, []rune("abc"))
|
||||
clearPatternCache()
|
||||
pat2 := BuildPattern(true, algo.FuzzyMatchV2, false, CaseSmart, true, true, []Range{}, Delimiter{}, []rune("Abc"))
|
||||
pat2 := BuildPattern(true, algo.FuzzyMatchV2, false, CaseSmart, false, true, true, []Range{}, Delimiter{}, []rune("Abc"))
|
||||
clearPatternCache()
|
||||
pat3 := BuildPattern(true, algo.FuzzyMatchV2, false, CaseIgnore, true, true, []Range{}, Delimiter{}, []rune("abc"))
|
||||
pat3 := BuildPattern(true, algo.FuzzyMatchV2, false, CaseIgnore, false, true, true, []Range{}, Delimiter{}, []rune("abc"))
|
||||
clearPatternCache()
|
||||
pat4 := BuildPattern(true, algo.FuzzyMatchV2, false, CaseIgnore, true, true, []Range{}, Delimiter{}, []rune("Abc"))
|
||||
pat4 := BuildPattern(true, algo.FuzzyMatchV2, false, CaseIgnore, false, true, true, []Range{}, Delimiter{}, []rune("Abc"))
|
||||
clearPatternCache()
|
||||
pat5 := BuildPattern(true, algo.FuzzyMatchV2, false, CaseRespect, true, true, []Range{}, Delimiter{}, []rune("abc"))
|
||||
pat5 := BuildPattern(true, algo.FuzzyMatchV2, false, CaseRespect, false, true, true, []Range{}, Delimiter{}, []rune("abc"))
|
||||
clearPatternCache()
|
||||
pat6 := BuildPattern(true, algo.FuzzyMatchV2, false, CaseRespect, true, true, []Range{}, Delimiter{}, []rune("Abc"))
|
||||
pat6 := BuildPattern(true, algo.FuzzyMatchV2, false, CaseRespect, false, true, true, []Range{}, Delimiter{}, []rune("Abc"))
|
||||
|
||||
if string(pat1.text) != "abc" || pat1.caseSensitive != false ||
|
||||
string(pat2.text) != "Abc" || pat2.caseSensitive != true ||
|
||||
@ -132,7 +132,7 @@ func TestCaseSensitivity(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestOrigTextAndTransformed(t *testing.T) {
|
||||
pattern := BuildPattern(true, algo.FuzzyMatchV2, true, CaseSmart, true, true, []Range{}, Delimiter{}, []rune("jg"))
|
||||
pattern := BuildPattern(true, algo.FuzzyMatchV2, true, CaseSmart, false, true, true, []Range{}, Delimiter{}, []rune("jg"))
|
||||
tokens := Tokenize(util.RunesToChars([]rune("junegunn")), Delimiter{})
|
||||
trans := Transform(tokens, []Range{Range{1, 1}})
|
||||
|
||||
@ -167,7 +167,7 @@ func TestOrigTextAndTransformed(t *testing.T) {
|
||||
|
||||
func TestCacheKey(t *testing.T) {
|
||||
test := func(extended bool, patStr string, expected string, cacheable bool) {
|
||||
pat := BuildPattern(true, algo.FuzzyMatchV2, extended, CaseSmart, true, true, []Range{}, Delimiter{}, []rune(patStr))
|
||||
pat := BuildPattern(true, algo.FuzzyMatchV2, extended, CaseSmart, false, true, true, []Range{}, Delimiter{}, []rune(patStr))
|
||||
if pat.CacheKey() != expected {
|
||||
t.Errorf("Expected: %s, actual: %s", expected, pat.CacheKey())
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user