From 45793d75c2d168d51be3578b8d785159ec81ff92 Mon Sep 17 00:00:00 2001 From: Junegunn Choi Date: Mon, 9 Jan 2017 03:12:23 +0900 Subject: [PATCH] Add --normalize option to normalize latin script characters Close #790 --- CHANGELOG.md | 2 + man/man1/fzf.1 | 4 + src/algo/algo.go | 119 ++++++++++-- src/algo/algo_test.go | 21 ++- src/algo/normalize.go | 408 ++++++++++++++++++++++++++++++++++++++++++ src/core.go | 2 +- src/options.go | 7 + src/pattern.go | 14 +- src/pattern_test.go | 24 +-- 9 files changed, 563 insertions(+), 38 deletions(-) create mode 100644 src/algo/normalize.go diff --git a/CHANGELOG.md b/CHANGELOG.md index f78270c..fbc1e41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ CHANGELOG - Added `--height HEIGHT[%]` option - Preview window will truncate long lines by default. Line wrap can be enabled by `:wrap` flag in `--preview-window`. +- Added `--normalize` option to normalize latin script letters before + matching. e.g. `sodanco` can match `Só Danço Samba`. 0.15.9 ------ diff --git a/man/man1/fzf.1 b/man/man1/fzf.1 index b6540e1..1bcfb63 100644 --- a/man/man1/fzf.1 +++ b/man/man1/fzf.1 @@ -48,6 +48,10 @@ Case-insensitive match (default: smart-case match) .B "+i" Case-sensitive match .TP +.B "--normalize" +Normalize latin script letters before matching. This is not enabled by default +to avoid performance overhead. +.TP .BI "--algo=" TYPE Fuzzy matching algorithm (default: v2) diff --git a/src/algo/algo.go b/src/algo/algo.go index 1b85594..2a3bc9d 100644 --- a/src/algo/algo.go +++ b/src/algo/algo.go @@ -234,9 +234,36 @@ func bonusAt(input util.Chars, idx int) int16 { return bonusFor(charClassOf(input.Get(idx-1)), charClassOf(input.Get(idx))) } -type Algo func(caseSensitive bool, forward bool, input util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) +func normalizeRune(r rune) rune { + if r < 0x00C0 || r > 0x2184 { + return r + } -func FuzzyMatchV2(caseSensitive bool, forward bool, input util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) { + n := normalized[r] + if n > 0 { + return n + } + return r +} + +func normalizeRunes(runes []rune) []rune { + ret := make([]rune, len(runes)) + copy(ret, runes) + for idx, r := range runes { + if r < 0x00C0 || r > 0x2184 { + continue + } + n := normalized[r] + if n > 0 { + ret[idx] = normalized[r] + } + } + return ret +} + +type Algo func(caseSensitive bool, normalize bool, forward bool, input util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) + +func FuzzyMatchV2(caseSensitive bool, normalize bool, forward bool, input util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) { // Assume that pattern is given in lowercase if case-insensitive. // First check if there's a match and calculate bonus for each position. // If the input string is too long, consider finding the matching chars in @@ -247,13 +274,17 @@ func FuzzyMatchV2(caseSensitive bool, forward bool, input util.Chars, pattern [] case 0: return Result{0, 0, 0}, posArray(withPos, M) case 1: - return ExactMatchNaive(caseSensitive, forward, input, pattern[0:1], withPos, slab) + return ExactMatchNaive(caseSensitive, normalize, forward, input, pattern[0:1], withPos, slab) } // Since O(nm) algorithm can be prohibitively expensive for large input, // we fall back to the greedy algorithm. if slab != nil && N*M > cap(slab.I16) { - return FuzzyMatchV1(caseSensitive, forward, input, pattern, withPos, slab) + return FuzzyMatchV1(caseSensitive, normalize, forward, input, pattern, withPos, slab) + } + + if normalize { + pattern = normalizeRunes(pattern) } // Reuse pre-allocated integer slice to avoid unnecessary sweeping of garbages @@ -285,6 +316,10 @@ func FuzzyMatchV2(caseSensitive bool, forward bool, input util.Chars, pattern [] } } + if normalize { + char = normalizeRune(char) + } + T[idx] = char B[idx] = bonusFor(prevClass, class) prevClass = class @@ -432,7 +467,7 @@ func FuzzyMatchV2(caseSensitive bool, forward bool, input util.Chars, pattern [] } // Implement the same sorting criteria as V2 -func calculateScore(caseSensitive bool, text util.Chars, pattern []rune, sidx int, eidx int, withPos bool) (int, *[]int) { +func calculateScore(caseSensitive bool, normalize bool, text util.Chars, pattern []rune, sidx int, eidx int, withPos bool) (int, *[]int) { pidx, score, inGap, consecutive, firstBonus := 0, 0, false, 0, int16(0) pos := posArray(withPos, len(pattern)) prevClass := charNonWord @@ -449,6 +484,10 @@ func calculateScore(caseSensitive bool, text util.Chars, pattern []rune, sidx in char = unicode.To(unicode.LowerCase, char) } } + // pattern is already normalized + if normalize { + char = normalizeRune(char) + } if char == pattern[pidx] { if withPos { *pos = append(*pos, idx) @@ -488,7 +527,7 @@ func calculateScore(caseSensitive bool, text util.Chars, pattern []rune, sidx in } // FuzzyMatchV1 performs fuzzy-match -func FuzzyMatchV1(caseSensitive bool, forward bool, text util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) { +func FuzzyMatchV1(caseSensitive bool, normalize bool, forward bool, text util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) { if len(pattern) == 0 { return Result{0, 0, 0}, nil } @@ -500,6 +539,10 @@ func FuzzyMatchV1(caseSensitive bool, forward bool, text util.Chars, pattern []r lenRunes := text.Length() lenPattern := len(pattern) + if normalize { + pattern = normalizeRunes(pattern) + } + for index := 0; index < lenRunes; index++ { char := text.Get(indexAt(index, lenRunes, forward)) // This is considerably faster than blindly applying strings.ToLower to the @@ -514,6 +557,9 @@ func FuzzyMatchV1(caseSensitive bool, forward bool, text util.Chars, pattern []r char = unicode.To(unicode.LowerCase, char) } } + if normalize { + char = normalizeRune(char) + } pchar := pattern[indexAt(pidx, lenPattern, forward)] if char == pchar { if sidx < 0 { @@ -553,7 +599,7 @@ func FuzzyMatchV1(caseSensitive bool, forward bool, text util.Chars, pattern []r sidx, eidx = lenRunes-eidx, lenRunes-sidx } - score, pos := calculateScore(caseSensitive, text, pattern, sidx, eidx, withPos) + score, pos := calculateScore(caseSensitive, normalize, text, pattern, sidx, eidx, withPos) return Result{sidx, eidx, score}, pos } return Result{-1, -1, 0}, nil @@ -568,7 +614,7 @@ func FuzzyMatchV1(caseSensitive bool, forward bool, text util.Chars, pattern []r // bonus point, instead of stopping immediately after finding the first match. // The solution is much cheaper since there is only one possible alignment of // the pattern. -func ExactMatchNaive(caseSensitive bool, forward bool, text util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) { +func ExactMatchNaive(caseSensitive bool, normalize bool, forward bool, text util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) { if len(pattern) == 0 { return Result{0, 0, 0}, nil } @@ -580,6 +626,10 @@ func ExactMatchNaive(caseSensitive bool, forward bool, text util.Chars, pattern return Result{-1, -1, 0}, nil } + if normalize { + pattern = normalizeRunes(pattern) + } + // For simplicity, only look at the bonus at the first character position pidx := 0 bestPos, bonus, bestBonus := -1, int16(0), int16(-1) @@ -593,6 +643,9 @@ func ExactMatchNaive(caseSensitive bool, forward bool, text util.Chars, pattern char = unicode.To(unicode.LowerCase, char) } } + if normalize { + char = normalizeRune(char) + } pidx_ := indexAt(pidx, lenPattern, forward) pchar := pattern[pidx_] if pchar == char { @@ -624,14 +677,14 @@ func ExactMatchNaive(caseSensitive bool, forward bool, text util.Chars, pattern sidx = lenRunes - (bestPos + 1) eidx = lenRunes - (bestPos - lenPattern + 1) } - score, _ := calculateScore(caseSensitive, text, pattern, sidx, eidx, false) + score, _ := calculateScore(caseSensitive, normalize, text, pattern, sidx, eidx, false) return Result{sidx, eidx, score}, nil } return Result{-1, -1, 0}, nil } // PrefixMatch performs prefix-match -func PrefixMatch(caseSensitive bool, forward bool, text util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) { +func PrefixMatch(caseSensitive bool, normalize bool, forward bool, text util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) { if len(pattern) == 0 { return Result{0, 0, 0}, nil } @@ -640,22 +693,29 @@ func PrefixMatch(caseSensitive bool, forward bool, text util.Chars, pattern []ru return Result{-1, -1, 0}, nil } + if normalize { + pattern = normalizeRunes(pattern) + } + for index, r := range pattern { char := text.Get(index) if !caseSensitive { char = unicode.ToLower(char) } + if normalize { + char = normalizeRune(char) + } if char != r { return Result{-1, -1, 0}, nil } } lenPattern := len(pattern) - score, _ := calculateScore(caseSensitive, text, pattern, 0, lenPattern, false) + score, _ := calculateScore(caseSensitive, normalize, text, pattern, 0, lenPattern, false) return Result{0, lenPattern, score}, nil } // SuffixMatch performs suffix-match -func SuffixMatch(caseSensitive bool, forward bool, text util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) { +func SuffixMatch(caseSensitive bool, normalize bool, forward bool, text util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) { lenRunes := text.Length() trimmedLen := lenRunes - text.TrailingWhitespaces() if len(pattern) == 0 { @@ -666,11 +726,18 @@ func SuffixMatch(caseSensitive bool, forward bool, text util.Chars, pattern []ru return Result{-1, -1, 0}, nil } + if normalize { + pattern = normalizeRunes(pattern) + } + for index, r := range pattern { char := text.Get(index + diff) if !caseSensitive { char = unicode.ToLower(char) } + if normalize { + char = normalizeRune(char) + } if char != r { return Result{-1, -1, 0}, nil } @@ -678,21 +745,37 @@ func SuffixMatch(caseSensitive bool, forward bool, text util.Chars, pattern []ru lenPattern := len(pattern) sidx := trimmedLen - lenPattern eidx := trimmedLen - score, _ := calculateScore(caseSensitive, text, pattern, sidx, eidx, false) + score, _ := calculateScore(caseSensitive, normalize, text, pattern, sidx, eidx, false) return Result{sidx, eidx, score}, nil } // EqualMatch performs equal-match -func EqualMatch(caseSensitive bool, forward bool, text util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) { +func EqualMatch(caseSensitive bool, normalize bool, forward bool, text util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) { lenPattern := len(pattern) if text.Length() != lenPattern { return Result{-1, -1, 0}, nil } - runesStr := text.ToString() - if !caseSensitive { - runesStr = strings.ToLower(runesStr) + match := true + if normalize { + runes := text.ToRunes() + for idx, pchar := range pattern { + char := runes[idx] + if !caseSensitive { + char = unicode.To(unicode.LowerCase, char) + } + if normalizeRune(pchar) != normalizeRune(char) { + match = false + break + } + } + } else { + runesStr := text.ToString() + if !caseSensitive { + runesStr = strings.ToLower(runesStr) + } + match = runesStr == string(pattern) } - if runesStr == string(pattern) { + if match { return Result{0, lenPattern, (scoreMatch+bonusBoundary)*lenPattern + (bonusFirstCharMultiplier-1)*bonusBoundary}, nil } diff --git a/src/algo/algo_test.go b/src/algo/algo_test.go index fc24f6d..df8b227 100644 --- a/src/algo/algo_test.go +++ b/src/algo/algo_test.go @@ -10,10 +10,14 @@ import ( ) func assertMatch(t *testing.T, fun Algo, caseSensitive, forward bool, input, pattern string, sidx int, eidx int, score int) { + assertMatch2(t, fun, caseSensitive, false, forward, input, pattern, sidx, eidx, score) +} + +func assertMatch2(t *testing.T, fun Algo, caseSensitive, normalize, forward bool, input, pattern string, sidx int, eidx int, score int) { if !caseSensitive { pattern = strings.ToLower(pattern) } - res, pos := fun(caseSensitive, forward, util.RunesToChars([]rune(input)), []rune(pattern), true, nil) + res, pos := fun(caseSensitive, normalize, forward, util.RunesToChars([]rune(input)), []rune(pattern), true, nil) var start, end int if pos == nil || len(*pos) == 0 { start = res.Start @@ -156,6 +160,21 @@ func TestEmptyPattern(t *testing.T) { } } +func TestNormalize(t *testing.T) { + caseSensitive := false + normalize := true + forward := true + test := func(input, pattern string, sidx, eidx, score int, funs ...Algo) { + for _, fun := range funs { + assertMatch2(t, fun, caseSensitive, normalize, forward, + input, pattern, sidx, eidx, score) + } + } + test("Só Danço Samba", "So", 0, 2, 56, FuzzyMatchV1, FuzzyMatchV2, PrefixMatch, ExactMatchNaive) + test("Só Danço Samba", "sodc", 0, 7, 89, FuzzyMatchV1, FuzzyMatchV2) + test("Danço", "danco", 0, 5, 128, FuzzyMatchV1, FuzzyMatchV2, PrefixMatch, SuffixMatch, ExactMatchNaive, EqualMatch) +} + func TestLongString(t *testing.T) { bytes := make([]byte, math.MaxUint16*2) for i := range bytes { diff --git a/src/algo/normalize.go b/src/algo/normalize.go new file mode 100644 index 0000000..1168a64 --- /dev/null +++ b/src/algo/normalize.go @@ -0,0 +1,408 @@ +// Normalization of latin script letters +// Reference: http://www.unicode.org/Public/UCD/latest/ucd/Index.txt + +package algo + +var normalized map[rune]rune = map[rune]rune{ + 0x00E1: 'a', // WITH ACUTE, LATIN SMALL LETTER + 0x0103: 'a', // WITH BREVE, LATIN SMALL LETTER + 0x01CE: 'a', // WITH CARON, LATIN SMALL LETTER + 0x00E2: 'a', // WITH CIRCUMFLEX, LATIN SMALL LETTER + 0x00E4: 'a', // WITH DIAERESIS, LATIN SMALL LETTER + 0x0227: 'a', // WITH DOT ABOVE, LATIN SMALL LETTER + 0x1EA1: 'a', // WITH DOT BELOW, LATIN SMALL LETTER + 0x0201: 'a', // WITH DOUBLE GRAVE, LATIN SMALL LETTER + 0x00E0: 'a', // WITH GRAVE, LATIN SMALL LETTER + 0x1EA3: 'a', // WITH HOOK ABOVE, LATIN SMALL LETTER + 0x0203: 'a', // WITH INVERTED BREVE, LATIN SMALL LETTER + 0x0101: 'a', // WITH MACRON, LATIN SMALL LETTER + 0x0105: 'a', // WITH OGONEK, LATIN SMALL LETTER + 0x1E9A: 'a', // WITH RIGHT HALF RING, LATIN SMALL LETTER + 0x00E5: 'a', // WITH RING ABOVE, LATIN SMALL LETTER + 0x1E01: 'a', // WITH RING BELOW, LATIN SMALL LETTER + 0x00E3: 'a', // WITH TILDE, LATIN SMALL LETTER + 0x0363: 'a', // , COMBINING LATIN SMALL LETTER + 0x0250: 'a', // , LATIN SMALL LETTER TURNED + 0x1E03: 'b', // WITH DOT ABOVE, LATIN SMALL LETTER + 0x1E05: 'b', // WITH DOT BELOW, LATIN SMALL LETTER + 0x0253: 'b', // WITH HOOK, LATIN SMALL LETTER + 0x1E07: 'b', // WITH LINE BELOW, LATIN SMALL LETTER + 0x0180: 'b', // WITH STROKE, LATIN SMALL LETTER + 0x0183: 'b', // WITH TOPBAR, LATIN SMALL LETTER + 0x0107: 'c', // WITH ACUTE, LATIN SMALL LETTER + 0x010D: 'c', // WITH CARON, LATIN SMALL LETTER + 0x00E7: 'c', // WITH CEDILLA, LATIN SMALL LETTER + 0x0109: 'c', // WITH CIRCUMFLEX, LATIN SMALL LETTER + 0x0255: 'c', // WITH CURL, LATIN SMALL LETTER + 0x010B: 'c', // WITH DOT ABOVE, LATIN SMALL LETTER + 0x0188: 'c', // WITH HOOK, LATIN SMALL LETTER + 0x023C: 'c', // WITH STROKE, LATIN SMALL LETTER + 0x0368: 'c', // , COMBINING LATIN SMALL LETTER + 0x0297: 'c', // , LATIN LETTER STRETCHED + 0x2184: 'c', // , LATIN SMALL LETTER REVERSED + 0x010F: 'd', // WITH CARON, LATIN SMALL LETTER + 0x1E11: 'd', // WITH CEDILLA, LATIN SMALL LETTER + 0x1E13: 'd', // WITH CIRCUMFLEX BELOW, LATIN SMALL LETTER + 0x0221: 'd', // WITH CURL, LATIN SMALL LETTER + 0x1E0B: 'd', // WITH DOT ABOVE, LATIN SMALL LETTER + 0x1E0D: 'd', // WITH DOT BELOW, LATIN SMALL LETTER + 0x0257: 'd', // WITH HOOK, LATIN SMALL LETTER + 0x1E0F: 'd', // WITH LINE BELOW, LATIN SMALL LETTER + 0x0111: 'd', // WITH STROKE, LATIN SMALL LETTER + 0x0256: 'd', // WITH TAIL, LATIN SMALL LETTER + 0x018C: 'd', // WITH TOPBAR, LATIN SMALL LETTER + 0x0369: 'd', // , COMBINING LATIN SMALL LETTER + 0x00E9: 'e', // WITH ACUTE, LATIN SMALL LETTER + 0x0115: 'e', // WITH BREVE, LATIN SMALL LETTER + 0x011B: 'e', // WITH CARON, LATIN SMALL LETTER + 0x0229: 'e', // WITH CEDILLA, LATIN SMALL LETTER + 0x1E19: 'e', // WITH CIRCUMFLEX BELOW, LATIN SMALL LETTER + 0x00EA: 'e', // WITH CIRCUMFLEX, LATIN SMALL LETTER + 0x00EB: 'e', // WITH DIAERESIS, LATIN SMALL LETTER + 0x0117: 'e', // WITH DOT ABOVE, LATIN SMALL LETTER + 0x1EB9: 'e', // WITH DOT BELOW, LATIN SMALL LETTER + 0x0205: 'e', // WITH DOUBLE GRAVE, LATIN SMALL LETTER + 0x00E8: 'e', // WITH GRAVE, LATIN SMALL LETTER + 0x1EBB: 'e', // WITH HOOK ABOVE, LATIN SMALL LETTER + 0x025D: 'e', // WITH HOOK, LATIN SMALL LETTER REVERSED OPEN + 0x0207: 'e', // WITH INVERTED BREVE, LATIN SMALL LETTER + 0x0113: 'e', // WITH MACRON, LATIN SMALL LETTER + 0x0119: 'e', // WITH OGONEK, LATIN SMALL LETTER + 0x0247: 'e', // WITH STROKE, LATIN SMALL LETTER + 0x1E1B: 'e', // WITH TILDE BELOW, LATIN SMALL LETTER + 0x1EBD: 'e', // WITH TILDE, LATIN SMALL LETTER + 0x0364: 'e', // , COMBINING LATIN SMALL LETTER + 0x029A: 'e', // , LATIN SMALL LETTER CLOSED OPEN + 0x025E: 'e', // , LATIN SMALL LETTER CLOSED REVERSED OPEN + 0x025B: 'e', // , LATIN SMALL LETTER OPEN + 0x0258: 'e', // , LATIN SMALL LETTER REVERSED + 0x025C: 'e', // , LATIN SMALL LETTER REVERSED OPEN + 0x01DD: 'e', // , LATIN SMALL LETTER TURNED + 0x1D08: 'e', // , LATIN SMALL LETTER TURNED OPEN + 0x1E1F: 'f', // WITH DOT ABOVE, LATIN SMALL LETTER + 0x0192: 'f', // WITH HOOK, LATIN SMALL LETTER + 0x01F5: 'g', // WITH ACUTE, LATIN SMALL LETTER + 0x011F: 'g', // WITH BREVE, LATIN SMALL LETTER + 0x01E7: 'g', // WITH CARON, LATIN SMALL LETTER + 0x0123: 'g', // WITH CEDILLA, LATIN SMALL LETTER + 0x011D: 'g', // WITH CIRCUMFLEX, LATIN SMALL LETTER + 0x0121: 'g', // WITH DOT ABOVE, LATIN SMALL LETTER + 0x0260: 'g', // WITH HOOK, LATIN SMALL LETTER + 0x1E21: 'g', // WITH MACRON, LATIN SMALL LETTER + 0x01E5: 'g', // WITH STROKE, LATIN SMALL LETTER + 0x0261: 'g', // , LATIN SMALL LETTER SCRIPT + 0x1E2B: 'h', // WITH BREVE BELOW, LATIN SMALL LETTER + 0x021F: 'h', // WITH CARON, LATIN SMALL LETTER + 0x1E29: 'h', // WITH CEDILLA, LATIN SMALL LETTER + 0x0125: 'h', // WITH CIRCUMFLEX, LATIN SMALL LETTER + 0x1E27: 'h', // WITH DIAERESIS, LATIN SMALL LETTER + 0x1E23: 'h', // WITH DOT ABOVE, LATIN SMALL LETTER + 0x1E25: 'h', // WITH DOT BELOW, LATIN SMALL LETTER + 0x02AE: 'h', // WITH FISHHOOK, LATIN SMALL LETTER TURNED + 0x0266: 'h', // WITH HOOK, LATIN SMALL LETTER + 0x1E96: 'h', // WITH LINE BELOW, LATIN SMALL LETTER + 0x0127: 'h', // WITH STROKE, LATIN SMALL LETTER + 0x036A: 'h', // , COMBINING LATIN SMALL LETTER + 0x0265: 'h', // , LATIN SMALL LETTER TURNED + 0x2095: 'h', // , LATIN SUBSCRIPT SMALL LETTER + 0x00ED: 'i', // WITH ACUTE, LATIN SMALL LETTER + 0x012D: 'i', // WITH BREVE, LATIN SMALL LETTER + 0x01D0: 'i', // WITH CARON, LATIN SMALL LETTER + 0x00EE: 'i', // WITH CIRCUMFLEX, LATIN SMALL LETTER + 0x00EF: 'i', // WITH DIAERESIS, LATIN SMALL LETTER + 0x1ECB: 'i', // WITH DOT BELOW, LATIN SMALL LETTER + 0x0209: 'i', // WITH DOUBLE GRAVE, LATIN SMALL LETTER + 0x00EC: 'i', // WITH GRAVE, LATIN SMALL LETTER + 0x1EC9: 'i', // WITH HOOK ABOVE, LATIN SMALL LETTER + 0x020B: 'i', // WITH INVERTED BREVE, LATIN SMALL LETTER + 0x012B: 'i', // WITH MACRON, LATIN SMALL LETTER + 0x012F: 'i', // WITH OGONEK, LATIN SMALL LETTER + 0x0268: 'i', // WITH STROKE, LATIN SMALL LETTER + 0x1E2D: 'i', // WITH TILDE BELOW, LATIN SMALL LETTER + 0x0129: 'i', // WITH TILDE, LATIN SMALL LETTER + 0x0365: 'i', // , COMBINING LATIN SMALL LETTER + 0x0131: 'i', // , LATIN SMALL LETTER DOTLESS + 0x1D09: 'i', // , LATIN SMALL LETTER TURNED + 0x1D62: 'i', // , LATIN SUBSCRIPT SMALL LETTER + 0x2071: 'i', // , SUPERSCRIPT LATIN SMALL LETTER + 0x01F0: 'j', // WITH CARON, LATIN SMALL LETTER + 0x0135: 'j', // WITH CIRCUMFLEX, LATIN SMALL LETTER + 0x029D: 'j', // WITH CROSSED-TAIL, LATIN SMALL LETTER + 0x0249: 'j', // WITH STROKE, LATIN SMALL LETTER + 0x025F: 'j', // WITH STROKE, LATIN SMALL LETTER DOTLESS + 0x0237: 'j', // , LATIN SMALL LETTER DOTLESS + 0x1E31: 'k', // WITH ACUTE, LATIN SMALL LETTER + 0x01E9: 'k', // WITH CARON, LATIN SMALL LETTER + 0x0137: 'k', // WITH CEDILLA, LATIN SMALL LETTER + 0x1E33: 'k', // WITH DOT BELOW, LATIN SMALL LETTER + 0x0199: 'k', // WITH HOOK, LATIN SMALL LETTER + 0x1E35: 'k', // WITH LINE BELOW, LATIN SMALL LETTER + 0x029E: 'k', // , LATIN SMALL LETTER TURNED + 0x2096: 'k', // , LATIN SUBSCRIPT SMALL LETTER + 0x013A: 'l', // WITH ACUTE, LATIN SMALL LETTER + 0x019A: 'l', // WITH BAR, LATIN SMALL LETTER + 0x026C: 'l', // WITH BELT, LATIN SMALL LETTER + 0x013E: 'l', // WITH CARON, LATIN SMALL LETTER + 0x013C: 'l', // WITH CEDILLA, LATIN SMALL LETTER + 0x1E3D: 'l', // WITH CIRCUMFLEX BELOW, LATIN SMALL LETTER + 0x0234: 'l', // WITH CURL, LATIN SMALL LETTER + 0x1E37: 'l', // WITH DOT BELOW, LATIN SMALL LETTER + 0x1E3B: 'l', // WITH LINE BELOW, LATIN SMALL LETTER + 0x0140: 'l', // WITH MIDDLE DOT, LATIN SMALL LETTER + 0x026B: 'l', // WITH MIDDLE TILDE, LATIN SMALL LETTER + 0x026D: 'l', // WITH RETROFLEX HOOK, LATIN SMALL LETTER + 0x0142: 'l', // WITH STROKE, LATIN SMALL LETTER + 0x2097: 'l', // , LATIN SUBSCRIPT SMALL LETTER + 0x1E3F: 'm', // WITH ACUTE, LATIN SMALL LETTER + 0x1E41: 'm', // WITH DOT ABOVE, LATIN SMALL LETTER + 0x1E43: 'm', // WITH DOT BELOW, LATIN SMALL LETTER + 0x0271: 'm', // WITH HOOK, LATIN SMALL LETTER + 0x0270: 'm', // WITH LONG LEG, LATIN SMALL LETTER TURNED + 0x036B: 'm', // , COMBINING LATIN SMALL LETTER + 0x1D1F: 'm', // , LATIN SMALL LETTER SIDEWAYS TURNED + 0x026F: 'm', // , LATIN SMALL LETTER TURNED + 0x2098: 'm', // , LATIN SUBSCRIPT SMALL LETTER + 0x0144: 'n', // WITH ACUTE, LATIN SMALL LETTER + 0x0148: 'n', // WITH CARON, LATIN SMALL LETTER + 0x0146: 'n', // WITH CEDILLA, LATIN SMALL LETTER + 0x1E4B: 'n', // WITH CIRCUMFLEX BELOW, LATIN SMALL LETTER + 0x0235: 'n', // WITH CURL, LATIN SMALL LETTER + 0x1E45: 'n', // WITH DOT ABOVE, LATIN SMALL LETTER + 0x1E47: 'n', // WITH DOT BELOW, LATIN SMALL LETTER + 0x01F9: 'n', // WITH GRAVE, LATIN SMALL LETTER + 0x0272: 'n', // WITH LEFT HOOK, LATIN SMALL LETTER + 0x1E49: 'n', // WITH LINE BELOW, LATIN SMALL LETTER + 0x019E: 'n', // WITH LONG RIGHT LEG, LATIN SMALL LETTER + 0x0273: 'n', // WITH RETROFLEX HOOK, LATIN SMALL LETTER + 0x00F1: 'n', // WITH TILDE, LATIN SMALL LETTER + 0x2099: 'n', // , LATIN SUBSCRIPT SMALL LETTER + 0x00F3: 'o', // WITH ACUTE, LATIN SMALL LETTER + 0x014F: 'o', // WITH BREVE, LATIN SMALL LETTER + 0x01D2: 'o', // WITH CARON, LATIN SMALL LETTER + 0x00F4: 'o', // WITH CIRCUMFLEX, LATIN SMALL LETTER + 0x00F6: 'o', // WITH DIAERESIS, LATIN SMALL LETTER + 0x022F: 'o', // WITH DOT ABOVE, LATIN SMALL LETTER + 0x1ECD: 'o', // WITH DOT BELOW, LATIN SMALL LETTER + 0x0151: 'o', // WITH DOUBLE ACUTE, LATIN SMALL LETTER + 0x020D: 'o', // WITH DOUBLE GRAVE, LATIN SMALL LETTER + 0x00F2: 'o', // WITH GRAVE, LATIN SMALL LETTER + 0x1ECF: 'o', // WITH HOOK ABOVE, LATIN SMALL LETTER + 0x01A1: 'o', // WITH HORN, LATIN SMALL LETTER + 0x020F: 'o', // WITH INVERTED BREVE, LATIN SMALL LETTER + 0x014D: 'o', // WITH MACRON, LATIN SMALL LETTER + 0x01EB: 'o', // WITH OGONEK, LATIN SMALL LETTER + 0x00F8: 'o', // WITH STROKE, LATIN SMALL LETTER + 0x1D13: 'o', // WITH STROKE, LATIN SMALL LETTER SIDEWAYS + 0x00F5: 'o', // WITH TILDE, LATIN SMALL LETTER + 0x0366: 'o', // , COMBINING LATIN SMALL LETTER + 0x0275: 'o', // , LATIN SMALL LETTER BARRED + 0x1D17: 'o', // , LATIN SMALL LETTER BOTTOM HALF + 0x0254: 'o', // , LATIN SMALL LETTER OPEN + 0x1D11: 'o', // , LATIN SMALL LETTER SIDEWAYS + 0x1D12: 'o', // , LATIN SMALL LETTER SIDEWAYS OPEN + 0x1D16: 'o', // , LATIN SMALL LETTER TOP HALF + 0x1E55: 'p', // WITH ACUTE, LATIN SMALL LETTER + 0x1E57: 'p', // WITH DOT ABOVE, LATIN SMALL LETTER + 0x01A5: 'p', // WITH HOOK, LATIN SMALL LETTER + 0x209A: 'p', // , LATIN SUBSCRIPT SMALL LETTER + 0x024B: 'q', // WITH HOOK TAIL, LATIN SMALL LETTER + 0x02A0: 'q', // WITH HOOK, LATIN SMALL LETTER + 0x0155: 'r', // WITH ACUTE, LATIN SMALL LETTER + 0x0159: 'r', // WITH CARON, LATIN SMALL LETTER + 0x0157: 'r', // WITH CEDILLA, LATIN SMALL LETTER + 0x1E59: 'r', // WITH DOT ABOVE, LATIN SMALL LETTER + 0x1E5B: 'r', // WITH DOT BELOW, LATIN SMALL LETTER + 0x0211: 'r', // WITH DOUBLE GRAVE, LATIN SMALL LETTER + 0x027E: 'r', // WITH FISHHOOK, LATIN SMALL LETTER + 0x027F: 'r', // WITH FISHHOOK, LATIN SMALL LETTER REVERSED + 0x027B: 'r', // WITH HOOK, LATIN SMALL LETTER TURNED + 0x0213: 'r', // WITH INVERTED BREVE, LATIN SMALL LETTER + 0x1E5F: 'r', // WITH LINE BELOW, LATIN SMALL LETTER + 0x027C: 'r', // WITH LONG LEG, LATIN SMALL LETTER + 0x027A: 'r', // WITH LONG LEG, LATIN SMALL LETTER TURNED + 0x024D: 'r', // WITH STROKE, LATIN SMALL LETTER + 0x027D: 'r', // WITH TAIL, LATIN SMALL LETTER + 0x036C: 'r', // , COMBINING LATIN SMALL LETTER + 0x0279: 'r', // , LATIN SMALL LETTER TURNED + 0x1D63: 'r', // , LATIN SUBSCRIPT SMALL LETTER + 0x015B: 's', // WITH ACUTE, LATIN SMALL LETTER + 0x0161: 's', // WITH CARON, LATIN SMALL LETTER + 0x015F: 's', // WITH CEDILLA, LATIN SMALL LETTER + 0x015D: 's', // WITH CIRCUMFLEX, LATIN SMALL LETTER + 0x0219: 's', // WITH COMMA BELOW, LATIN SMALL LETTER + 0x1E61: 's', // WITH DOT ABOVE, LATIN SMALL LETTER + 0x1E9B: 's', // WITH DOT ABOVE, LATIN SMALL LETTER LONG + 0x1E63: 's', // WITH DOT BELOW, LATIN SMALL LETTER + 0x0282: 's', // WITH HOOK, LATIN SMALL LETTER + 0x023F: 's', // WITH SWASH TAIL, LATIN SMALL LETTER + 0x017F: 's', // , LATIN SMALL LETTER LONG + 0x00DF: 's', // , LATIN SMALL LETTER SHARP + 0x209B: 's', // , LATIN SUBSCRIPT SMALL LETTER + 0x0165: 't', // WITH CARON, LATIN SMALL LETTER + 0x0163: 't', // WITH CEDILLA, LATIN SMALL LETTER + 0x1E71: 't', // WITH CIRCUMFLEX BELOW, LATIN SMALL LETTER + 0x021B: 't', // WITH COMMA BELOW, LATIN SMALL LETTER + 0x0236: 't', // WITH CURL, LATIN SMALL LETTER + 0x1E97: 't', // WITH DIAERESIS, LATIN SMALL LETTER + 0x1E6B: 't', // WITH DOT ABOVE, LATIN SMALL LETTER + 0x1E6D: 't', // WITH DOT BELOW, LATIN SMALL LETTER + 0x01AD: 't', // WITH HOOK, LATIN SMALL LETTER + 0x1E6F: 't', // WITH LINE BELOW, LATIN SMALL LETTER + 0x01AB: 't', // WITH PALATAL HOOK, LATIN SMALL LETTER + 0x0288: 't', // WITH RETROFLEX HOOK, LATIN SMALL LETTER + 0x0167: 't', // WITH STROKE, LATIN SMALL LETTER + 0x036D: 't', // , COMBINING LATIN SMALL LETTER + 0x0287: 't', // , LATIN SMALL LETTER TURNED + 0x209C: 't', // , LATIN SUBSCRIPT SMALL LETTER + 0x0289: 'u', // BAR, LATIN SMALL LETTER + 0x00FA: 'u', // WITH ACUTE, LATIN SMALL LETTER + 0x016D: 'u', // WITH BREVE, LATIN SMALL LETTER + 0x01D4: 'u', // WITH CARON, LATIN SMALL LETTER + 0x1E77: 'u', // WITH CIRCUMFLEX BELOW, LATIN SMALL LETTER + 0x00FB: 'u', // WITH CIRCUMFLEX, LATIN SMALL LETTER + 0x1E73: 'u', // WITH DIAERESIS BELOW, LATIN SMALL LETTER + 0x00FC: 'u', // WITH DIAERESIS, LATIN SMALL LETTER + 0x1EE5: 'u', // WITH DOT BELOW, LATIN SMALL LETTER + 0x0171: 'u', // WITH DOUBLE ACUTE, LATIN SMALL LETTER + 0x0215: 'u', // WITH DOUBLE GRAVE, LATIN SMALL LETTER + 0x00F9: 'u', // WITH GRAVE, LATIN SMALL LETTER + 0x1EE7: 'u', // WITH HOOK ABOVE, LATIN SMALL LETTER + 0x01B0: 'u', // WITH HORN, LATIN SMALL LETTER + 0x0217: 'u', // WITH INVERTED BREVE, LATIN SMALL LETTER + 0x016B: 'u', // WITH MACRON, LATIN SMALL LETTER + 0x0173: 'u', // WITH OGONEK, LATIN SMALL LETTER + 0x016F: 'u', // WITH RING ABOVE, LATIN SMALL LETTER + 0x1E75: 'u', // WITH TILDE BELOW, LATIN SMALL LETTER + 0x0169: 'u', // WITH TILDE, LATIN SMALL LETTER + 0x0367: 'u', // , COMBINING LATIN SMALL LETTER + 0x1D1D: 'u', // , LATIN SMALL LETTER SIDEWAYS + 0x1D1E: 'u', // , LATIN SMALL LETTER SIDEWAYS DIAERESIZED + 0x1D64: 'u', // , LATIN SUBSCRIPT SMALL LETTER + 0x1E7F: 'v', // WITH DOT BELOW, LATIN SMALL LETTER + 0x028B: 'v', // WITH HOOK, LATIN SMALL LETTER + 0x1E7D: 'v', // WITH TILDE, LATIN SMALL LETTER + 0x036E: 'v', // , COMBINING LATIN SMALL LETTER + 0x028C: 'v', // , LATIN SMALL LETTER TURNED + 0x1D65: 'v', // , LATIN SUBSCRIPT SMALL LETTER + 0x1E83: 'w', // WITH ACUTE, LATIN SMALL LETTER + 0x0175: 'w', // WITH CIRCUMFLEX, LATIN SMALL LETTER + 0x1E85: 'w', // WITH DIAERESIS, LATIN SMALL LETTER + 0x1E87: 'w', // WITH DOT ABOVE, LATIN SMALL LETTER + 0x1E89: 'w', // WITH DOT BELOW, LATIN SMALL LETTER + 0x1E81: 'w', // WITH GRAVE, LATIN SMALL LETTER + 0x1E98: 'w', // WITH RING ABOVE, LATIN SMALL LETTER + 0x028D: 'w', // , LATIN SMALL LETTER TURNED + 0x1E8D: 'x', // WITH DIAERESIS, LATIN SMALL LETTER + 0x1E8B: 'x', // WITH DOT ABOVE, LATIN SMALL LETTER + 0x036F: 'x', // , COMBINING LATIN SMALL LETTER + 0x00FD: 'y', // WITH ACUTE, LATIN SMALL LETTER + 0x0177: 'y', // WITH CIRCUMFLEX, LATIN SMALL LETTER + 0x00FF: 'y', // WITH DIAERESIS, LATIN SMALL LETTER + 0x1E8F: 'y', // WITH DOT ABOVE, LATIN SMALL LETTER + 0x1EF5: 'y', // WITH DOT BELOW, LATIN SMALL LETTER + 0x1EF3: 'y', // WITH GRAVE, LATIN SMALL LETTER + 0x1EF7: 'y', // WITH HOOK ABOVE, LATIN SMALL LETTER + 0x01B4: 'y', // WITH HOOK, LATIN SMALL LETTER + 0x0233: 'y', // WITH MACRON, LATIN SMALL LETTER + 0x1E99: 'y', // WITH RING ABOVE, LATIN SMALL LETTER + 0x024F: 'y', // WITH STROKE, LATIN SMALL LETTER + 0x1EF9: 'y', // WITH TILDE, LATIN SMALL LETTER + 0x028E: 'y', // , LATIN SMALL LETTER TURNED + 0x017A: 'z', // WITH ACUTE, LATIN SMALL LETTER + 0x017E: 'z', // WITH CARON, LATIN SMALL LETTER + 0x1E91: 'z', // WITH CIRCUMFLEX, LATIN SMALL LETTER + 0x0291: 'z', // WITH CURL, LATIN SMALL LETTER + 0x017C: 'z', // WITH DOT ABOVE, LATIN SMALL LETTER + 0x1E93: 'z', // WITH DOT BELOW, LATIN SMALL LETTER + 0x0225: 'z', // WITH HOOK, LATIN SMALL LETTER + 0x1E95: 'z', // WITH LINE BELOW, LATIN SMALL LETTER + 0x0290: 'z', // WITH RETROFLEX HOOK, LATIN SMALL LETTER + 0x01B6: 'z', // WITH STROKE, LATIN SMALL LETTER + 0x0240: 'z', // WITH SWASH TAIL, LATIN SMALL LETTER + 0x0251: 'a', // , latin small letter script + 0x00C1: 'A', // WITH ACUTE, LATIN CAPITAL LETTER + 0x00C2: 'A', // WITH CIRCUMFLEX, LATIN CAPITAL LETTER + 0x00C4: 'A', // WITH DIAERESIS, LATIN CAPITAL LETTER + 0x00C0: 'A', // WITH GRAVE, LATIN CAPITAL LETTER + 0x00C5: 'A', // WITH RING ABOVE, LATIN CAPITAL LETTER + 0x023A: 'A', // WITH STROKE, LATIN CAPITAL LETTER + 0x00C3: 'A', // WITH TILDE, LATIN CAPITAL LETTER + 0x1D00: 'A', // , LATIN LETTER SMALL CAPITAL + 0x0181: 'B', // WITH HOOK, LATIN CAPITAL LETTER + 0x0243: 'B', // WITH STROKE, LATIN CAPITAL LETTER + 0x0299: 'B', // , LATIN LETTER SMALL CAPITAL + 0x1D03: 'B', // , LATIN LETTER SMALL CAPITAL BARRED + 0x00C7: 'C', // WITH CEDILLA, LATIN CAPITAL LETTER + 0x023B: 'C', // WITH STROKE, LATIN CAPITAL LETTER + 0x1D04: 'C', // , LATIN LETTER SMALL CAPITAL + 0x018A: 'D', // WITH HOOK, LATIN CAPITAL LETTER + 0x0189: 'D', // , LATIN CAPITAL LETTER AFRICAN + 0x1D05: 'D', // , LATIN LETTER SMALL CAPITAL + 0x00C9: 'E', // WITH ACUTE, LATIN CAPITAL LETTER + 0x00CA: 'E', // WITH CIRCUMFLEX, LATIN CAPITAL LETTER + 0x00CB: 'E', // WITH DIAERESIS, LATIN CAPITAL LETTER + 0x00C8: 'E', // WITH GRAVE, LATIN CAPITAL LETTER + 0x0246: 'E', // WITH STROKE, LATIN CAPITAL LETTER + 0x0190: 'E', // , LATIN CAPITAL LETTER OPEN + 0x018E: 'E', // , LATIN CAPITAL LETTER REVERSED + 0x1D07: 'E', // , LATIN LETTER SMALL CAPITAL + 0x0193: 'G', // WITH HOOK, LATIN CAPITAL LETTER + 0x029B: 'G', // WITH HOOK, LATIN LETTER SMALL CAPITAL + 0x0262: 'G', // , LATIN LETTER SMALL CAPITAL + 0x029C: 'H', // , LATIN LETTER SMALL CAPITAL + 0x00CD: 'I', // WITH ACUTE, LATIN CAPITAL LETTER + 0x00CE: 'I', // WITH CIRCUMFLEX, LATIN CAPITAL LETTER + 0x00CF: 'I', // WITH DIAERESIS, LATIN CAPITAL LETTER + 0x0130: 'I', // WITH DOT ABOVE, LATIN CAPITAL LETTER + 0x00CC: 'I', // WITH GRAVE, LATIN CAPITAL LETTER + 0x0197: 'I', // WITH STROKE, LATIN CAPITAL LETTER + 0x026A: 'I', // , LATIN LETTER SMALL CAPITAL + 0x0248: 'J', // WITH STROKE, LATIN CAPITAL LETTER + 0x1D0A: 'J', // , LATIN LETTER SMALL CAPITAL + 0x1D0B: 'K', // , LATIN LETTER SMALL CAPITAL + 0x023D: 'L', // WITH BAR, LATIN CAPITAL LETTER + 0x1D0C: 'L', // WITH STROKE, LATIN LETTER SMALL CAPITAL + 0x029F: 'L', // , LATIN LETTER SMALL CAPITAL + 0x019C: 'M', // , LATIN CAPITAL LETTER TURNED + 0x1D0D: 'M', // , LATIN LETTER SMALL CAPITAL + 0x019D: 'N', // WITH LEFT HOOK, LATIN CAPITAL LETTER + 0x0220: 'N', // WITH LONG RIGHT LEG, LATIN CAPITAL LETTER + 0x00D1: 'N', // WITH TILDE, LATIN CAPITAL LETTER + 0x0274: 'N', // , LATIN LETTER SMALL CAPITAL + 0x1D0E: 'N', // , LATIN LETTER SMALL CAPITAL REVERSED + 0x00D3: 'O', // WITH ACUTE, LATIN CAPITAL LETTER + 0x00D4: 'O', // WITH CIRCUMFLEX, LATIN CAPITAL LETTER + 0x00D6: 'O', // WITH DIAERESIS, LATIN CAPITAL LETTER + 0x00D2: 'O', // WITH GRAVE, LATIN CAPITAL LETTER + 0x019F: 'O', // WITH MIDDLE TILDE, LATIN CAPITAL LETTER + 0x00D8: 'O', // WITH STROKE, LATIN CAPITAL LETTER + 0x00D5: 'O', // WITH TILDE, LATIN CAPITAL LETTER + 0x0186: 'O', // , LATIN CAPITAL LETTER OPEN + 0x1D0F: 'O', // , LATIN LETTER SMALL CAPITAL + 0x1D10: 'O', // , LATIN LETTER SMALL CAPITAL OPEN + 0x1D18: 'P', // , LATIN LETTER SMALL CAPITAL + 0x024A: 'Q', // WITH HOOK TAIL, LATIN CAPITAL LETTER SMALL + 0x024C: 'R', // WITH STROKE, LATIN CAPITAL LETTER + 0x0280: 'R', // , LATIN LETTER SMALL CAPITAL + 0x0281: 'R', // , LATIN LETTER SMALL CAPITAL INVERTED + 0x1D19: 'R', // , LATIN LETTER SMALL CAPITAL REVERSED + 0x1D1A: 'R', // , LATIN LETTER SMALL CAPITAL TURNED + 0x023E: 'T', // WITH DIAGONAL STROKE, LATIN CAPITAL LETTER + 0x01AE: 'T', // WITH RETROFLEX HOOK, LATIN CAPITAL LETTER + 0x1D1B: 'T', // , LATIN LETTER SMALL CAPITAL + 0x0244: 'U', // BAR, LATIN CAPITAL LETTER + 0x00DA: 'U', // WITH ACUTE, LATIN CAPITAL LETTER + 0x00DB: 'U', // WITH CIRCUMFLEX, LATIN CAPITAL LETTER + 0x00DC: 'U', // WITH DIAERESIS, LATIN CAPITAL LETTER + 0x00D9: 'U', // WITH GRAVE, LATIN CAPITAL LETTER + 0x1D1C: 'U', // , LATIN LETTER SMALL CAPITAL + 0x01B2: 'V', // WITH HOOK, LATIN CAPITAL LETTER + 0x0245: 'V', // , LATIN CAPITAL LETTER TURNED + 0x1D20: 'V', // , LATIN LETTER SMALL CAPITAL + 0x1D21: 'W', // , LATIN LETTER SMALL CAPITAL + 0x00DD: 'Y', // WITH ACUTE, LATIN CAPITAL LETTER + 0x0178: 'Y', // WITH DIAERESIS, LATIN CAPITAL LETTER + 0x024E: 'Y', // WITH STROKE, LATIN CAPITAL LETTER + 0x028F: 'Y', // , LATIN LETTER SMALL CAPITAL + 0x1D22: 'Z', // , LATIN LETTER SMALL CAPITAL +} diff --git a/src/core.go b/src/core.go index f7dae1a..3e27ed7 100644 --- a/src/core.go +++ b/src/core.go @@ -143,7 +143,7 @@ func Run(opts *Options) { } patternBuilder := func(runes []rune) *Pattern { return BuildPattern( - opts.Fuzzy, opts.FuzzyAlgo, opts.Extended, opts.Case, forward, + opts.Fuzzy, opts.FuzzyAlgo, opts.Extended, opts.Case, opts.Normalize, forward, opts.Filter == nil, opts.Nth, opts.Delimiter, runes) } matcher := NewMatcher(patternBuilder, sort, opts.Tac, eventBox) diff --git a/src/options.go b/src/options.go index 0c6661f..f34ee84 100644 --- a/src/options.go +++ b/src/options.go @@ -24,6 +24,7 @@ const usage = `usage: fzf [options] --algo=TYPE Fuzzy matching algorithm: [v1|v2] (default: v2) -i Case-insensitive match (default: smart-case match) +i Case-sensitive match + --normalize Normalize latin script letters before matching -n, --nth=N[,..] Comma-separated list of field index expressions for limiting search scope. Each can be a non-zero integer or a range expression ([BEGIN]..[END]). @@ -138,6 +139,7 @@ type Options struct { FuzzyAlgo algo.Algo Extended bool Case Case + Normalize bool Nth []Range WithNth []Range Delimiter Delimiter @@ -185,6 +187,7 @@ func defaultOptions() *Options { FuzzyAlgo: algo.FuzzyMatchV2, Extended: true, Case: CaseSmart, + Normalize: false, Nth: make([]Range, 0), WithNth: make([]Range, 0), Delimiter: Delimiter{}, @@ -887,6 +890,10 @@ func parseOptions(opts *Options, allArgs []string) { case "-f", "--filter": filter := nextString(allArgs, &i, "query string required") opts.Filter = &filter + case "--normalize": + opts.Normalize = true + case "--no-normalize": + opts.Normalize = false case "--algo": opts.FuzzyAlgo = parseAlgo(nextString(allArgs, &i, "algorithm required (v1|v2)")) case "--expect": diff --git a/src/pattern.go b/src/pattern.go index 82272af..8f1d9bc 100644 --- a/src/pattern.go +++ b/src/pattern.go @@ -43,6 +43,7 @@ type Pattern struct { fuzzyAlgo algo.Algo extended bool caseSensitive bool + normalize bool forward bool text []rune termSets []termSet @@ -75,7 +76,7 @@ func clearChunkCache() { } // BuildPattern builds Pattern object from the given arguments -func BuildPattern(fuzzy bool, fuzzyAlgo algo.Algo, extended bool, caseMode Case, forward bool, +func BuildPattern(fuzzy bool, fuzzyAlgo algo.Algo, extended bool, caseMode Case, normalize bool, forward bool, cacheable bool, nth []Range, delimiter Delimiter, runes []rune) *Pattern { var asString string @@ -120,6 +121,7 @@ func BuildPattern(fuzzy bool, fuzzyAlgo algo.Algo, extended bool, caseMode Case, fuzzyAlgo: fuzzyAlgo, extended: extended, caseSensitive: caseSensitive, + normalize: normalize, forward: forward, text: []rune(asString), termSets: termSets, @@ -309,9 +311,9 @@ func (p *Pattern) MatchItem(item *Item, withPos bool, slab *util.Slab) (*Result, func (p *Pattern) basicMatch(item *Item, withPos bool, slab *util.Slab) (Offset, int, int, *[]int) { input := p.prepareInput(item) if p.fuzzy { - return p.iter(p.fuzzyAlgo, input, p.caseSensitive, p.forward, p.text, withPos, slab) + return p.iter(p.fuzzyAlgo, input, p.caseSensitive, p.normalize, p.forward, p.text, withPos, slab) } - return p.iter(algo.ExactMatchNaive, input, p.caseSensitive, p.forward, p.text, withPos, slab) + return p.iter(algo.ExactMatchNaive, input, p.caseSensitive, p.normalize, p.forward, p.text, withPos, slab) } func (p *Pattern) extendedMatch(item *Item, withPos bool, slab *util.Slab) ([]Offset, int, int, *[]int) { @@ -330,7 +332,7 @@ func (p *Pattern) extendedMatch(item *Item, withPos bool, slab *util.Slab) ([]Of matched := false for _, term := range termSet { pfun := p.procFun[term.typ] - off, score, tLen, pos := p.iter(pfun, input, term.caseSensitive, p.forward, term.text, withPos, slab) + off, score, tLen, pos := p.iter(pfun, input, term.caseSensitive, p.normalize, p.forward, term.text, withPos, slab) if sidx := off[0]; sidx >= 0 { if term.inv { continue @@ -378,9 +380,9 @@ func (p *Pattern) prepareInput(item *Item) []Token { return ret } -func (p *Pattern) iter(pfun algo.Algo, tokens []Token, caseSensitive bool, forward bool, pattern []rune, withPos bool, slab *util.Slab) (Offset, int, int, *[]int) { +func (p *Pattern) iter(pfun algo.Algo, tokens []Token, caseSensitive bool, normalize bool, forward bool, pattern []rune, withPos bool, slab *util.Slab) (Offset, int, int, *[]int) { for _, part := range tokens { - if res, pos := pfun(caseSensitive, forward, *part.text, pattern, withPos, slab); res.Start >= 0 { + if res, pos := pfun(caseSensitive, normalize, forward, *part.text, pattern, withPos, slab); res.Start >= 0 { sidx := int32(res.Start) + part.prefixLength eidx := int32(res.End) + part.prefixLength if pos != nil { diff --git a/src/pattern_test.go b/src/pattern_test.go index 9b6d394..66c0041 100644 --- a/src/pattern_test.go +++ b/src/pattern_test.go @@ -75,10 +75,10 @@ func TestParseTermsEmpty(t *testing.T) { func TestExact(t *testing.T) { defer clearPatternCache() clearPatternCache() - pattern := BuildPattern(true, algo.FuzzyMatchV2, true, CaseSmart, true, true, + pattern := BuildPattern(true, algo.FuzzyMatchV2, true, CaseSmart, false, true, true, []Range{}, Delimiter{}, []rune("'abc")) res, pos := algo.ExactMatchNaive( - pattern.caseSensitive, pattern.forward, util.RunesToChars([]rune("aabbcc abc")), pattern.termSets[0][0].text, true, nil) + pattern.caseSensitive, pattern.normalize, pattern.forward, util.RunesToChars([]rune("aabbcc abc")), pattern.termSets[0][0].text, true, nil) if res.Start != 7 || res.End != 10 { t.Errorf("%s / %d / %d", pattern.termSets, res.Start, res.End) } @@ -90,11 +90,11 @@ func TestExact(t *testing.T) { func TestEqual(t *testing.T) { defer clearPatternCache() clearPatternCache() - pattern := BuildPattern(true, algo.FuzzyMatchV2, true, CaseSmart, true, true, []Range{}, Delimiter{}, []rune("^AbC$")) + pattern := BuildPattern(true, algo.FuzzyMatchV2, true, CaseSmart, false, true, true, []Range{}, Delimiter{}, []rune("^AbC$")) match := func(str string, sidxExpected int, eidxExpected int) { res, pos := algo.EqualMatch( - pattern.caseSensitive, pattern.forward, util.RunesToChars([]rune(str)), pattern.termSets[0][0].text, true, nil) + pattern.caseSensitive, pattern.normalize, pattern.forward, util.RunesToChars([]rune(str)), pattern.termSets[0][0].text, true, nil) if res.Start != sidxExpected || res.End != eidxExpected { t.Errorf("%s / %d / %d", pattern.termSets, res.Start, res.End) } @@ -109,17 +109,17 @@ func TestEqual(t *testing.T) { func TestCaseSensitivity(t *testing.T) { defer clearPatternCache() clearPatternCache() - pat1 := BuildPattern(true, algo.FuzzyMatchV2, false, CaseSmart, true, true, []Range{}, Delimiter{}, []rune("abc")) + pat1 := BuildPattern(true, algo.FuzzyMatchV2, false, CaseSmart, false, true, true, []Range{}, Delimiter{}, []rune("abc")) clearPatternCache() - pat2 := BuildPattern(true, algo.FuzzyMatchV2, false, CaseSmart, true, true, []Range{}, Delimiter{}, []rune("Abc")) + pat2 := BuildPattern(true, algo.FuzzyMatchV2, false, CaseSmart, false, true, true, []Range{}, Delimiter{}, []rune("Abc")) clearPatternCache() - pat3 := BuildPattern(true, algo.FuzzyMatchV2, false, CaseIgnore, true, true, []Range{}, Delimiter{}, []rune("abc")) + pat3 := BuildPattern(true, algo.FuzzyMatchV2, false, CaseIgnore, false, true, true, []Range{}, Delimiter{}, []rune("abc")) clearPatternCache() - pat4 := BuildPattern(true, algo.FuzzyMatchV2, false, CaseIgnore, true, true, []Range{}, Delimiter{}, []rune("Abc")) + pat4 := BuildPattern(true, algo.FuzzyMatchV2, false, CaseIgnore, false, true, true, []Range{}, Delimiter{}, []rune("Abc")) clearPatternCache() - pat5 := BuildPattern(true, algo.FuzzyMatchV2, false, CaseRespect, true, true, []Range{}, Delimiter{}, []rune("abc")) + pat5 := BuildPattern(true, algo.FuzzyMatchV2, false, CaseRespect, false, true, true, []Range{}, Delimiter{}, []rune("abc")) clearPatternCache() - pat6 := BuildPattern(true, algo.FuzzyMatchV2, false, CaseRespect, true, true, []Range{}, Delimiter{}, []rune("Abc")) + pat6 := BuildPattern(true, algo.FuzzyMatchV2, false, CaseRespect, false, true, true, []Range{}, Delimiter{}, []rune("Abc")) if string(pat1.text) != "abc" || pat1.caseSensitive != false || string(pat2.text) != "Abc" || pat2.caseSensitive != true || @@ -132,7 +132,7 @@ func TestCaseSensitivity(t *testing.T) { } func TestOrigTextAndTransformed(t *testing.T) { - pattern := BuildPattern(true, algo.FuzzyMatchV2, true, CaseSmart, true, true, []Range{}, Delimiter{}, []rune("jg")) + pattern := BuildPattern(true, algo.FuzzyMatchV2, true, CaseSmart, false, true, true, []Range{}, Delimiter{}, []rune("jg")) tokens := Tokenize(util.RunesToChars([]rune("junegunn")), Delimiter{}) trans := Transform(tokens, []Range{Range{1, 1}}) @@ -167,7 +167,7 @@ func TestOrigTextAndTransformed(t *testing.T) { func TestCacheKey(t *testing.T) { test := func(extended bool, patStr string, expected string, cacheable bool) { - pat := BuildPattern(true, algo.FuzzyMatchV2, extended, CaseSmart, true, true, []Range{}, Delimiter{}, []rune(patStr)) + pat := BuildPattern(true, algo.FuzzyMatchV2, extended, CaseSmart, false, true, true, []Range{}, Delimiter{}, []rune(patStr)) if pat.CacheKey() != expected { t.Errorf("Expected: %s, actual: %s", expected, pat.CacheKey()) }