fzf/src/tokenizer.go

243 lines
5.2 KiB
Go
Raw Normal View History

2015-01-01 19:49:30 +00:00
package fzf
import (
"bytes"
2015-01-01 19:49:30 +00:00
"regexp"
"strconv"
"strings"
2015-01-12 03:56:17 +00:00
"github.com/junegunn/fzf/src/util"
2015-01-01 19:49:30 +00:00
)
2015-01-11 18:01:24 +00:00
const rangeEllipsis = 0
2015-01-01 19:49:30 +00:00
2015-01-11 18:01:24 +00:00
// Range represents nth-expression
2015-01-01 19:49:30 +00:00
type Range struct {
begin int
end int
}
2015-01-11 18:01:24 +00:00
// Token contains the tokenized part of the strings and its prefix length
2015-01-01 19:49:30 +00:00
type Token struct {
text *util.Chars
prefixLength int32
2015-01-01 19:49:30 +00:00
}
// Delimiter for tokenizing the input
type Delimiter struct {
regex *regexp.Regexp
str *string
}
func newRange(begin int, end int) Range {
if begin == 1 {
begin = rangeEllipsis
}
if end == -1 {
end = rangeEllipsis
}
return Range{begin, end}
}
2015-01-11 18:01:24 +00:00
// ParseRange parses nth-expression and returns the corresponding Range object
2015-01-01 19:49:30 +00:00
func ParseRange(str *string) (Range, bool) {
if (*str) == ".." {
return newRange(rangeEllipsis, rangeEllipsis), true
2015-01-01 19:49:30 +00:00
} else if strings.HasPrefix(*str, "..") {
end, err := strconv.Atoi((*str)[2:])
if err != nil || end == 0 {
return Range{}, false
}
return newRange(rangeEllipsis, end), true
2015-01-01 19:49:30 +00:00
} else if strings.HasSuffix(*str, "..") {
begin, err := strconv.Atoi((*str)[:len(*str)-2])
if err != nil || begin == 0 {
return Range{}, false
}
return newRange(begin, rangeEllipsis), true
2015-01-01 19:49:30 +00:00
} else if strings.Contains(*str, "..") {
ns := strings.Split(*str, "..")
if len(ns) != 2 {
return Range{}, false
}
begin, err1 := strconv.Atoi(ns[0])
end, err2 := strconv.Atoi(ns[1])
if err1 != nil || err2 != nil || begin == 0 || end == 0 {
2015-01-01 19:49:30 +00:00
return Range{}, false
}
return newRange(begin, end), true
2015-01-01 19:49:30 +00:00
}
n, err := strconv.Atoi(*str)
if err != nil || n == 0 {
return Range{}, false
}
return newRange(n, n), true
2015-01-01 19:49:30 +00:00
}
func withPrefixLengths(tokens []string, begin int) []Token {
2015-01-01 19:49:30 +00:00
ret := make([]Token, len(tokens))
prefixLength := begin
for idx := range tokens {
chars := util.ToChars([]byte(tokens[idx]))
ret[idx] = Token{&chars, int32(prefixLength)}
prefixLength += chars.Length()
2015-01-01 19:49:30 +00:00
}
return ret
}
const (
2015-01-11 18:01:24 +00:00
awkNil = iota
awkBlack
awkWhite
2015-01-01 19:49:30 +00:00
)
func awkTokenizer(input string) ([]string, int) {
2015-01-01 19:49:30 +00:00
// 9, 32
ret := []string{}
2015-01-01 19:49:30 +00:00
prefixLength := 0
2015-01-11 18:01:24 +00:00
state := awkNil
begin := 0
end := 0
for idx := 0; idx < len(input); idx++ {
r := input[idx]
2015-01-01 19:49:30 +00:00
white := r == 9 || r == 32
switch state {
2015-01-11 18:01:24 +00:00
case awkNil:
2015-01-01 19:49:30 +00:00
if white {
prefixLength++
} else {
state, begin, end = awkBlack, idx, idx+1
2015-01-01 19:49:30 +00:00
}
2015-01-11 18:01:24 +00:00
case awkBlack:
end = idx + 1
2015-01-01 19:49:30 +00:00
if white {
2015-01-11 18:01:24 +00:00
state = awkWhite
2015-01-01 19:49:30 +00:00
}
2015-01-11 18:01:24 +00:00
case awkWhite:
2015-01-01 19:49:30 +00:00
if white {
end = idx + 1
2015-01-01 19:49:30 +00:00
} else {
ret = append(ret, input[begin:end])
state, begin, end = awkBlack, idx, idx+1
2015-01-01 19:49:30 +00:00
}
}
}
if begin < end {
ret = append(ret, input[begin:end])
2015-01-01 19:49:30 +00:00
}
return ret, prefixLength
}
2015-01-11 18:01:24 +00:00
// Tokenize tokenizes the given string with the delimiter
func Tokenize(text string, delimiter Delimiter) []Token {
if delimiter.str == nil && delimiter.regex == nil {
2015-01-01 19:49:30 +00:00
// AWK-style (\S+\s*)
tokens, prefixLength := awkTokenizer(text)
2015-01-01 19:49:30 +00:00
return withPrefixLengths(tokens, prefixLength)
}
if delimiter.str != nil {
return withPrefixLengths(strings.SplitAfter(text, *delimiter.str), 0)
}
// FIXME performance
var tokens []string
if delimiter.regex != nil {
for len(text) > 0 {
loc := delimiter.regex.FindStringIndex(text)
if loc == nil {
loc = []int{0, len(text)}
}
last := util.Max(loc[1], 1)
tokens = append(tokens, text[:last])
text = text[last:]
}
}
return withPrefixLengths(tokens, 0)
2015-01-01 19:49:30 +00:00
}
func joinTokens(tokens []Token) string {
var output bytes.Buffer
for _, token := range tokens {
output.WriteString(token.text.ToString())
2015-01-01 19:49:30 +00:00
}
return output.String()
}
2015-01-11 18:01:24 +00:00
// Transform is used to transform the input when --with-nth option is given
func Transform(tokens []Token, withNth []Range) []Token {
2015-01-01 19:49:30 +00:00
transTokens := make([]Token, len(withNth))
numTokens := len(tokens)
for idx, r := range withNth {
parts := []*util.Chars{}
2015-01-01 19:49:30 +00:00
minIdx := 0
if r.begin == r.end {
idx := r.begin
2015-01-11 18:01:24 +00:00
if idx == rangeEllipsis {
chars := util.ToChars([]byte(joinTokens(tokens)))
parts = append(parts, &chars)
2015-01-01 19:49:30 +00:00
} else {
if idx < 0 {
idx += numTokens + 1
}
if idx >= 1 && idx <= numTokens {
minIdx = idx - 1
parts = append(parts, tokens[idx-1].text)
2015-01-01 19:49:30 +00:00
}
}
} else {
var begin, end int
2015-01-11 18:01:24 +00:00
if r.begin == rangeEllipsis { // ..N
2015-01-01 19:49:30 +00:00
begin, end = 1, r.end
if end < 0 {
end += numTokens + 1
}
2015-01-11 18:01:24 +00:00
} else if r.end == rangeEllipsis { // N..
2015-01-01 19:49:30 +00:00
begin, end = r.begin, numTokens
if begin < 0 {
begin += numTokens + 1
}
} else {
begin, end = r.begin, r.end
if begin < 0 {
begin += numTokens + 1
}
if end < 0 {
end += numTokens + 1
}
}
2015-01-12 03:56:17 +00:00
minIdx = util.Max(0, begin-1)
2015-01-01 19:49:30 +00:00
for idx := begin; idx <= end; idx++ {
if idx >= 1 && idx <= numTokens {
parts = append(parts, tokens[idx-1].text)
2015-01-01 19:49:30 +00:00
}
}
}
// Merge multiple parts
var merged util.Chars
switch len(parts) {
case 0:
merged = util.ToChars([]byte{})
case 1:
merged = *parts[0]
default:
var output bytes.Buffer
for _, part := range parts {
output.WriteString(part.ToString())
}
merged = util.ToChars([]byte(output.String()))
}
var prefixLength int32
if minIdx < numTokens {
prefixLength = tokens[minIdx].prefixLength
} else {
prefixLength = 0
}
transTokens[idx] = Token{&merged, prefixLength}
2015-01-01 19:49:30 +00:00
}
return transTokens
2015-01-01 19:49:30 +00:00
}