Replace RuneWidth to StringWidth to handle grapheme clusters

Fix #2482

go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/mattn/go-isatty v0.0.12
 	github.com/mattn/go-runewidth v0.0.12
 	github.com/mattn/go-shellwords v1.0.11
-	github.com/rivo/uniseg v0.2.0 // indirect
+	github.com/rivo/uniseg v0.2.0
 	github.com/saracen/walker v0.1.2
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
 	golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57

src/options.go

@@ -1536,15 +1536,13 @@ func validateSign(sign string, signOptName string) error {
 	if sign == "" {
 		return fmt.Errorf("%v cannot be empty", signOptName)
 	}
-	widthSum := 0
 	for _, r := range sign {
 		if !unicode.IsGraphic(r) {
 			return fmt.Errorf("invalid character in %v", signOptName)
 		}
-		widthSum += runewidth.RuneWidth(r)
-		if widthSum > 2 {
-			return fmt.Errorf("%v display width should be up to 2", signOptName)
 	}
+	if runewidth.StringWidth(sign) > 2 {
+		return fmt.Errorf("%v display width should be up to 2", signOptName)
 	}
 	return nil
 }

src/terminal.go

@@ -2,7 +2,6 @@ package fzf
 
 import (
 	"bufio"
-	"bytes"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -15,6 +14,9 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/mattn/go-runewidth"
+	"github.com/rivo/uniseg"
+
 	"github.com/junegunn/fzf/src/tui"
 	"github.com/junegunn/fzf/src/util"
 )
@@ -673,11 +675,8 @@ func (t *Terminal) sortSelected() []selectedItem {
 }
 
 func (t *Terminal) displayWidth(runes []rune) int {
-	l := 0
+	width, _ := util.RunesWidth(runes, 0, t.tabstop, 0)
-	for _, r := range runes {
+	return width
-		l += util.RuneWidth(r, l, t.tabstop)
-	}
-	return l
 }
 
 const (
@@ -1141,28 +1140,18 @@ func (t *Terminal) printItem(result Result, line int, i int, current bool) {
 	t.prevLines[i] = newLine
 }
 
-func (t *Terminal) trimRight(runes []rune, width int) ([]rune, int) {
+func (t *Terminal) trimRight(runes []rune, width int) ([]rune, bool) {
 	// We start from the beginning to handle tab characters
-	l := 0
+	width, overflowIdx := util.RunesWidth(runes, 0, t.tabstop, width)
-	for idx, r := range runes {
+	if overflowIdx >= 0 {
-		l += util.RuneWidth(r, l, t.tabstop)
+		return runes[:overflowIdx], true
-		if l > width {
-			return runes[:idx], len(runes) - idx
 	}
-	}
+	return runes, false
-	return runes, 0
 }
 
 func (t *Terminal) displayWidthWithLimit(runes []rune, prefixWidth int, limit int) int {
-	l := 0
+	width, _ := util.RunesWidth(runes, prefixWidth, t.tabstop, limit)
-	for _, r := range runes {
+	return width
-		l += util.RuneWidth(r, l+prefixWidth, t.tabstop)
-		if l > limit {
-			// Early exit
-			return l
-		}
-	}
-	return l
 }
 
 func (t *Terminal) trimLeft(runes []rune, width int) ([]rune, int32) {
@@ -1362,9 +1351,9 @@ func (t *Terminal) renderPreviewText(height int, lines []string, lineNo int, unc
 			prefixWidth := 0
 			_, _, ansi = extractColor(line, ansi, func(str string, ansi *ansiState) bool {
 				trimmed := []rune(str)
-				trimmedLen := 0
+				isTrimmed := false
 				if !t.previewOpts.wrap {
-					trimmed, trimmedLen = t.trimRight(trimmed, maxWidth-t.pwindow.X())
+					trimmed, isTrimmed = t.trimRight(trimmed, maxWidth-t.pwindow.X())
 				}
 				str, width := t.processTabs(trimmed, prefixWidth)
 				prefixWidth += width
@@ -1374,7 +1363,7 @@ func (t *Terminal) renderPreviewText(height int, lines []string, lineNo int, unc
 				} else {
 					fillRet = t.pwindow.CFill(tui.ColPreview.Fg(), tui.ColPreview.Bg(), tui.AttrRegular, str)
 				}
-				return trimmedLen == 0 &&
+				return !isTrimmed &&
 					(fillRet == tui.FillContinue || t.previewOpts.wrap && fillRet == tui.FillNextLine)
 			})
 			t.previewer.scrollable = t.previewer.scrollable || t.pwindow.Y() == height-1 && t.pwindow.X() == t.pwindow.Width()
@@ -1430,16 +1419,21 @@ func (t *Terminal) printPreviewDelayed() {
 }
 
 func (t *Terminal) processTabs(runes []rune, prefixWidth int) (string, int) {
-	var strbuf bytes.Buffer
+	var strbuf strings.Builder
 	l := prefixWidth
-	for _, r := range runes {
+	gr := uniseg.NewGraphemes(string(runes))
-		w := util.RuneWidth(r, l, t.tabstop)
+	for gr.Next() {
-		l += w
+		rs := gr.Runes()
-		if r == '\t' {
+		str := string(rs)
+		var w int
+		if len(rs) == 1 && rs[0] == '\t' {
+			w = t.tabstop - l%t.tabstop
 			strbuf.WriteString(strings.Repeat(" ", w))
 		} else {
-			strbuf.WriteRune(r)
+			w = runewidth.StringWidth(str)
+			strbuf.WriteString(str)
 		}
+		l += w
 	}
 	return strbuf.String(), l
 }

src/tui/light.go

@@ -10,7 +10,8 @@ import (
 	"time"
 	"unicode/utf8"
 
-	"github.com/junegunn/fzf/src/util"
+	"github.com/mattn/go-runewidth"
+	"github.com/rivo/uniseg"
 
 	"golang.org/x/term"
 )
@@ -50,7 +51,7 @@ func (r *LightRenderer) stderrInternal(str string, allowNLCR bool) {
 		}
 		bytes = bytes[sz:]
 	}
-	r.queued += string(runes)
+	r.queued.WriteString(string(runes))
 }
 
 func (r *LightRenderer) csi(code string) {
@@ -58,9 +59,9 @@ func (r *LightRenderer) csi(code string) {
 }
 
 func (r *LightRenderer) flush() {
-	if len(r.queued) > 0 {
+	if r.queued.Len() > 0 {
-		fmt.Fprint(os.Stderr, r.queued)
+		fmt.Fprint(os.Stderr, r.queued.String())
-		r.queued = ""
+		r.queued.Reset()
 	}
 }
 
@@ -82,7 +83,7 @@ type LightRenderer struct {
 	escDelay      int
 	fullscreen    bool
 	upOneLine     bool
-	queued        string
+	queued        strings.Builder
 	y             int
 	x             int
 	maxHeightFunc func(int) int
@@ -889,20 +890,26 @@ func wrapLine(input string, prefixLength int, max int, tabstop int) []wrappedLin
 	lines := []wrappedLine{}
 	width := 0
 	line := ""
-	for _, r := range input {
+	gr := uniseg.NewGraphemes(input)
-		w := util.RuneWidth(r, prefixLength+width, 8)
+	for gr.Next() {
-		width += w
+		rs := gr.Runes()
-		str := string(r)
+		str := string(rs)
-		if r == '\t' {
+		var w int
+		if len(rs) == 1 && rs[0] == '\t' {
+			w = tabstop - (prefixLength+width)%tabstop
 			str = repeat(' ', w)
+		} else {
+			w = runewidth.StringWidth(str)
 		}
+		width += w
+
 		if prefixLength+width <= max {
 			line += str
 		} else {
 			lines = append(lines, wrappedLine{string(line), width - w})
 			line = str
 			prefixLength = 0
-			width = util.RuneWidth(r, prefixLength, 8)
+			width = w
 		}
 	}
 	lines = append(lines, wrappedLine{string(line), width})

src/tui/tcell.go

@@ -5,7 +5,6 @@ package tui
 import (
 	"os"
 	"time"
-	"unicode/utf8"
 
 	"runtime"
 
@@ -13,6 +12,7 @@ import (
 	"github.com/gdamore/tcell/encoding"
 
 	"github.com/mattn/go-runewidth"
+	"github.com/rivo/uniseg"
 )
 
 func HasFullscreenRenderer() bool {
@@ -482,7 +482,6 @@ func (w *TcellWindow) Print(text string) {
 }
 
 func (w *TcellWindow) printString(text string, pair ColorPair) {
-	t := text
 	lx := 0
 	a := pair.Attr()
 
@@ -496,33 +495,28 @@ func (w *TcellWindow) printString(text string, pair ColorPair) {
 			Dim(a&Attr(tcell.AttrDim) != 0)
 	}
 
-	for {
+	gr := uniseg.NewGraphemes(text)
-		if len(t) == 0 {
+	for gr.Next() {
-			break
+		rs := gr.Runes()
-		}
-		r, size := utf8.DecodeRuneInString(t)
-		t = t[size:]
 
+		if len(rs) == 1 {
+			r := rs[0]
 			if r < rune(' ') { // ignore control characters
 				continue
-		}
+			} else if r == '\n' {
-
-		if r == '\n' {
 				w.lastY++
 				lx = 0
-		} else {
+				continue
-
+			} else if r == '\u000D' { // skip carriage return
-			if r == '\u000D' { // skip carriage return
 				continue
 			}
-
+		}
 		var xPos = w.left + w.lastX + lx
 		var yPos = w.top + w.lastY
 		if xPos < (w.left+w.width) && yPos < (w.top+w.height) {
-				_screen.SetContent(xPos, yPos, r, nil, style)
+			_screen.SetContent(xPos, yPos, rs[0], rs[1:], style)
-			}
-			lx += runewidth.RuneWidth(r)
 		}
+		lx += runewidth.StringWidth(string(rs))
 	}
 	w.lastX += lx
 }
@@ -549,30 +543,32 @@ func (w *TcellWindow) fillString(text string, pair ColorPair) FillReturn {
 		Underline(a&Attr(tcell.AttrUnderline) != 0).
 		Italic(a&Attr(tcell.AttrItalic) != 0)
 
-	for _, r := range text {
+	gr := uniseg.NewGraphemes(text)
-		if r == '\n' {
+	for gr.Next() {
+		rs := gr.Runes()
+		if len(rs) == 1 && rs[0] == '\n' {
 			w.lastY++
 			w.lastX = 0
 			lx = 0
-		} else {
+			continue
-			var xPos = w.left + w.lastX + lx
+		}
 
 		// word wrap:
+		xPos := w.left + w.lastX + lx
 		if xPos >= (w.left + w.width) {
 			w.lastY++
 			w.lastX = 0
 			lx = 0
 			xPos = w.left
 		}
-			var yPos = w.top + w.lastY
 
+		yPos := w.top + w.lastY
 		if yPos >= (w.top + w.height) {
 			return FillSuspend
 		}
 
-			_screen.SetContent(xPos, yPos, r, nil, style)
+		_screen.SetContent(xPos, yPos, rs[0], rs[1:], style)
-			lx += runewidth.RuneWidth(r)
+		lx += runewidth.StringWidth(string(rs))
-		}
 	}
 	w.lastX += lx
 	if w.lastX == w.width {

src/util/util.go

@@ -7,22 +7,29 @@ import (
 
 	"github.com/mattn/go-isatty"
 	"github.com/mattn/go-runewidth"
+	"github.com/rivo/uniseg"
 )
 
-var _runeWidths = make(map[rune]int)
+// RunesWidth returns runes width
-
+func RunesWidth(runes []rune, prefixWidth int, tabstop int, limit int) (int, int) {
-// RuneWidth returns rune width
+	width := 0
-func RuneWidth(r rune, prefixWidth int, tabstop int) int {
+	gr := uniseg.NewGraphemes(string(runes))
-	if r == '\t' {
+	idx := 0
-		return tabstop - prefixWidth%tabstop
+	for gr.Next() {
-	} else if w, found := _runeWidths[r]; found {
+		rs := gr.Runes()
-		return w
+		var w int
-	} else if r == '\n' || r == '\r' {
+		if len(rs) == 1 && rs[0] == '\t' {
-		return 1
+			w = tabstop - (prefixWidth+width)%tabstop
+		} else {
+			w = runewidth.StringWidth(string(rs))
 		}
-	w := runewidth.RuneWidth(r)
+		width += w
-	_runeWidths[r] = w
+		if limit > 0 && width > limit {
-	return w
+			return width, idx
+		}
+		idx += len(rs)
+	}
+	return width, -1
 }
 
 // Max returns the largest integer