Render CR and LF as ␍ and ␊

Close #2529

CHANGELOG.md

@@ -13,7 +13,17 @@ CHANGELOG
   # Say hello
   curl "localhost:$(cat /tmp/fzf-port)" -d 'preview:echo Hello, fzf is listening on $FZF_PORT.'
   ```
-- Bug fixes
+- A carriage return and a line feed character will be rendered as dim ␍ and
+  ␊ respectively.
+  ```sh
+  printf "foo\rbar\nbaz" | fzf --read0 --preview 'echo {}'
+  ```
+- fzf will stop rendering a non-displayable characters as a space. This will
+  likely cause less glitches in the preview window.
+  ```sh
+  fzf --preview 'head -1000 /dev/random'
+  ```
+- Bug fixes and improvements
 
 0.38.0
 ------

src/terminal.go

@@ -752,7 +752,7 @@ func (t *Terminal) ansiLabelPrinter(str string, color *tui.ColorPair, fill bool)
 
 	// Simpler printer for strings without ANSI colors or tab characters
 	if colors == nil && strings.IndexRune(str, '\t') < 0 {
-		length := runewidth.StringWidth(str)
+		length := util.StringWidth(str)
 		if length == 0 {
 			return nil, 0
 		}
@@ -1415,7 +1415,7 @@ func (t *Terminal) printInfo() {
 		pos = t.promptLen + t.queryLen[0] + t.queryLen[1] + 1
 		str := t.infoSep
 		maxWidth := t.window.Width() - pos
-		width := runewidth.StringWidth(str)
+		width := util.StringWidth(str)
 		if width > maxWidth {
 			trimmed, _ := t.trimRight([]rune(str), maxWidth)
 			str = string(trimmed)
@@ -1950,7 +1950,7 @@ func (t *Terminal) processTabs(runes []rune, prefixWidth int) (string, int) {
 			w = t.tabstop - l%t.tabstop
 			strbuf.WriteString(strings.Repeat(" ", w))
 		} else {
-			w = runewidth.StringWidth(str)
+			w = util.StringWidth(str)
 			strbuf.WriteString(str)
 		}
 		l += w

src/tui/light.go

@@ -32,20 +32,26 @@ var offsetRegexp *regexp.Regexp = regexp.MustCompile("(.*)\x1b\\[([0-9]+);([0-9]
 var offsetRegexpBegin *regexp.Regexp = regexp.MustCompile("^\x1b\\[[0-9]+;[0-9]+R")
 
 func (r *LightRenderer) stderr(str string) {
-	r.stderrInternal(str, true)
+	r.stderrInternal(str, true, "")
 }
 
-// FIXME: Need better handling of non-displayable characters
+const CR string = "\x1b[2m␍"
-func (r *LightRenderer) stderrInternal(str string, allowNLCR bool) {
+const LF string = "\x1b[2m␊"
+
+func (r *LightRenderer) stderrInternal(str string, allowNLCR bool, resetCode string) {
 	bytes := []byte(str)
 	runes := []rune{}
 	for len(bytes) > 0 {
 		r, sz := utf8.DecodeRune(bytes)
 		nlcr := r == '\n' || r == '\r'
 		if r >= 32 || r == '\x1b' || nlcr {
-			if r == utf8.RuneError || nlcr && !allowNLCR {
+			if nlcr && !allowNLCR {
-				runes = append(runes, ' ')
+				if r == '\r' {
-			} else {
+					runes = append(runes, []rune(CR+resetCode)...)
+				} else {
+					runes = append(runes, []rune(LF+resetCode)...)
+				}
+			} else if r != utf8.RuneError {
 				runes = append(runes, r)
 			}
 		}
@@ -54,8 +60,10 @@ func (r *LightRenderer) stderrInternal(str string, allowNLCR bool) {
 	r.queued.WriteString(string(runes))
 }
 
-func (r *LightRenderer) csi(code string) {
+func (r *LightRenderer) csi(code string) string {
-	r.stderr("\x1b[" + code)
+	fullcode := "\x1b[" + code
+	r.stderr(fullcode)
+	return fullcode
 }
 
 func (r *LightRenderer) flush() {
@@ -825,12 +833,12 @@ func (w *LightWindow) drawBorderAround(onlyHorizontal bool) {
 	w.CPrint(color, string(w.border.bottomLeft)+repeat(w.border.horizontal, (w.width-bcw)/hw)+repeat(' ', rem)+string(w.border.bottomRight))
 }
 
-func (w *LightWindow) csi(code string) {
+func (w *LightWindow) csi(code string) string {
-	w.renderer.csi(code)
+	return w.renderer.csi(code)
 }
 
-func (w *LightWindow) stderrInternal(str string, allowNLCR bool) {
+func (w *LightWindow) stderrInternal(str string, allowNLCR bool, resetCode string) {
-	w.renderer.stderrInternal(str, allowNLCR)
+	w.renderer.stderrInternal(str, allowNLCR, resetCode)
 }
 
 func (w *LightWindow) Top() int {
@@ -936,10 +944,10 @@ func colorCodes(fg Color, bg Color) []string {
 	return codes
 }
 
-func (w *LightWindow) csiColor(fg Color, bg Color, attr Attr) bool {
+func (w *LightWindow) csiColor(fg Color, bg Color, attr Attr) (bool, string) {
 	codes := append(attrCodes(attr), colorCodes(fg, bg)...)
-	w.csi(";" + strings.Join(codes, ";") + "m")
+	code := w.csi(";" + strings.Join(codes, ";") + "m")
-	return len(codes) > 0
+	return len(codes) > 0, code
 }
 
 func (w *LightWindow) Print(text string) {
@@ -951,16 +959,17 @@ func cleanse(str string) string {
 }
 
 func (w *LightWindow) CPrint(pair ColorPair, text string) {
-	w.csiColor(pair.Fg(), pair.Bg(), pair.Attr())
+	_, code := w.csiColor(pair.Fg(), pair.Bg(), pair.Attr())
-	w.stderrInternal(cleanse(text), false)
+	w.stderrInternal(cleanse(text), false, code)
 	w.csi("m")
 }
 
 func (w *LightWindow) cprint2(fg Color, bg Color, attr Attr, text string) {
-	if w.csiColor(fg, bg, attr) {
+	hasColors, code := w.csiColor(fg, bg, attr)
+	if hasColors {
 		defer w.csi("m")
 	}
-	w.stderrInternal(cleanse(text), false)
+	w.stderrInternal(cleanse(text), false, code)
 }
 
 type wrappedLine struct {
@@ -980,6 +989,8 @@ func wrapLine(input string, prefixLength int, max int, tabstop int) []wrappedLin
 		if len(rs) == 1 && rs[0] == '\t' {
 			w = tabstop - (prefixLength+width)%tabstop
 			str = repeat(' ', w)
+		} else if rs[0] == '\r' {
+			w++
 		} else {
 			w = runewidth.StringWidth(str)
 		}
@@ -998,12 +1009,12 @@ func wrapLine(input string, prefixLength int, max int, tabstop int) []wrappedLin
 	return lines
 }
 
-func (w *LightWindow) fill(str string, onMove func()) FillReturn {
+func (w *LightWindow) fill(str string, resetCode string) FillReturn {
 	allLines := strings.Split(str, "\n")
 	for i, line := range allLines {
 		lines := wrapLine(line, w.posx, w.width, w.tabstop)
 		for j, wl := range lines {
-			w.stderrInternal(wl.text, false)
+			w.stderrInternal(wl.text, false, resetCode)
 			w.posx += wl.displayWidth
 
 			// Wrap line
@@ -1013,7 +1024,7 @@ func (w *LightWindow) fill(str string, onMove func()) FillReturn {
 				}
 				w.MoveAndClear(w.posy, w.posx)
 				w.Move(w.posy+1, 0)
-				onMove()
+				w.renderer.stderr(resetCode)
 			}
 		}
 	}
@@ -1022,22 +1033,26 @@ func (w *LightWindow) fill(str string, onMove func()) FillReturn {
 			return FillSuspend
 		}
 		w.Move(w.posy+1, 0)
-		onMove()
+		w.renderer.stderr(resetCode)
 		return FillNextLine
 	}
 	return FillContinue
 }
 
-func (w *LightWindow) setBg() {
+func (w *LightWindow) setBg() string {
 	if w.bg != colDefault {
-		w.csiColor(colDefault, w.bg, AttrRegular)
+		_, code := w.csiColor(colDefault, w.bg, AttrRegular)
+		return code
 	}
+	// Should clear dim attribute after ␍ in the preview window
+	// e.g. printf "foo\rbar" | fzf --ansi --preview 'printf "foo\rbar"'
+	return "\x1b[m"
 }
 
 func (w *LightWindow) Fill(text string) FillReturn {
 	w.Move(w.posy, w.posx)
-	w.setBg()
+	code := w.setBg()
-	return w.fill(text, w.setBg)
+	return w.fill(text, code)
 }
 
 func (w *LightWindow) CFill(fg Color, bg Color, attr Attr, text string) FillReturn {
@@ -1048,11 +1063,11 @@ func (w *LightWindow) CFill(fg Color, bg Color, attr Attr, text string) FillRetu
 	if bg == colDefault {
 		bg = w.bg
 	}
-	if w.csiColor(fg, bg, attr) {
+	if hasColors, resetCode := w.csiColor(fg, bg, attr); hasColors {
 		defer w.csi("m")
-		return w.fill(text, func() { w.csiColor(fg, bg, attr) })
+		return w.fill(text, resetCode)
 	}
-	return w.fill(text, w.setBg)
+	return w.fill(text, w.setBg())
 }
 
 func (w *LightWindow) FinishFill() {

src/tui/tcell.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/gdamore/tcell/v2"
 	"github.com/gdamore/tcell/v2/encoding"
+	"github.com/junegunn/fzf/src/util"
 
 	"github.com/mattn/go-runewidth"
 	"github.com/rivo/uniseg"
@@ -572,26 +573,27 @@ func (w *TcellWindow) printString(text string, pair ColorPair) {
 
 	gr := uniseg.NewGraphemes(text)
 	for gr.Next() {
+		st := style
 		rs := gr.Runes()
 
 		if len(rs) == 1 {
 			r := rs[0]
-			if r < rune(' ') { // ignore control characters
+			if r == '\r' {
-				continue
+				st = style.Dim(true)
+				rs[0] = '␍'
 			} else if r == '\n' {
-				w.lastY++
+				st = style.Dim(true)
-				lx = 0
+				rs[0] = '␊'
-				continue
+			} else if r < rune(' ') { // ignore control characters
-			} else if r == '\u000D' { // skip carriage return
 				continue
 			}
 		}
 		var xPos = w.left + w.lastX + lx
 		var yPos = w.top + w.lastY
 		if xPos < (w.left+w.width) && yPos < (w.top+w.height) {
-			_screen.SetContent(xPos, yPos, rs[0], rs[1:], style)
+			_screen.SetContent(xPos, yPos, rs[0], rs[1:], st)
 		}
-		lx += runewidth.StringWidth(string(rs))
+		lx += util.StringWidth(string(rs))
 	}
 	w.lastX += lx
 }
@@ -620,13 +622,22 @@ func (w *TcellWindow) fillString(text string, pair ColorPair) FillReturn {
 		Italic(a&Attr(tcell.AttrItalic) != 0)
 
 	gr := uniseg.NewGraphemes(text)
+Loop:
 	for gr.Next() {
+		st := style
 		rs := gr.Runes()
-		if len(rs) == 1 && rs[0] == '\n' {
+		if len(rs) == 1 {
-			w.lastY++
+			r := rs[0]
-			w.lastX = 0
+			switch r {
-			lx = 0
+			case '\r':
-			continue
+				st = style.Dim(true)
+				rs[0] = '␍'
+			case '\n':
+				w.lastY++
+				w.lastX = 0
+				lx = 0
+				continue Loop
+			}
 		}
 
 		// word wrap:
@@ -643,8 +654,8 @@ func (w *TcellWindow) fillString(text string, pair ColorPair) FillReturn {
 			return FillSuspend
 		}
 
-		_screen.SetContent(xPos, yPos, rs[0], rs[1:], style)
+		_screen.SetContent(xPos, yPos, rs[0], rs[1:], st)
-		lx += runewidth.StringWidth(string(rs))
+		lx += util.StringWidth(string(rs))
 	}
 	w.lastX += lx
 	if w.lastX == w.width {

src/util/util.go

@@ -11,6 +11,11 @@ import (
 	"github.com/rivo/uniseg"
 )
 
+// StringWidth returns string width where each CR/LF character takes 1 column
+func StringWidth(s string) int {
+	return runewidth.StringWidth(s) + strings.Count(s, "\n") + strings.Count(s, "\r")
+}
+
 // RunesWidth returns runes width
 func RunesWidth(runes []rune, prefixWidth int, tabstop int, limit int) (int, int) {
 	width := 0
@@ -22,8 +27,7 @@ func RunesWidth(runes []rune, prefixWidth int, tabstop int, limit int) (int, int
 		if len(rs) == 1 && rs[0] == '\t' {
 			w = tabstop - (prefixWidth+width)%tabstop
 		} else {
-			s := string(rs)
+			w = StringWidth(string(rs))
-			w = runewidth.StringWidth(s) + strings.Count(s, "\n")
 		}
 		width += w
 		if width > limit {
@@ -41,7 +45,7 @@ func Truncate(input string, limit int) ([]rune, int) {
 	gr := uniseg.NewGraphemes(input)
 	for gr.Next() {
 		rs := gr.Runes()
-		w := runewidth.StringWidth(string(rs))
+		w := StringWidth(string(rs))
 		if width+w > limit {
 			return runes, width
 		}

test/test_go.rb

@@ -1930,7 +1930,7 @@ class TestGoFZF < TestBase
 
   def test_keep_right
     tmux.send_keys "seq 10000 | #{FZF} --read0 --keep-right", :Enter
-    tmux.until { |lines| assert lines.any_include?('9999 10000') }
+    tmux.until { |lines| assert lines.any_include?('9999␊10000') }
   end
 
   def test_backward_eof
@@ -2807,9 +2807,9 @@ module TestShell
     tmux.send_keys 'C-r'
     tmux.until { |lines| assert_equal '>', lines[-1] }
     tmux.send_keys 'foo bar'
-    tmux.until { |lines| assert lines[-3]&.end_with?('bar"') }
+    tmux.until { |lines| assert lines[-3]&.match?(/bar"␊?/) }
     tmux.send_keys :Enter
-    tmux.until { |lines| assert lines[-1]&.end_with?('bar"') }
+    tmux.until { |lines| assert lines[-1]&.match?(/bar"␊?/) }
     tmux.send_keys :Enter
     tmux.until { |lines| assert_equal %w[foo bar], lines[-2..] }
   end