Properly detect mintty output redirection

mintty on windows always uses pipes to connect stdout between processes
and for the terminal output. The previous implementation always assumed
that stdout connected to a pipe means that stdout is displayed on a
mintty terminal. However, this detection breaks when using pipes to
connect processes and for powershell which uses pipes when redirecting
to a file.

Now the pipe filename is queried and matched against the pattern used by
msys / cygwin when connected to the terminal. In all other cases assume
that a pipe is just a regular pipe.

go.mod

@@ -37,7 +37,7 @@ require (
 	golang.org/x/net v0.0.0-20200904194848-62affa334b73
 	golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43
 	golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208
-	golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4
+	golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b
 	golang.org/x/text v0.3.4
 	google.golang.org/api v0.32.0
 	gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b // indirect

go.sum

@@ -379,8 +379,9 @@ golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201101102859-da207088b7d1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k=
 golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b h1:ggRgirZABFolTmi3sn6Ivd9SipZwLedQ5wR0aAKnFxU=
+golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221 h1:/ZHdbVpdR/jk3g30/d4yUL0JU9kksj8+F/bnQUVLGDM=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

internal/ui/termstatus/terminal_windows.go

@@ -4,6 +4,7 @@ package termstatus
 
 import (
 	"io"
+	"strings"
 	"syscall"
 	"unsafe"
 
@@ -80,6 +81,22 @@ func isPipe(fd uintptr) bool {
 	return err == nil && typ == windows.FILE_TYPE_PIPE
 }
 
+func getFileNameByHandle(fd uintptr) (string, error) {
+	type FILE_NAME_INFO struct {
+		FileNameLength int32
+		FileName       [windows.MAX_LONG_PATH]uint16
+	}
+
+	var fi FILE_NAME_INFO
+	err := windows.GetFileInformationByHandleEx(windows.Handle(fd), windows.FileNameInfo, (*byte)(unsafe.Pointer(&fi)), uint32(unsafe.Sizeof(fi)))
+	if err != nil {
+		return "", err
+	}
+
+	filename := syscall.UTF16ToString(fi.FileName[:])
+	return filename, nil
+}
+
 // CanUpdateStatus returns true if status lines can be printed, the process
 // output is not redirected to a file or pipe.
 func CanUpdateStatus(fd uintptr) bool {
@@ -88,11 +105,23 @@ func CanUpdateStatus(fd uintptr) bool {
 		return true
 	}
 
-	// check that the output file type is a pipe (0x0003)
+	// pipes require special handling
 	if !isPipe(fd) {
 		return false
 	}
 
-	// assume we're running in mintty/cygwin
+	fn, err := getFileNameByHandle(fd)
-	return true
+	if err != nil {
+		return false
+	}
+
+	// inspired by https://github.com/RyanGlScott/mintty/blob/master/src/System/Console/MinTTY/Win32.hsc
+	// terminal: \msys-dd50a72ab4668b33-pty0-to-master
+	// pipe to cat: \msys-dd50a72ab4668b33-13244-pipe-0x16
+	if (strings.HasPrefix(fn, "\\cygwin-") || strings.HasPrefix(fn, "\\msys-")) &&
+		strings.Contains(fn, "-pty") && strings.HasSuffix(fn, "-master") {
+		return true
+	}
+
+	return false
 }