Use textfile.Read() to read files

This converts the text to UTF-8 and removes a Byte Order Mark.

Gopkg.lock

@@ -219,7 +219,7 @@
 
 [[projects]]
   name = "golang.org/x/text"
-  packages = ["collate","collate/build","internal/colltab","internal/gen","internal/tag","internal/triegen","internal/ucd","language","secure/bidirule","transform","unicode/bidi","unicode/cldr","unicode/norm","unicode/rangetable"]
+  packages = ["collate","collate/build","encoding","encoding/internal","encoding/internal/identifier","encoding/unicode","internal/colltab","internal/gen","internal/tag","internal/triegen","internal/ucd","internal/utf8internal","language","runes","secure/bidirule","transform","unicode/bidi","unicode/cldr","unicode/norm","unicode/rangetable"]
   revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
   version = "v0.3.0"
 
@@ -250,6 +250,6 @@
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "44a8f2ed127a6eaa38c1449b97d298fc703c961617bd93565b89bcc6c9a41483"
+  inputs-digest = "a5de339cba7570216b212439b90e1e6c384c94be8342fe7755b7cb66aa0a3440"
   solver-name = "gps-cdcl"
   solver-version = 1

changelog/unreleased/issue-1433

@@ -0,0 +1,12 @@
+Enhancement: Support UTF-16 encoding and process Byte Order Mark
+
+On Windows, text editors commonly leave a Byte Order Mark at the beginning of
+the file to define which encoding is used (oftentimes UTF-16). We've added code
+to support processing the BOMs in text files, like the exclude files, the
+password file and the file passed via `--files-from`. This does not apply to
+any file being saved in a backup, those are not touched and archived as they
+are.
+
+https://github.com/restic/restic/issues/1433
+https://github.com/restic/restic/issues/1738
+https://github.com/restic/restic/pull/1748

cmd/restic/cmd_backup.go

@@ -2,8 +2,9 @@ package main
 
 import (
 	"bufio"
+	"bytes"
 	"context"
-	"io"
+	"io/ioutil"
 	"os"
 	"strconv"
 	"strings"
@@ -18,6 +19,7 @@ import (
 	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/textfile"
 	"github.com/restic/restic/internal/ui"
 	"github.com/restic/restic/internal/ui/termstatus"
 )
@@ -127,19 +129,24 @@ func readLinesFromFile(filename string) ([]string, error) {
 		return nil, nil
 	}
 
-	var r io.Reader = os.Stdin
-	if filename != "-" {
-		f, err := os.Open(filename)
-		if err != nil {
-			return nil, err
-		}
-		defer f.Close()
-		r = f
+	var (
+		data []byte
+		err  error
+	)
+
+	if filename == "-" {
+		data, err = ioutil.ReadAll(os.Stdin)
+	} else {
+		data, err = textfile.Read(filename)
+	}
+
+	if err != nil {
+		return nil, err
 	}
 
 	var lines []string
 
-	scanner := bufio.NewScanner(r)
+	scanner := bufio.NewScanner(bytes.NewReader(data))
 	for scanner.Scan() {
 		line := strings.TrimSpace(scanner.Text())
 		// ignore empty lines
@@ -232,18 +239,12 @@ func readExcludePatternsFromFiles(excludeFiles []string) []string {
 	var excludes []string
 	for _, filename := range excludeFiles {
 		err := func() (err error) {
-			file, err := fs.Open(filename)
+			data, err := textfile.Read(filename)
 			if err != nil {
 				return err
 			}
-			defer func() {
-				// return pre-close error if there was one
-				if errClose := file.Close(); err == nil {
-					err = errClose
-				}
-			}()
 
-			scanner := bufio.NewScanner(file)
+			scanner := bufio.NewScanner(bytes.NewReader(data))
 			for scanner.Scan() {
 				line := strings.TrimSpace(scanner.Text())
 

cmd/restic/global.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -30,6 +29,7 @@ import (
 	"github.com/restic/restic/internal/options"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/textfile"
 
 	"github.com/restic/restic/internal/errors"
 
@@ -235,8 +235,8 @@ func Exitf(exitcode int, format string, args ...interface{}) {
 // resolvePassword determines the password to be used for opening the repository.
 func resolvePassword(opts GlobalOptions, env string) (string, error) {
 	if opts.PasswordFile != "" {
-		s, err := ioutil.ReadFile(opts.PasswordFile)
-		if os.IsNotExist(err) {
+		s, err := textfile.Read(opts.PasswordFile)
+		if os.IsNotExist(errors.Cause(err)) {
 			return "", errors.Fatalf("%s does not exist", opts.PasswordFile)
 		}
 		return strings.TrimSpace(string(s)), errors.Wrap(err, "Readfile")