octoleo/restic
2
2
Fork 0
mirror of https://github.com/octoleo/restic.git synced 2024-11-27 07:16:40 +00:00
Code Issues Releases Wiki Activity

Merge pull request #4611 from zmanda/windows-metadata-support

Browse Source 
Back up and restore windows metadata like created ts, file attribs like hidden, readonly, encrypted with a common extensible mechanism
This commit is contained in:
Michael Eischer 2024-02-23 18:16:09 +00:00 committed by GitHub
commit b953dc8f58
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
21 changed files with 1435 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
changelog/unreleased/pull-4611 Normal file
View File

@ -0,0 +1,7 @@
Enhancement: Back up windows created time and file attributes like hidden flag
Restic did not back up windows-specific meta-data like created time and file attributes like hidden flag.
Restic now backs up file created time and file attributes like hidden, readonly and encrypted flag when backing up files and folders on windows.
https://github.com/restic/restic/pull/4611

1
cmd/restic/cmd_find.go
View File

@ -126,6 +126,7 @@ func (s *statefulOutput) PrintPatternJSON(path string, node *restic.Node) {
// Make the following attributes disappear
Name byte `json:"name,omitempty"`
ExtendedAttributes byte `json:"extended_attributes,omitempty"`
GenericAttributes byte `json:"generic_attributes,omitempty"`
Device byte `json:"device,omitempty"`
Content byte `json:"content,omitempty"`
Subtree byte `json:"subtree,omitempty"`

3
cmd/restic/cmd_restore.go
View File

@ -178,6 +178,9 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
totalErrors++
return nil
}
res.Warn = func(message string) {
msg.E("Warning: %s\n", message)
}
excludePatterns := filter.ParsePatterns(opts.Exclude)
insensitiveExcludePatterns := filter.ParsePatterns(opts.InsensitiveExclude)

1
doc/040_backup.rst
View File

@ -487,7 +487,6 @@ particular note are:
* File creation date on Unix platforms
* Inode flags on Unix platforms
* File ownership and ACLs on Windows
* The "hidden" flag on Windows
Reading data from a command
***************************

31
internal/errors/errors.go
View File

@ -2,6 +2,7 @@ package errors
import (
stderrors "errors"
"fmt"
"github.com/pkg/errors"
)
@ -22,12 +23,42 @@ var Wrap = errors.Wrap
// nil, Wrapf returns nil.
var Wrapf = errors.Wrapf
// WithStack annotates err with a stack trace at the point WithStack was called.
// If err is nil, WithStack returns nil.
var WithStack = errors.WithStack
// Go 1.13-style error handling.
// As finds the first error in err's tree that matches target, and if one is found,
// sets target to that error value and returns true. Otherwise, it returns false.
func As(err error, tgt interface{}) bool { return stderrors.As(err, tgt) }
// Is reports whether any error in err's tree matches target.
func Is(x, y error) bool { return stderrors.Is(x, y) }
// Unwrap returns the result of calling the Unwrap method on err, if err's type contains
// an Unwrap method returning error. Otherwise, Unwrap returns nil.
//
// Unwrap only calls a method of the form "Unwrap() error". In particular Unwrap does not
// unwrap errors returned by [Join].
func Unwrap(err error) error { return stderrors.Unwrap(err) }
// CombineErrors combines multiple errors into a single error.
func CombineErrors(errors ...error) error {
var combinedErrorMsg string
for _, err := range errors {
if err != nil {
if combinedErrorMsg != "" {
combinedErrorMsg += "; " // Separate error messages with a delimiter
}
combinedErrorMsg += err.Error()
}
}
if combinedErrorMsg == "" {
return nil // No errors, return nil
}
return fmt.Errorf("multiple errors occurred: [%s]", combinedErrorMsg)
}

14
internal/fs/file.go
View File

@ -124,3 +124,17 @@ func RemoveIfExists(filename string) error {
func Chtimes(name string, atime time.Time, mtime time.Time) error {
return os.Chtimes(fixpath(name), atime, mtime)
}
// IsAccessDenied checks if the error is due to permission error.
func IsAccessDenied(err error) bool {
return os.IsPermission(err)
}
// ResetPermissions resets the permissions of the file at the specified path
func ResetPermissions(path string) error {
// Set the default file permissions
if err := os.Chmod(path, 0600); err != nil {
return err
}
return nil
}

26
internal/fs/file_windows.go
View File

@ -77,3 +77,29 @@ func TempFile(dir, prefix string) (f *os.File, err error) {
func Chmod(name string, mode os.FileMode) error {
return os.Chmod(fixpath(name), mode)
}
// ClearSystem removes the system attribute from the file.
func ClearSystem(path string) error {
return ClearAttribute(path, windows.FILE_ATTRIBUTE_SYSTEM)
}
// ClearAttribute removes the specified attribute from the file.
func ClearAttribute(path string, attribute uint32) error {
ptr, err := windows.UTF16PtrFromString(path)
if err != nil {
return err
}
fileAttributes, err := windows.GetFileAttributes(ptr)
if err != nil {
return err
}
if fileAttributes&attribute != 0 {
// Clear the attribute
fileAttributes &= ^uint32(attribute)
err = windows.SetFileAttributes(ptr, fileAttributes)
if err != nil {
return err
}
}
return nil
}

253
internal/restic/node.go
View File

@ -6,7 +6,9 @@ import (
"fmt"
"os"
"os/user"
"reflect"
"strconv"
"strings"
"sync"
"syscall"
"time"
@ -20,12 +22,53 @@ import (
"github.com/restic/restic/internal/fs"
)
// ExtendedAttribute is a tuple storing the xattr name and value.
// ExtendedAttribute is a tuple storing the xattr name and value for various filesystems.
type ExtendedAttribute struct {
Name string `json:"name"`
Value []byte `json:"value"`
}
// GenericAttributeType can be used for OS specific functionalities by defining specific types
// in node.go to be used by the specific node_xx files.
// OS specific attribute types should follow the convention <OS>Attributes.
// GenericAttributeTypes should follow the convention <OS specific attribute type>.<attribute name>
// The attributes in OS specific attribute types must be pointers as we want to distinguish nil values
// and not create GenericAttributes for them.
type GenericAttributeType string
// OSType is the type created to represent each specific OS
type OSType string
const (
// When new GenericAttributeType are defined, they must be added in the init function as well.
// Below are windows specific attributes.
// TypeCreationTime is the GenericAttributeType used for storing creation time for windows files within the generic attributes map.
TypeCreationTime GenericAttributeType = "windows.creation_time"
// TypeFileAttributes is the GenericAttributeType used for storing file attributes for windows files within the generic attributes map.
TypeFileAttributes GenericAttributeType = "windows.file_attributes"
// Generic Attributes for other OS types should be defined here.
)
// init is called when the package is initialized. Any new GenericAttributeTypes being created must be added here as well.
func init() {
storeGenericAttributeType(TypeCreationTime, TypeFileAttributes)
}
// genericAttributesForOS maintains a map of known genericAttributesForOS to the OSType
var genericAttributesForOS = map[GenericAttributeType]OSType{}
// storeGenericAttributeType adds and entry in genericAttributesForOS map
func storeGenericAttributeType(attributeTypes ...GenericAttributeType) {
for _, attributeType := range attributeTypes {
// Get the OS attribute type from the GenericAttributeType
osAttributeName := strings.Split(string(attributeType), ".")[0]
genericAttributesForOS[attributeType] = OSType(osAttributeName)
}
}
// Node is a file, directory or other item in a backup.
type Node struct {
Name string `json:"name"`
@ -47,11 +90,12 @@ type Node struct {
// This allows storing arbitrary byte-sequences, which are possible as symlink targets on unix systems,
// as LinkTarget without breaking backwards-compatibility.
// Must only be set of the linktarget cannot be encoded as valid utf8.
LinkTargetRaw []byte `json:"linktarget_raw,omitempty"`
ExtendedAttributes []ExtendedAttribute `json:"extended_attributes,omitempty"`
Device uint64 `json:"device,omitempty"` // in case of Type == "dev", stat.st_rdev
Content IDs `json:"content"`
Subtree *ID `json:"subtree,omitempty"`
LinkTargetRaw []byte `json:"linktarget_raw,omitempty"`
ExtendedAttributes []ExtendedAttribute `json:"extended_attributes,omitempty"`
GenericAttributes map[GenericAttributeType]json.RawMessage `json:"generic_attributes,omitempty"`
Device uint64 `json:"device,omitempty"` // in case of Type == "dev", stat.st_rdev
Content IDs `json:"content"`
Subtree *ID `json:"subtree,omitempty"`
Error string `json:"error,omitempty"`
@ -180,8 +224,8 @@ func (node *Node) CreateAt(ctx context.Context, path string, repo BlobLoader) er
}
// RestoreMetadata restores node metadata
func (node Node) RestoreMetadata(path string) error {
err := node.restoreMetadata(path)
func (node Node) RestoreMetadata(path string, warn func(msg string)) error {
err := node.restoreMetadata(path, warn)
if err != nil {
debug.Log("restoreMetadata(%s) error %v", path, err)
}
@ -189,7 +233,7 @@ func (node Node) RestoreMetadata(path string) error {
return err
}
func (node Node) restoreMetadata(path string) error {
func (node Node) restoreMetadata(path string, warn func(msg string)) error {
var firsterr error
if err := lchown(path, int(node.UID), int(node.GID)); err != nil {
@ -203,14 +247,6 @@ func (node Node) restoreMetadata(path string) error {
}
}
if node.Type != "symlink" {
if err := fs.Chmod(path, node.Mode); err != nil {
if firsterr != nil {
firsterr = errors.WithStack(err)
}
}
}
if err := node.RestoreTimestamps(path); err != nil {
debug.Log("error restoring timestamps for dir %v: %v", path, err)
if firsterr != nil {
@ -225,6 +261,24 @@ func (node Node) restoreMetadata(path string) error {
}
}
if err := node.restoreGenericAttributes(path, warn); err != nil {
debug.Log("error restoring generic attributes for %v: %v", path, err)
if firsterr != nil {
firsterr = err
}
}
// Moving RestoreTimestamps and restoreExtendedAttributes calls above as for readonly files in windows
// calling Chmod below will no longer allow any modifications to be made on the file and the
// calls above would fail.
if node.Type != "symlink" {
if err := fs.Chmod(path, node.Mode); err != nil {
if firsterr != nil {
firsterr = errors.WithStack(err)
}
}
}
return firsterr
}
@ -438,6 +492,9 @@ func (node Node) Equals(other Node) bool {
if !node.sameExtendedAttributes(other) {
return false
}
if !node.sameGenericAttributes(other) {
return false
}
if node.Subtree != nil {
if other.Subtree == nil {
return false
@ -480,8 +537,13 @@ func (node Node) sameContent(other Node) bool {
}
func (node Node) sameExtendedAttributes(other Node) bool {
if len(node.ExtendedAttributes) != len(other.ExtendedAttributes) {
ln := len(node.ExtendedAttributes)
lo := len(other.ExtendedAttributes)
if ln != lo {
return false
} else if ln == 0 {
// This means lo is also of length 0
return true
}
// build a set of all attributes that node has
@ -525,6 +587,33 @@ func (node Node) sameExtendedAttributes(other Node) bool {
return true
}
func (node Node) sameGenericAttributes(other Node) bool {
return deepEqual(node.GenericAttributes, other.GenericAttributes)
}
func deepEqual(map1, map2 map[GenericAttributeType]json.RawMessage) bool {
// Check if the maps have the same number of keys
if len(map1) != len(map2) {
return false
}
// Iterate over each key-value pair in map1
for key, value1 := range map1 {
// Check if the key exists in map2
value2, ok := map2[key]
if !ok {
return false
}
// Check if the JSON.RawMessage values are equal byte by byte
if !bytes.Equal(value1, value2) {
return false
}
}
return true
}
func (node *Node) fillUser(stat *statT) {
uid, gid := stat.uid(), stat.gid()
node.UID, node.GID = uid, gid
@ -627,7 +716,17 @@ func (node *Node) fillExtra(path string, fi os.FileInfo) error {
return errors.Errorf("unsupported file type %q", node.Type)
}
return node.fillExtendedAttributes(path)
allowExtended, err := node.fillGenericAttributes(path, fi, stat)
if allowExtended {
// Skip processing ExtendedAttributes if allowExtended is false.
errEx := node.fillExtendedAttributes(path)
if err == nil {
err = errEx
} else {
debug.Log("Error filling extended attributes for %v at %v : %v", node.Name, path, errEx)
}
}
return err
}
func (node *Node) fillExtendedAttributes(path string) error {
@ -665,3 +764,119 @@ func (node *Node) fillTimes(stat *statT) {
node.ChangeTime = time.Unix(ctim.Unix())
node.AccessTime = time.Unix(atim.Unix())
}
// HandleUnknownGenericAttributesFound is used for handling and distinguing between scenarios related to future versions and cross-OS repositories
func HandleUnknownGenericAttributesFound(unknownAttribs []GenericAttributeType, warn func(msg string)) {
for _, unknownAttrib := range unknownAttribs {
handleUnknownGenericAttributeFound(unknownAttrib, warn)
}
}
// handleUnknownGenericAttributeFound is used for handling and distinguing between scenarios related to future versions and cross-OS repositories
func handleUnknownGenericAttributeFound(genericAttributeType GenericAttributeType, warn func(msg string)) {
if checkGenericAttributeNameNotHandledAndPut(genericAttributeType) {
// Print the unique error only once for a given execution
os, exists := genericAttributesForOS[genericAttributeType]
if exists {
// If genericAttributesForOS contains an entry but we still got here, it means the specific node_xx.go for the current OS did not handle it and the repository may have been originally created on a different OS.
// The fact that node.go knows about the attribute, means it is not a new attribute. This may be a common situation if a repo is used across OSs.
debug.Log("Ignoring a generic attribute found in the repository: %s which may not be compatible with your OS. Compatible OS: %s", genericAttributeType, os)
} else {
// If genericAttributesForOS in node.go does not know about this attribute, then the repository may have been created by a newer version which has a newer GenericAttributeType.
warn(fmt.Sprintf("Found an unrecognized generic attribute in the repository: %s. You may need to upgrade to latest version of restic.", genericAttributeType))
}
}
}
// handleAllUnknownGenericAttributesFound performs validations for all generic attributes in the node.
// This is not used on windows currently because windows has handling for generic attributes.
// nolint:unused
func (node Node) handleAllUnknownGenericAttributesFound(warn func(msg string)) error {
for name := range node.GenericAttributes {
handleUnknownGenericAttributeFound(name, warn)
}
return nil
}
var unknownGenericAttributesHandlingHistory sync.Map
// checkGenericAttributeNameNotHandledAndPut checks if the GenericAttributeType name entry
// already exists and puts it in the map if not.
func checkGenericAttributeNameNotHandledAndPut(value GenericAttributeType) bool {
// If Key doesn't exist, put the value and return true because it is not already handled
_, exists := unknownGenericAttributesHandlingHistory.LoadOrStore(value, "")
// Key exists, then it is already handled so return false
return !exists
}
// The functions below are common helper functions which can be used for generic attributes support
// across different OS.
// genericAttributesToOSAttrs gets the os specific attribute from the generic attribute using reflection
// nolint:unused
func genericAttributesToOSAttrs(attrs map[GenericAttributeType]json.RawMessage, attributeType reflect.Type, attributeValuePtr *reflect.Value, keyPrefix string) (unknownAttribs []GenericAttributeType, err error) {
attributeValue := *attributeValuePtr
for key, rawMsg := range attrs {
found := false
for i := 0; i < attributeType.NumField(); i++ {
if getFQKeyByIndex(attributeType, i, keyPrefix) == key {
found = true
fieldValue := attributeValue.Field(i)
// For directly supported types, use json.Unmarshal directly
if err := json.Unmarshal(rawMsg, fieldValue.Addr().Interface()); err != nil {
return unknownAttribs, errors.Wrap(err, "Unmarshal")
}
break
}
}
if !found {
unknownAttribs = append(unknownAttribs, key)
}
}
return unknownAttribs, nil
}
// getFQKey gets the fully qualified key for the field
// nolint:unused
func getFQKey(field reflect.StructField, keyPrefix string) GenericAttributeType {
return GenericAttributeType(fmt.Sprintf("%s.%s", keyPrefix, field.Tag.Get("generic")))
}
// getFQKeyByIndex gets the fully qualified key for the field index
// nolint:unused
func getFQKeyByIndex(attributeType reflect.Type, index int, keyPrefix string) GenericAttributeType {
return getFQKey(attributeType.Field(index), keyPrefix)
}
// osAttrsToGenericAttributes gets the generic attribute from the os specific attribute using reflection
// nolint:unused
func osAttrsToGenericAttributes(attributeType reflect.Type, attributeValuePtr *reflect.Value, keyPrefix string) (attrs map[GenericAttributeType]json.RawMessage, err error) {
attributeValue := *attributeValuePtr
attrs = make(map[GenericAttributeType]json.RawMessage)
// Iterate over the fields of the struct
for i := 0; i < attributeType.NumField(); i++ {
field := attributeType.Field(i)
// Get the field value using reflection
fieldValue := attributeValue.FieldByName(field.Name)
// Check if the field is nil
if fieldValue.IsNil() {
// If it's nil, skip this field
continue
}
// Marshal the field value into a json.RawMessage
var fieldBytes []byte
if fieldBytes, err = json.Marshal(fieldValue.Interface()); err != nil {
return attrs, errors.Wrap(err, "Marshal")
}
// Insert the field into the map
attrs[getFQKey(field, keyPrefix)] = json.RawMessage(fieldBytes)
}
return attrs, nil
}

17
internal/restic/node_aix.go
View File

@ -3,9 +3,12 @@
package restic
import "syscall"
import (
"os"
"syscall"
)
func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error {
func (node Node) restoreSymlinkTimestamps(_ string, _ [2]syscall.Timespec) error {
return nil
}
@ -34,3 +37,13 @@ func Listxattr(path string) ([]string, error) {
func Setxattr(path, name string, data []byte) error {
return nil
}
// restoreGenericAttributes is no-op on AIX.
func (node *Node) restoreGenericAttributes(_ string, warn func(msg string)) error {
return node.handleAllUnknownGenericAttributesFound(warn)
}
// fillGenericAttributes is a no-op on AIX.
func (node *Node) fillGenericAttributes(_ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) {
return true, nil
}

24
internal/restic/node_netbsd.go
View File

@ -1,8 +1,11 @@
package restic
import "syscall"
import (
"os"
"syscall"
)
func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error {
func (node Node) restoreSymlinkTimestamps(_ string, _ [2]syscall.Timespec) error {
return nil
}
@ -10,18 +13,27 @@ func (s statT) atim() syscall.Timespec { return s.Atimespec }
func (s statT) mtim() syscall.Timespec { return s.Mtimespec }
func (s statT) ctim() syscall.Timespec { return s.Ctimespec }
// Getxattr retrieves extended attribute data associated with path.
// Getxattr is a no-op on netbsd.
func Getxattr(path, name string) ([]byte, error) {
return nil, nil
}
// Listxattr retrieves a list of names of extended attributes associated with the
// given path in the file system.
// Listxattr is a no-op on netbsd.
func Listxattr(path string) ([]string, error) {
return nil, nil
}
// Setxattr associates name and data together as an attribute of path.
// Setxattr is a no-op on netbsd.
func Setxattr(path, name string, data []byte) error {
return nil
}
// restoreGenericAttributes is no-op on netbsd.
func (node *Node) restoreGenericAttributes(_ string, warn func(msg string)) error {
return node.handleAllUnknownGenericAttributesFound(warn)
}
// fillGenericAttributes is a no-op on netbsd.
func (node *Node) fillGenericAttributes(_ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) {
return true, nil
}

24
internal/restic/node_openbsd.go
View File

@ -1,8 +1,11 @@
package restic
import "syscall"
import (
"os"
"syscall"
)
func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error {
func (node Node) restoreSymlinkTimestamps(_ string, _ [2]syscall.Timespec) error {
return nil
}
@ -10,18 +13,27 @@ func (s statT) atim() syscall.Timespec { return s.Atim }
func (s statT) mtim() syscall.Timespec { return s.Mtim }
func (s statT) ctim() syscall.Timespec { return s.Ctim }
// Getxattr retrieves extended attribute data associated with path.
// Getxattr is a no-op on openbsd.
func Getxattr(path, name string) ([]byte, error) {
return nil, nil
}
// Listxattr retrieves a list of names of extended attributes associated with the
// given path in the file system.
// Listxattr is a no-op on openbsd.
func Listxattr(path string) ([]string, error) {
return nil, nil
}
// Setxattr associates name and data together as an attribute of path.
// Setxattr is a no-op on openbsd.
func Setxattr(path, name string, data []byte) error {
return nil
}
// restoreGenericAttributes is no-op on openbsd.
func (node *Node) restoreGenericAttributes(_ string, warn func(msg string)) error {
return node.handleAllUnknownGenericAttributesFound(warn)
}
// fillGenericAttributes is a no-op on openbsd.
func (node *Node) fillGenericAttributes(_ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) {
return true, nil
}

37
internal/restic/node_test.go
View File

@ -1,4 +1,4 @@
package restic_test
package restic
import (
"context"
@ -11,7 +11,6 @@ import (
"testing"
"time"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test"
rtest "github.com/restic/restic/internal/test"
)
@ -32,7 +31,7 @@ func BenchmarkNodeFillUser(t *testing.B) {
t.ResetTimer()
for i := 0; i < t.N; i++ {
_, err := restic.NodeFromFileInfo(path, fi)
_, err := NodeFromFileInfo(path, fi)
rtest.OK(t, err)
}
@ -56,7 +55,7 @@ func BenchmarkNodeFromFileInfo(t *testing.B) {
t.ResetTimer()
for i := 0; i < t.N; i++ {
_, err := restic.NodeFromFileInfo(path, fi)
_, err := NodeFromFileInfo(path, fi)
if err != nil {
t.Fatal(err)
}
@ -75,11 +74,11 @@ func parseTime(s string) time.Time {
return t.Local()
}
var nodeTests = []restic.Node{
var nodeTests = []Node{
{
Name: "testFile",
Type: "file",
Content: restic.IDs{},
Content: IDs{},
UID: uint32(os.Getuid()),
GID: uint32(os.Getgid()),
Mode: 0604,
@ -90,7 +89,7 @@ var nodeTests = []restic.Node{
{
Name: "testSuidFile",
Type: "file",
Content: restic.IDs{},
Content: IDs{},
UID: uint32(os.Getuid()),
GID: uint32(os.Getgid()),
Mode: 0755 | os.ModeSetuid,
@ -101,7 +100,7 @@ var nodeTests = []restic.Node{
{
Name: "testSuidFile2",
Type: "file",
Content: restic.IDs{},
Content: IDs{},
UID: uint32(os.Getuid()),
GID: uint32(os.Getgid()),
Mode: 0755 | os.ModeSetgid,
@ -112,7 +111,7 @@ var nodeTests = []restic.Node{
{
Name: "testSticky",
Type: "file",
Content: restic.IDs{},
Content: IDs{},
UID: uint32(os.Getuid()),
GID: uint32(os.Getgid()),
Mode: 0755 | os.ModeSticky,
@ -148,7 +147,7 @@ var nodeTests = []restic.Node{
{
Name: "testFile",
Type: "file",
Content: restic.IDs{},
Content: IDs{},
UID: uint32(os.Getuid()),
GID: uint32(os.Getgid()),
Mode: 0604,
@ -170,14 +169,14 @@ var nodeTests = []restic.Node{
{
Name: "testXattrFile",
Type: "file",
Content: restic.IDs{},
Content: IDs{},
UID: uint32(os.Getuid()),
GID: uint32(os.Getgid()),
Mode: 0604,
ModTime: parseTime("2005-05-14 21:07:03.111"),
AccessTime: parseTime("2005-05-14 21:07:04.222"),
ChangeTime: parseTime("2005-05-14 21:07:05.333"),
ExtendedAttributes: []restic.ExtendedAttribute{
ExtendedAttributes: []ExtendedAttribute{
{"user.foo", []byte("bar")},
},
},
@ -191,7 +190,7 @@ var nodeTests = []restic.Node{
ModTime: parseTime("2005-05-14 21:07:03.111"),
AccessTime: parseTime("2005-05-14 21:07:04.222"),
ChangeTime: parseTime("2005-05-14 21:07:05.333"),
ExtendedAttributes: []restic.ExtendedAttribute{
ExtendedAttributes: []ExtendedAttribute{
{"user.foo", []byte("bar")},
},
},
@ -219,7 +218,7 @@ func TestNodeRestoreAt(t *testing.T) {
nodePath = filepath.Join(tempdir, test.Name)
}
rtest.OK(t, test.CreateAt(context.TODO(), nodePath, nil))
rtest.OK(t, test.RestoreMetadata(nodePath))
rtest.OK(t, test.RestoreMetadata(nodePath, func(msg string) { rtest.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", nodePath, msg)) }))
if test.Type == "dir" {
rtest.OK(t, test.RestoreTimestamps(nodePath))
@ -228,7 +227,7 @@ func TestNodeRestoreAt(t *testing.T) {
fi, err := os.Lstat(nodePath)
rtest.OK(t, err)
n2, err := restic.NodeFromFileInfo(nodePath, fi)
n2, err := NodeFromFileInfo(nodePath, fi)
rtest.OK(t, err)
rtest.Assert(t, test.Name == n2.Name,
@ -330,7 +329,7 @@ func TestFixTime(t *testing.T) {
for _, test := range tests {
t.Run("", func(t *testing.T) {
res := restic.FixTime(test.src)
res := FixTime(test.src)
if !res.Equal(test.want) {
t.Fatalf("wrong result for %v, want:\n %v\ngot:\n %v", test.src, test.want, res)
}
@ -343,12 +342,12 @@ func TestSymlinkSerialization(t *testing.T) {
"válîd \t Üñi¢òde \n śẗŕinǵ",
string([]byte{0, 1, 2, 0xfa, 0xfb, 0xfc}),
} {
n := restic.Node{
n := Node{
LinkTarget: link,
}
ser, err := json.Marshal(n)
test.OK(t, err)
var n2 restic.Node
var n2 Node
err = json.Unmarshal(ser, &n2)
test.OK(t, err)
fmt.Println(string(ser))
@ -365,7 +364,7 @@ func TestSymlinkSerializationFormat(t *testing.T) {
{`{"linktarget":"test"}`, "test"},
{`{"linktarget":"\u0000\u0001\u0002\ufffd\ufffd\ufffd","linktarget_raw":"AAEC+vv8"}`, string([]byte{0, 1, 2, 0xfa, 0xfb, 0xfc})},
} {
var n2 restic.Node
var n2 Node
err := json.Unmarshal([]byte(d.ser), &n2)
test.OK(t, err)
test.Equals(t, d.linkTarget, n2.LinkTarget)

224
internal/restic/node_windows.go
View File

@ -1,21 +1,47 @@
package restic
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"reflect"
"runtime"
"strings"
"syscall"
"unsafe"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/fs"
"golang.org/x/sys/windows"
)
// WindowsAttributes are the genericAttributes for Windows OS
type WindowsAttributes struct {
// CreationTime is used for storing creation time for windows files.
CreationTime *syscall.Filetime `generic:"creation_time"`
// FileAttributes is used for storing file attributes for windows files.
FileAttributes *uint32 `generic:"file_attributes"`
}
var (
modAdvapi32 = syscall.NewLazyDLL("advapi32.dll")
procEncryptFile = modAdvapi32.NewProc("EncryptFileW")
procDecryptFile = modAdvapi32.NewProc("DecryptFileW")
)
// mknod is not supported on Windows.
func mknod(path string, mode uint32, dev uint64) (err error) {
func mknod(_ string, mode uint32, dev uint64) (err error) {
return errors.New("device nodes cannot be created on windows")
}
// Windows doesn't need lchown
func lchown(path string, uid int, gid int) (err error) {
func lchown(_ string, uid int, gid int) (err error) {
return nil
}
// restoreSymlinkTimestamps restores timestamps for symlinks
func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error {
// tweaked version of UtimesNano from go/src/syscall/syscall_windows.go
pathp, e := syscall.UTF16PtrFromString(path)
@ -28,7 +54,14 @@ func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespe
if e != nil {
return e
}
defer syscall.Close(h)
defer func() {
err := syscall.Close(h)
if err != nil {
debug.Log("Error closing file handle for %s: %v\n", path, err)
}
}()
a := syscall.NsecToFiletime(syscall.TimespecToNsec(utimes[0]))
w := syscall.NsecToFiletime(syscall.TimespecToNsec(utimes[1]))
return syscall.SetFileTime(h, nil, &a, &w)
@ -83,3 +116,188 @@ func (s statT) ctim() syscall.Timespec {
// Windows does not have the concept of a "change time" in the sense Unix uses it, so we're using the LastWriteTime here.
return syscall.NsecToTimespec(s.LastWriteTime.Nanoseconds())
}
// restoreGenericAttributes restores generic attributes for Windows
func (node Node) restoreGenericAttributes(path string, warn func(msg string)) (err error) {
if len(node.GenericAttributes) == 0 {
return nil
}
var errs []error
windowsAttributes, unknownAttribs, err := genericAttributesToWindowsAttrs(node.GenericAttributes)
if err != nil {
return fmt.Errorf("error parsing generic attribute for: %s : %v", path, err)
}
if windowsAttributes.CreationTime != nil {
if err := restoreCreationTime(path, windowsAttributes.CreationTime); err != nil {
errs = append(errs, fmt.Errorf("error restoring creation time for: %s : %v", path, err))
}
}
if windowsAttributes.FileAttributes != nil {
if err := restoreFileAttributes(path, windowsAttributes.FileAttributes); err != nil {
errs = append(errs, fmt.Errorf("error restoring file attributes for: %s : %v", path, err))
}
}
HandleUnknownGenericAttributesFound(unknownAttribs, warn)
return errors.CombineErrors(errs...)
}
// genericAttributesToWindowsAttrs converts the generic attributes map to a WindowsAttributes and also returns a string of unkown attributes that it could not convert.
func genericAttributesToWindowsAttrs(attrs map[GenericAttributeType]json.RawMessage) (windowsAttributes WindowsAttributes, unknownAttribs []GenericAttributeType, err error) {
waValue := reflect.ValueOf(&windowsAttributes).Elem()
unknownAttribs, err = genericAttributesToOSAttrs(attrs, reflect.TypeOf(windowsAttributes), &waValue, "windows")
return windowsAttributes, unknownAttribs, err
}
// restoreCreationTime gets the creation time from the data and sets it to the file/folder at
// the specified path.
func restoreCreationTime(path string, creationTime *syscall.Filetime) (err error) {
pathPointer, err := syscall.UTF16PtrFromString(path)
if err != nil {
return err
}
handle, err := syscall.CreateFile(pathPointer,
syscall.FILE_WRITE_ATTRIBUTES, syscall.FILE_SHARE_WRITE, nil,
syscall.OPEN_EXISTING, syscall.FILE_FLAG_BACKUP_SEMANTICS, 0)
if err != nil {
return err
}
defer func() {
if err := syscall.Close(handle); err != nil {
debug.Log("Error closing file handle for %s: %v\n", path, err)
}
}()
return syscall.SetFileTime(handle, creationTime, nil, nil)
}
// restoreFileAttributes gets the File Attributes from the data and sets them to the file/folder
// at the specified path.
func restoreFileAttributes(path string, fileAttributes *uint32) (err error) {
pathPointer, err := syscall.UTF16PtrFromString(path)
if err != nil {
return err
}
err = fixEncryptionAttribute(path, fileAttributes, pathPointer)
if err != nil {
debug.Log("Could not change encryption attribute for path: %s: %v", path, err)
}
return syscall.SetFileAttributes(pathPointer, *fileAttributes)
}
// fixEncryptionAttribute checks if a file needs to be marked encrypted and is not already encrypted, it sets
// the FILE_ATTRIBUTE_ENCRYPTED. Conversely, if the file needs to be marked unencrypted and it is already
// marked encrypted, it removes the FILE_ATTRIBUTE_ENCRYPTED.
func fixEncryptionAttribute(path string, attrs *uint32, pathPointer *uint16) (err error) {
if *attrs&windows.FILE_ATTRIBUTE_ENCRYPTED != 0 {
// File should be encrypted.
err = encryptFile(pathPointer)
if err != nil {
if fs.IsAccessDenied(err) {
// If existing file already has readonly or system flag, encrypt file call fails.
// We have already cleared readonly flag, clearing system flag if needed.
// The readonly and system flags will be set again at the end of this func if they are needed.
err = fs.ClearSystem(path)
if err != nil {
return fmt.Errorf("failed to encrypt file: failed to clear system flag: %s : %v", path, err)
}
err = encryptFile(pathPointer)
if err != nil {
return fmt.Errorf("failed to encrypt file: %s : %v", path, err)
}
} else {
return fmt.Errorf("failed to encrypt file: %s : %v", path, err)
}
}
} else {
existingAttrs, err := windows.GetFileAttributes(pathPointer)
if err != nil {
return fmt.Errorf("failed to get file attributes for existing file: %s : %v", path, err)
}
if existingAttrs&windows.FILE_ATTRIBUTE_ENCRYPTED != 0 {
// File should not be encrypted, but its already encrypted. Decrypt it.
err = decryptFile(pathPointer)
if err != nil {
if fs.IsAccessDenied(err) {
// If existing file already has readonly or system flag, decrypt file call fails.
// We have already cleared readonly flag, clearing system flag if needed.
// The readonly and system flags will be set again after this func if they are needed.
err = fs.ClearSystem(path)
if err != nil {
return fmt.Errorf("failed to decrypt file: failed to clear system flag: %s : %v", path, err)
}
err = decryptFile(pathPointer)
if err != nil {
return fmt.Errorf("failed to decrypt file: %s : %v", path, err)
}
} else {
return fmt.Errorf("failed to decrypt file: %s : %v", path, err)
}
}
}
}
return err
}
// encryptFile set the encrypted flag on the file.
func encryptFile(pathPointer *uint16) error {
// Call EncryptFile function
ret, _, err := procEncryptFile.Call(uintptr(unsafe.Pointer(pathPointer)))
if ret == 0 {
return err
}
return nil
}
// decryptFile removes the encrypted flag from the file.
func decryptFile(pathPointer *uint16) error {
// Call DecryptFile function
ret, _, err := procDecryptFile.Call(uintptr(unsafe.Pointer(pathPointer)))
if ret == 0 {
return err
}
return nil
}
// fillGenericAttributes fills in the generic attributes for windows like File Attributes,
// Created time etc.
func (node *Node) fillGenericAttributes(path string, fi os.FileInfo, stat *statT) (allowExtended bool, err error) {
if strings.Contains(filepath.Base(path), ":") {
//Do not process for Alternate Data Streams in Windows
// Also do not allow processing of extended attributes for ADS.
return false, nil
}
if !strings.HasSuffix(filepath.Clean(path), `\`) {
// Do not process file attributes and created time for windows directories like
// C:, D:
// Filepath.Clean(path) ends with '\' for Windows root drives only.
// Add Windows attributes
node.GenericAttributes, err = WindowsAttrsToGenericAttributes(WindowsAttributes{
CreationTime: getCreationTime(fi, path),
FileAttributes: &stat.FileAttributes,
})
}
return true, err
}
// windowsAttrsToGenericAttributes converts the WindowsAttributes to a generic attributes map using reflection
func WindowsAttrsToGenericAttributes(windowsAttributes WindowsAttributes) (attrs map[GenericAttributeType]json.RawMessage, err error) {
// Get the value of the WindowsAttributes
windowsAttributesValue := reflect.ValueOf(windowsAttributes)
return osAttrsToGenericAttributes(reflect.TypeOf(windowsAttributes), &windowsAttributesValue, runtime.GOOS)
}
// getCreationTime gets the value for the WindowsAttribute CreationTime in a windows specific time format.
// The value is a 64-bit value representing the number of 100-nanosecond intervals since January 1, 1601 (UTC)
// split into two 32-bit parts: the low-order DWORD and the high-order DWORD for efficiency and interoperability.
// The low-order DWORD represents the number of 100-nanosecond intervals elapsed since January 1, 1601, modulo
// 2^32. The high-order DWORD represents the number of times the low-order DWORD has overflowed.
func getCreationTime(fi os.FileInfo, path string) (creationTimeAttribute *syscall.Filetime) {
attrib, success := fi.Sys().(*syscall.Win32FileAttributeData)
if success && attrib != nil {
return &attrib.CreationTime
} else {
debug.Log("Could not get create time for path: %s", path)
return nil
}
}

210
internal/restic/node_windows_test.go Normal file
View File

@ -0,0 +1,210 @@
//go:build windows
// +build windows
package restic
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"syscall"
"testing"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/test"
"golang.org/x/sys/windows"
)
func TestRestoreCreationTime(t *testing.T) {
t.Parallel()
path := t.TempDir()
fi, err := os.Lstat(path)
test.OK(t, errors.Wrapf(err, "Could not Lstat for path: %s", path))
creationTimeAttribute := getCreationTime(fi, path)
test.OK(t, errors.Wrapf(err, "Could not get creation time for path: %s", path))
//Using the temp dir creation time as the test creation time for the test file and folder
runGenericAttributesTest(t, path, TypeCreationTime, WindowsAttributes{CreationTime: creationTimeAttribute}, false)
}
func TestRestoreFileAttributes(t *testing.T) {
t.Parallel()
genericAttributeName := TypeFileAttributes
tempDir := t.TempDir()
normal := uint32(syscall.FILE_ATTRIBUTE_NORMAL)
hidden := uint32(syscall.FILE_ATTRIBUTE_HIDDEN)
system := uint32(syscall.FILE_ATTRIBUTE_SYSTEM)
archive := uint32(syscall.FILE_ATTRIBUTE_ARCHIVE)
encrypted := uint32(windows.FILE_ATTRIBUTE_ENCRYPTED)
fileAttributes := []WindowsAttributes{
//normal
{FileAttributes: &normal},
//hidden
{FileAttributes: &hidden},
//system
{FileAttributes: &system},
//archive
{FileAttributes: &archive},
//encrypted
{FileAttributes: &encrypted},
}
for i, fileAttr := range fileAttributes {
genericAttrs, err := WindowsAttrsToGenericAttributes(fileAttr)
test.OK(t, err)
expectedNodes := []Node{
{
Name: fmt.Sprintf("testfile%d", i),
Type: "file",
Mode: 0655,
ModTime: parseTime("2005-05-14 21:07:03.111"),
AccessTime: parseTime("2005-05-14 21:07:04.222"),
ChangeTime: parseTime("2005-05-14 21:07:05.333"),
GenericAttributes: genericAttrs,
},
}
runGenericAttributesTestForNodes(t, expectedNodes, tempDir, genericAttributeName, fileAttr, false)
}
normal = uint32(syscall.FILE_ATTRIBUTE_DIRECTORY)
hidden = uint32(syscall.FILE_ATTRIBUTE_DIRECTORY | syscall.FILE_ATTRIBUTE_HIDDEN)
system = uint32(syscall.FILE_ATTRIBUTE_DIRECTORY | windows.FILE_ATTRIBUTE_SYSTEM)
archive = uint32(syscall.FILE_ATTRIBUTE_DIRECTORY | windows.FILE_ATTRIBUTE_ARCHIVE)
encrypted = uint32(syscall.FILE_ATTRIBUTE_DIRECTORY | windows.FILE_ATTRIBUTE_ENCRYPTED)
folderAttributes := []WindowsAttributes{
//normal
{FileAttributes: &normal},
//hidden
{FileAttributes: &hidden},
//system
{FileAttributes: &system},
//archive
{FileAttributes: &archive},
//encrypted
{FileAttributes: &encrypted},
}
for i, folderAttr := range folderAttributes {
genericAttrs, err := WindowsAttrsToGenericAttributes(folderAttr)
test.OK(t, err)
expectedNodes := []Node{
{
Name: fmt.Sprintf("testdirectory%d", i),
Type: "dir",
Mode: 0755,
ModTime: parseTime("2005-05-14 21:07:03.111"),
AccessTime: parseTime("2005-05-14 21:07:04.222"),
ChangeTime: parseTime("2005-05-14 21:07:05.333"),
GenericAttributes: genericAttrs,
},
}
runGenericAttributesTestForNodes(t, expectedNodes, tempDir, genericAttributeName, folderAttr, false)
}
}
func runGenericAttributesTest(t *testing.T, tempDir string, genericAttributeName GenericAttributeType, genericAttributeExpected WindowsAttributes, warningExpected bool) {
genericAttributes, err := WindowsAttrsToGenericAttributes(genericAttributeExpected)
test.OK(t, err)
expectedNodes := []Node{
{
Name: "testfile",
Type: "file",
Mode: 0644,
ModTime: parseTime("2005-05-14 21:07:03.111"),
AccessTime: parseTime("2005-05-14 21:07:04.222"),
ChangeTime: parseTime("2005-05-14 21:07:05.333"),
GenericAttributes: genericAttributes,
},
{
Name: "testdirectory",
Type: "dir",
Mode: 0755,
ModTime: parseTime("2005-05-14 21:07:03.111"),
AccessTime: parseTime("2005-05-14 21:07:04.222"),
ChangeTime: parseTime("2005-05-14 21:07:05.333"),
GenericAttributes: genericAttributes,
},
}
runGenericAttributesTestForNodes(t, expectedNodes, tempDir, genericAttributeName, genericAttributeExpected, warningExpected)
}
func runGenericAttributesTestForNodes(t *testing.T, expectedNodes []Node, tempDir string, genericAttr GenericAttributeType, genericAttributeExpected WindowsAttributes, warningExpected bool) {
for _, testNode := range expectedNodes {
testPath, node := restoreAndGetNode(t, tempDir, testNode, warningExpected)
rawMessage := node.GenericAttributes[genericAttr]
genericAttrsExpected, err := WindowsAttrsToGenericAttributes(genericAttributeExpected)
test.OK(t, err)
rawMessageExpected := genericAttrsExpected[genericAttr]
test.Equals(t, rawMessageExpected, rawMessage, "Generic attribute: %s got from NodeFromFileInfo not equal for path: %s", string(genericAttr), testPath)
}
}
func restoreAndGetNode(t *testing.T, tempDir string, testNode Node, warningExpected bool) (string, *Node) {
testPath := filepath.Join(tempDir, "001", testNode.Name)
err := os.MkdirAll(filepath.Dir(testPath), testNode.Mode)
test.OK(t, errors.Wrapf(err, "Failed to create parent directories for: %s", testPath))
if testNode.Type == "file" {
testFile, err := os.Create(testPath)
test.OK(t, errors.Wrapf(err, "Failed to create test file: %s", testPath))
testFile.Close()
} else if testNode.Type == "dir" {
err := os.Mkdir(testPath, testNode.Mode)
test.OK(t, errors.Wrapf(err, "Failed to create test directory: %s", testPath))
}
err = testNode.RestoreMetadata(testPath, func(msg string) {
if warningExpected {
test.Assert(t, warningExpected, "Warning triggered as expected: %s", msg)
} else {
// If warning is not expected, this code should not get triggered.
test.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", testPath, msg))
}
})
test.OK(t, errors.Wrapf(err, "Failed to restore metadata for: %s", testPath))
fi, err := os.Lstat(testPath)
test.OK(t, errors.Wrapf(err, "Could not Lstat for path: %s", testPath))
nodeFromFileInfo, err := NodeFromFileInfo(testPath, fi)
test.OK(t, errors.Wrapf(err, "Could not get NodeFromFileInfo for path: %s", testPath))
return testPath, nodeFromFileInfo
}
const TypeSomeNewAttribute GenericAttributeType = "MockAttributes.SomeNewAttribute"
func TestNewGenericAttributeType(t *testing.T) {
t.Parallel()
newGenericAttribute := map[GenericAttributeType]json.RawMessage{}
newGenericAttribute[TypeSomeNewAttribute] = []byte("any value")
tempDir := t.TempDir()
expectedNodes := []Node{
{
Name: "testfile",
Type: "file",
Mode: 0644,
ModTime: parseTime("2005-05-14 21:07:03.111"),
AccessTime: parseTime("2005-05-14 21:07:04.222"),
ChangeTime: parseTime("2005-05-14 21:07:05.333"),
GenericAttributes: newGenericAttribute,
},
{
Name: "testdirectory",
Type: "dir",
Mode: 0755,
ModTime: parseTime("2005-05-14 21:07:03.111"),
AccessTime: parseTime("2005-05-14 21:07:04.222"),
ChangeTime: parseTime("2005-05-14 21:07:05.333"),
GenericAttributes: newGenericAttribute,
},
}
for _, testNode := range expectedNodes {
testPath, node := restoreAndGetNode(t, tempDir, testNode, true)
_, ua, err := genericAttributesToWindowsAttrs(node.GenericAttributes)
test.OK(t, err)
// Since this GenericAttribute is unknown to this version of the software, it will not get set on the file.
test.Assert(t, len(ua) == 0, "Unkown attributes: %s found for path: %s", ua, testPath)
}
}

11
internal/restic/node_xattr.go
View File

@ -4,6 +4,7 @@
package restic
import (
"os"
"syscall"
"github.com/restic/restic/internal/errors"
@ -47,3 +48,13 @@ func handleXattrErr(err error) error {
return errors.WithStack(e)
}
}
// restoreGenericAttributes is no-op.
func (node *Node) restoreGenericAttributes(_ string, warn func(msg string)) error {
return node.handleAllUnknownGenericAttributesFound(warn)
}
// fillGenericAttributes is a no-op.
func (node *Node) fillGenericAttributes(_ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) {
return true, nil
}

28
internal/restorer/fileswriter.go
View File

@ -50,16 +50,26 @@ func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, create
bucket.files[path].users++
return wr, nil
}
var flags int
var f *os.File
var err error
if createSize >= 0 {
flags = os.O_CREATE | os.O_TRUNC | os.O_WRONLY
} else {
flags = os.O_WRONLY
}
f, err := os.OpenFile(path, flags, 0600)
if err != nil {
if f, err = os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600); err != nil {
if fs.IsAccessDenied(err) {
// If file is readonly, clear the readonly flag by resetting the
// permissions of the file and try again
// as the metadata will be set again in the second pass and the
// readonly flag will be applied again if needed.
if err = fs.ResetPermissions(path); err != nil {
return nil, err
}
if f, err = os.OpenFile(path, os.O_TRUNC|os.O_WRONLY, 0600); err != nil {
return nil, err
}
} else {
return nil, err
}
}
} else if f, err = os.OpenFile(path, os.O_WRONLY, 0600); err != nil {
return nil, err
}

19
internal/restorer/restorer.go
View File

@ -24,6 +24,7 @@ type Restorer struct {
progress *restoreui.Progress
Error func(location string, err error) error
Warn func(message string)
SelectFilter func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool)
}
@ -178,7 +179,7 @@ func (res *Restorer) restoreNodeTo(ctx context.Context, node *restic.Node, targe
func (res *Restorer) restoreNodeMetadataTo(node *restic.Node, target, location string) error {
debug.Log("restoreNodeMetadata %v %v %v", node.Name, target, location)
err := node.RestoreMetadata(target)
err := node.RestoreMetadata(target, res.Warn)
if err != nil {
debug.Log("node.RestoreMetadata(%s) error %v", target, err)
}
@ -204,11 +205,19 @@ func (res *Restorer) restoreHardlinkAt(node *restic.Node, target, path, location
func (res *Restorer) restoreEmptyFileAt(node *restic.Node, target, location string) error {
wr, err := os.OpenFile(target, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
if err != nil {
return err
if fs.IsAccessDenied(err) {
// If file is readonly, clear the readonly flag by resetting the
// permissions of the file and try again
// as the metadata will be set again in the second pass and the
// readonly flag will be applied again if needed.
if err = fs.ResetPermissions(target); err != nil {
return err
}
if wr, err = os.OpenFile(target, os.O_TRUNC|os.O_WRONLY, 0600); err != nil {
return err
}
}
err = wr.Close()
if err != nil {
if err = wr.Close(); err != nil {
return err
}

86
internal/restorer/restorer_test.go
View File

@ -3,6 +3,7 @@ package restorer
import (
"bytes"
"context"
"encoding/json"
"io"
"math"
"os"
@ -27,17 +28,27 @@ type Snapshot struct {
}
type File struct {
Data string
Links uint64
Inode uint64
Mode os.FileMode
ModTime time.Time
Data string
Links uint64
Inode uint64
Mode os.FileMode
ModTime time.Time
attributes *FileAttributes
}
type Dir struct {
Nodes map[string]Node
Mode os.FileMode
ModTime time.Time
Nodes map[string]Node
Mode os.FileMode
ModTime time.Time
attributes *FileAttributes
}
type FileAttributes struct {
ReadOnly bool
Hidden bool
System bool
Archive bool
Encrypted bool
}
func saveFile(t testing.TB, repo restic.BlobSaver, node File) restic.ID {
@ -52,7 +63,7 @@ func saveFile(t testing.TB, repo restic.BlobSaver, node File) restic.ID {
return id
}
func saveDir(t testing.TB, repo restic.BlobSaver, nodes map[string]Node, inode uint64) restic.ID {
func saveDir(t testing.TB, repo restic.BlobSaver, nodes map[string]Node, inode uint64, getGenericAttributes func(attr *FileAttributes, isDir bool) (genericAttributes map[restic.GenericAttributeType]json.RawMessage)) restic.ID {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@ -78,20 +89,21 @@ func saveDir(t testing.TB, repo restic.BlobSaver, nodes map[string]Node, inode u
mode = 0644
}
err := tree.Insert(&restic.Node{
Type: "file",
Mode: mode,
ModTime: node.ModTime,
Name: name,
UID: uint32(os.Getuid()),
GID: uint32(os.Getgid()),
Content: fc,
Size: uint64(len(n.(File).Data)),
Inode: fi,
Links: lc,
Type: "file",
Mode: mode,
ModTime: node.ModTime,
Name: name,
UID: uint32(os.Getuid()),
GID: uint32(os.Getgid()),
Content: fc,
Size: uint64(len(n.(File).Data)),
Inode: fi,
Links: lc,
GenericAttributes: getGenericAttributes(node.attributes, false),
})
rtest.OK(t, err)
case Dir:
id := saveDir(t, repo, node.Nodes, inode)
id := saveDir(t, repo, node.Nodes, inode, getGenericAttributes)
mode := node.Mode
if mode == 0 {
@ -99,13 +111,14 @@ func saveDir(t testing.TB, repo restic.BlobSaver, nodes map[string]Node, inode u
}
err := tree.Insert(&restic.Node{
Type: "dir",
Mode: mode,
ModTime: node.ModTime,
Name: name,
UID: uint32(os.Getuid()),
GID: uint32(os.Getgid()),
Subtree: &id,
Type: "dir",
Mode: mode,
ModTime: node.ModTime,
Name: name,
UID: uint32(os.Getuid()),
GID: uint32(os.Getgid()),
Subtree: &id,
GenericAttributes: getGenericAttributes(node.attributes, false),
})
rtest.OK(t, err)
default:
@ -121,13 +134,13 @@ func saveDir(t testing.TB, repo restic.BlobSaver, nodes map[string]Node, inode u
return id
}
func saveSnapshot(t testing.TB, repo restic.Repository, snapshot Snapshot) (*restic.Snapshot, restic.ID) {
func saveSnapshot(t testing.TB, repo restic.Repository, snapshot Snapshot, getGenericAttributes func(attr *FileAttributes, isDir bool) (genericAttributes map[restic.GenericAttributeType]json.RawMessage)) (*restic.Snapshot, restic.ID) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
wg, wgCtx := errgroup.WithContext(ctx)
repo.StartPackUploader(wgCtx, wg)
treeID := saveDir(t, repo, snapshot.Nodes, 1000)
treeID := saveDir(t, repo, snapshot.Nodes, 1000, getGenericAttributes)
err := repo.Flush(ctx)
if err != nil {
t.Fatal(err)
@ -147,6 +160,11 @@ func saveSnapshot(t testing.TB, repo restic.Repository, snapshot Snapshot) (*res
return sn, id
}
var noopGetGenericAttributes = func(attr *FileAttributes, isDir bool) (genericAttributes map[restic.GenericAttributeType]json.RawMessage) {
// No-op
return nil
}
func TestRestorer(t *testing.T) {
var tests = []struct {
Snapshot
@ -322,7 +340,7 @@ func TestRestorer(t *testing.T) {
for _, test := range tests {
t.Run("", func(t *testing.T) {
repo := repository.TestRepository(t)
sn, id := saveSnapshot(t, repo, test.Snapshot)
sn, id := saveSnapshot(t, repo, test.Snapshot, noopGetGenericAttributes)
t.Logf("snapshot saved as %v", id.Str())
res := NewRestorer(repo, sn, false, nil)
@ -439,7 +457,7 @@ func TestRestorerRelative(t *testing.T) {
t.Run("", func(t *testing.T) {
repo := repository.TestRepository(t)
sn, id := saveSnapshot(t, repo, test.Snapshot)
sn, id := saveSnapshot(t, repo, test.Snapshot, noopGetGenericAttributes)
t.Logf("snapshot saved as %v", id.Str())
res := NewRestorer(repo, sn, false, nil)
@ -669,7 +687,7 @@ func TestRestorerTraverseTree(t *testing.T) {
for _, test := range tests {
t.Run("", func(t *testing.T) {
repo := repository.TestRepository(t)
sn, _ := saveSnapshot(t, repo, test.Snapshot)
sn, _ := saveSnapshot(t, repo, test.Snapshot, noopGetGenericAttributes)
res := NewRestorer(repo, sn, false, nil)
@ -745,7 +763,7 @@ func TestRestorerConsistentTimestampsAndPermissions(t *testing.T) {
},
},
},
})
}, noopGetGenericAttributes)
res := NewRestorer(repo, sn, false, nil)
@ -800,7 +818,7 @@ func TestVerifyCancel(t *testing.T) {
}
repo := repository.TestRepository(t)
sn, _ := saveSnapshot(t, repo, snapshot)
sn, _ := saveSnapshot(t, repo, snapshot, noopGetGenericAttributes)
res := NewRestorer(repo, sn, false, nil)

4
internal/restorer/restorer_unix_test.go
View File

@ -29,7 +29,7 @@ func TestRestorerRestoreEmptyHardlinkedFileds(t *testing.T) {
},
},
},
})
}, noopGetGenericAttributes)
res := NewRestorer(repo, sn, false, nil)
@ -95,7 +95,7 @@ func TestRestorerProgressBar(t *testing.T) {
},
"file2": File{Links: 1, Inode: 2, Data: "example"},
},
})
}, noopGetGenericAttributes)
mock := &printerMock{}
progress := restoreui.NewProgress(mock, 0)

506
internal/restorer/restorer_windows_test.go
View File

@ -4,11 +4,20 @@
package restorer
import (
"context"
"encoding/json"
"math"
"os"
"path"
"syscall"
"testing"
"time"
"unsafe"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test"
rtest "github.com/restic/restic/internal/test"
"golang.org/x/sys/windows"
)
@ -33,3 +42,500 @@ func getBlockCount(t *testing.T, filename string) int64 {
return int64(math.Ceil(float64(result) / 512))
}
type DataStreamInfo struct {
name string
data string
}
type NodeInfo struct {
DataStreamInfo
parentDir string
attributes FileAttributes
Exists bool
IsDirectory bool
}
func TestFileAttributeCombination(t *testing.T) {
testFileAttributeCombination(t, false)
}
func TestEmptyFileAttributeCombination(t *testing.T) {
testFileAttributeCombination(t, true)
}
func testFileAttributeCombination(t *testing.T, isEmpty bool) {
t.Parallel()
//Generate combination of 5 attributes.
attributeCombinations := generateCombinations(5, []bool{})
fileName := "TestFile.txt"
// Iterate through each attribute combination
for _, attr1 := range attributeCombinations {
//Set up the required file information
fileInfo := NodeInfo{
DataStreamInfo: getDataStreamInfo(isEmpty, fileName),
parentDir: "dir",
attributes: getFileAttributes(attr1),
Exists: false,
}
//Get the current test name
testName := getCombinationTestName(fileInfo, fileName, fileInfo.attributes)
//Run test
t.Run(testName, func(t *testing.T) {
mainFilePath := runAttributeTests(t, fileInfo, fileInfo.attributes)
verifyFileRestores(isEmpty, mainFilePath, t, fileInfo)
})
}
}
func generateCombinations(n int, prefix []bool) [][]bool {
if n == 0 {
// Return a slice containing the current permutation
return [][]bool{append([]bool{}, prefix...)}
}
// Generate combinations with True
prefixTrue := append(prefix, true)
permsTrue := generateCombinations(n-1, prefixTrue)
// Generate combinations with False
prefixFalse := append(prefix, false)
permsFalse := generateCombinations(n-1, prefixFalse)
// Combine combinations with True and False
return append(permsTrue, permsFalse...)
}
func getDataStreamInfo(isEmpty bool, fileName string) DataStreamInfo {
var dataStreamInfo DataStreamInfo
if isEmpty {
dataStreamInfo = DataStreamInfo{
name: fileName,
}
} else {
dataStreamInfo = DataStreamInfo{
name: fileName,
data: "Main file data stream.",
}
}
return dataStreamInfo
}
func getFileAttributes(values []bool) FileAttributes {
return FileAttributes{
ReadOnly: values[0],
Hidden: values[1],
System: values[2],
Archive: values[3],
Encrypted: values[4],
}
}
func getCombinationTestName(fi NodeInfo, fileName string, overwriteAttr FileAttributes) string {
if fi.attributes.ReadOnly {
fileName += "-ReadOnly"
}
if fi.attributes.Hidden {
fileName += "-Hidden"
}
if fi.attributes.System {
fileName += "-System"
}
if fi.attributes.Archive {
fileName += "-Archive"
}
if fi.attributes.Encrypted {
fileName += "-Encrypted"
}
if fi.Exists {
fileName += "-Overwrite"
if overwriteAttr.ReadOnly {
fileName += "-R"
}
if overwriteAttr.Hidden {
fileName += "-H"
}
if overwriteAttr.System {
fileName += "-S"
}
if overwriteAttr.Archive {
fileName += "-A"
}
if overwriteAttr.Encrypted {
fileName += "-E"
}
}
return fileName
}
func runAttributeTests(t *testing.T, fileInfo NodeInfo, existingFileAttr FileAttributes) string {
testDir := t.TempDir()
res, _ := setupWithFileAttributes(t, fileInfo, testDir, existingFileAttr)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
err := res.RestoreTo(ctx, testDir)
rtest.OK(t, err)
mainFilePath := path.Join(testDir, fileInfo.parentDir, fileInfo.name)
//Verify restore
verifyFileAttributes(t, mainFilePath, fileInfo.attributes)
return mainFilePath
}
func setupWithFileAttributes(t *testing.T, nodeInfo NodeInfo, testDir string, existingFileAttr FileAttributes) (*Restorer, []int) {
t.Helper()
if nodeInfo.Exists {
if !nodeInfo.IsDirectory {
err := os.MkdirAll(path.Join(testDir, nodeInfo.parentDir), os.ModeDir)
rtest.OK(t, err)
filepath := path.Join(testDir, nodeInfo.parentDir, nodeInfo.name)
if existingFileAttr.Encrypted {
err := createEncryptedFileWriteData(filepath, nodeInfo)
rtest.OK(t, err)
} else {
// Write the data to the file
file, err := os.OpenFile(path.Clean(filepath), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
rtest.OK(t, err)
_, err = file.Write([]byte(nodeInfo.data))
rtest.OK(t, err)
err = file.Close()
rtest.OK(t, err)
}
} else {
err := os.MkdirAll(path.Join(testDir, nodeInfo.parentDir, nodeInfo.name), os.ModeDir)
rtest.OK(t, err)
}
pathPointer, err := syscall.UTF16PtrFromString(path.Join(testDir, nodeInfo.parentDir, nodeInfo.name))
rtest.OK(t, err)
syscall.SetFileAttributes(pathPointer, getAttributeValue(&existingFileAttr))
}
index := 0
order := []int{}
streams := []DataStreamInfo{}
if !nodeInfo.IsDirectory {
order = append(order, index)
index++
streams = append(streams, nodeInfo.DataStreamInfo)
}
return setup(t, getNodes(nodeInfo.parentDir, nodeInfo.name, order, streams, nodeInfo.IsDirectory, &nodeInfo.attributes)), order
}
func createEncryptedFileWriteData(filepath string, fileInfo NodeInfo) (err error) {
var ptr *uint16
if ptr, err = windows.UTF16PtrFromString(filepath); err != nil {
return err
}
var handle windows.Handle
//Create the file with encrypted flag
if handle, err = windows.CreateFile(ptr, uint32(windows.GENERIC_READ|windows.GENERIC_WRITE), uint32(windows.FILE_SHARE_READ), nil, uint32(windows.CREATE_ALWAYS), windows.FILE_ATTRIBUTE_ENCRYPTED, 0); err != nil {
return err
}
//Write data to file
if _, err = windows.Write(handle, []byte(fileInfo.data)); err != nil {
return err
}
//Close handle
return windows.CloseHandle(handle)
}
func setup(t *testing.T, nodesMap map[string]Node) *Restorer {
repo := repository.TestRepository(t)
getFileAttributes := func(attr *FileAttributes, isDir bool) (genericAttributes map[restic.GenericAttributeType]json.RawMessage) {
if attr == nil {
return
}
fileattr := getAttributeValue(attr)
if isDir {
//If the node is a directory add FILE_ATTRIBUTE_DIRECTORY to attributes
fileattr |= windows.FILE_ATTRIBUTE_DIRECTORY
}
attrs, err := restic.WindowsAttrsToGenericAttributes(restic.WindowsAttributes{FileAttributes: &fileattr})
test.OK(t, err)
return attrs
}
sn, _ := saveSnapshot(t, repo, Snapshot{
Nodes: nodesMap,
}, getFileAttributes)
res := NewRestorer(repo, sn, false, nil)
return res
}
func getAttributeValue(attr *FileAttributes) uint32 {
var fileattr uint32
if attr.ReadOnly {
fileattr |= windows.FILE_ATTRIBUTE_READONLY
}
if attr.Hidden {
fileattr |= windows.FILE_ATTRIBUTE_HIDDEN
}
if attr.Encrypted {
fileattr |= windows.FILE_ATTRIBUTE_ENCRYPTED
}
if attr.Archive {
fileattr |= windows.FILE_ATTRIBUTE_ARCHIVE
}
if attr.System {
fileattr |= windows.FILE_ATTRIBUTE_SYSTEM
}
return fileattr
}
func getNodes(dir string, mainNodeName string, order []int, streams []DataStreamInfo, isDirectory bool, attributes *FileAttributes) map[string]Node {
var mode os.FileMode
if isDirectory {
mode = os.FileMode(2147484159)
} else {
if attributes != nil && attributes.ReadOnly {
mode = os.FileMode(0o444)
} else {
mode = os.FileMode(0o666)
}
}
getFileNodes := func() map[string]Node {
nodes := map[string]Node{}
if isDirectory {
//Add a directory node at the same level as the other streams
nodes[mainNodeName] = Dir{
ModTime: time.Now(),
attributes: attributes,
Mode: mode,
}
}
if len(streams) > 0 {
for _, index := range order {
stream := streams[index]
var attr *FileAttributes = nil
if mainNodeName == stream.name {
attr = attributes
} else if attributes != nil && attributes.Encrypted {
//Set encrypted attribute
attr = &FileAttributes{Encrypted: true}
}
nodes[stream.name] = File{
ModTime: time.Now(),
Data: stream.data,
Mode: mode,
attributes: attr,
}
}
}
return nodes
}
return map[string]Node{
dir: Dir{
Mode: normalizeFileMode(0750 | mode),
ModTime: time.Now(),
Nodes: getFileNodes(),
},
}
}
func verifyFileAttributes(t *testing.T, mainFilePath string, attr FileAttributes) {
ptr, err := windows.UTF16PtrFromString(mainFilePath)
rtest.OK(t, err)
//Get file attributes using syscall
fileAttributes, err := syscall.GetFileAttributes(ptr)
rtest.OK(t, err)
//Test positive and negative scenarios
if attr.ReadOnly {
rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_READONLY != 0, "Expected read only attibute.")
} else {
rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_READONLY == 0, "Unexpected read only attibute.")
}
if attr.Hidden {
rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_HIDDEN != 0, "Expected hidden attibute.")
} else {
rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_HIDDEN == 0, "Unexpected hidden attibute.")
}
if attr.System {
rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_SYSTEM != 0, "Expected system attibute.")
} else {
rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_SYSTEM == 0, "Unexpected system attibute.")
}
if attr.Archive {
rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_ARCHIVE != 0, "Expected archive attibute.")
} else {
rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_ARCHIVE == 0, "Unexpected archive attibute.")
}
if attr.Encrypted {
rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_ENCRYPTED != 0, "Expected encrypted attibute.")
} else {
rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_ENCRYPTED == 0, "Unexpected encrypted attibute.")
}
}
func verifyFileRestores(isEmpty bool, mainFilePath string, t *testing.T, fileInfo NodeInfo) {
if isEmpty {
_, err1 := os.Stat(mainFilePath)
rtest.Assert(t, !errors.Is(err1, os.ErrNotExist), "The file "+fileInfo.name+" does not exist")
} else {
verifyMainFileRestore(t, mainFilePath, fileInfo)
}
}
func verifyMainFileRestore(t *testing.T, mainFilePath string, fileInfo NodeInfo) {
fi, err1 := os.Stat(mainFilePath)
rtest.Assert(t, !errors.Is(err1, os.ErrNotExist), "The file "+fileInfo.name+" does not exist")
size := fi.Size()
rtest.Assert(t, size > 0, "The file "+fileInfo.name+" exists but is empty")
content, err := os.ReadFile(mainFilePath)
rtest.OK(t, err)
rtest.Assert(t, string(content) == fileInfo.data, "The file "+fileInfo.name+" exists but the content is not overwritten")
}
func TestDirAttributeCombination(t *testing.T) {
t.Parallel()
attributeCombinations := generateCombinations(4, []bool{})
dirName := "TestDir"
// Iterate through each attribute combination
for _, attr1 := range attributeCombinations {
//Set up the required directory information
dirInfo := NodeInfo{
DataStreamInfo: DataStreamInfo{
name: dirName,
},
parentDir: "dir",
attributes: getDirFileAttributes(attr1),
Exists: false,
IsDirectory: true,
}
//Get the current test name
testName := getCombinationTestName(dirInfo, dirName, dirInfo.attributes)
//Run test
t.Run(testName, func(t *testing.T) {
mainDirPath := runAttributeTests(t, dirInfo, dirInfo.attributes)
//Check directory exists
_, err1 := os.Stat(mainDirPath)
rtest.Assert(t, !errors.Is(err1, os.ErrNotExist), "The directory "+dirInfo.name+" does not exist")
})
}
}
func getDirFileAttributes(values []bool) FileAttributes {
return FileAttributes{
// readonly not valid for directories
Hidden: values[0],
System: values[1],
Archive: values[2],
Encrypted: values[3],
}
}
func TestFileAttributeCombinationsOverwrite(t *testing.T) {
testFileAttributeCombinationsOverwrite(t, false)
}
func TestEmptyFileAttributeCombinationsOverwrite(t *testing.T) {
testFileAttributeCombinationsOverwrite(t, true)
}
func testFileAttributeCombinationsOverwrite(t *testing.T, isEmpty bool) {
t.Parallel()
//Get attribute combinations
attributeCombinations := generateCombinations(5, []bool{})
//Get overwrite file attribute combinations
overwriteCombinations := generateCombinations(5, []bool{})
fileName := "TestOverwriteFile"
//Iterate through each attribute combination
for _, attr1 := range attributeCombinations {
fileInfo := NodeInfo{
DataStreamInfo: getDataStreamInfo(isEmpty, fileName),
parentDir: "dir",
attributes: getFileAttributes(attr1),
Exists: true,
}
overwriteFileAttributes := []FileAttributes{}
for _, overwrite := range overwriteCombinations {
overwriteFileAttributes = append(overwriteFileAttributes, getFileAttributes(overwrite))
}
//Iterate through each overwrite attribute combination
for _, overwriteFileAttr := range overwriteFileAttributes {
//Get the test name
testName := getCombinationTestName(fileInfo, fileName, overwriteFileAttr)
//Run test
t.Run(testName, func(t *testing.T) {
mainFilePath := runAttributeTests(t, fileInfo, overwriteFileAttr)
verifyFileRestores(isEmpty, mainFilePath, t, fileInfo)
})
}
}
}
func TestDirAttributeCombinationsOverwrite(t *testing.T) {
t.Parallel()
//Get attribute combinations
attributeCombinations := generateCombinations(4, []bool{})
//Get overwrite dir attribute combinations
overwriteCombinations := generateCombinations(4, []bool{})
dirName := "TestOverwriteDir"
//Iterate through each attribute combination
for _, attr1 := range attributeCombinations {
dirInfo := NodeInfo{
DataStreamInfo: DataStreamInfo{
name: dirName,
},
parentDir: "dir",
attributes: getDirFileAttributes(attr1),
Exists: true,
IsDirectory: true,
}
overwriteDirFileAttributes := []FileAttributes{}
for _, overwrite := range overwriteCombinations {
overwriteDirFileAttributes = append(overwriteDirFileAttributes, getDirFileAttributes(overwrite))
}
//Iterate through each overwrite attribute combinations
for _, overwriteDirAttr := range overwriteDirFileAttributes {
//Get the test name
testName := getCombinationTestName(dirInfo, dirName, overwriteDirAttr)
//Run test
t.Run(testName, func(t *testing.T) {
mainDirPath := runAttributeTests(t, dirInfo, dirInfo.attributes)
//Check directory exists
_, err1 := os.Stat(mainDirPath)
rtest.Assert(t, !errors.Is(err1, os.ErrNotExist), "The directory "+dirInfo.name+" does not exist")
})
}
}
}

17
internal/test/helpers.go
View File

@ -3,6 +3,7 @@ package test
import (
"compress/bzip2"
"compress/gzip"
"fmt"
"io"
"os"
"os/exec"
@ -47,10 +48,22 @@ func OKs(tb testing.TB, errs []error) {
}
// Equals fails the test if exp is not equal to act.
func Equals(tb testing.TB, exp, act interface{}) {
// msg is optional message to be printed, first param being format string and rest being arguments.
func Equals(tb testing.TB, exp, act interface{}, msgs ...string) {
tb.Helper()
if !reflect.DeepEqual(exp, act) {
tb.Fatalf("\033[31m\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", exp, act)
var msgString string
length := len(msgs)
if length == 1 {
msgString = msgs[0]
} else if length > 1 {
args := make([]interface{}, length-1)
for i, msg := range msgs[1:] {
args[i] = msg
}
msgString = fmt.Sprintf(msgs[0], args...)
}
tb.Fatalf("\033[31m\n\n\t"+msgString+"\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", exp, act)
}
}