From 56440797077c3b8e5687a2f7bd07d7ba268e9a77 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Mon, 26 Aug 2024 22:35:22 +0200 Subject: [PATCH 1/8] restic: prepare extraction of fs code from Node --- internal/fs/ea_windows_test.go | 6 +- internal/restic/node.go | 84 ++++++++++++-------------- internal/restic/node_aix.go | 20 +++--- internal/restic/node_darwin.go | 2 +- internal/restic/node_freebsd.go | 2 +- internal/restic/node_linux.go | 2 +- internal/restic/node_netbsd.go | 20 +++--- internal/restic/node_openbsd.go | 18 +++--- internal/restic/node_solaris.go | 2 +- internal/restic/node_test.go | 8 +-- internal/restic/node_windows.go | 16 ++--- internal/restic/node_windows_test.go | 12 ++-- internal/restic/node_xattr.go | 14 ++--- internal/restic/node_xattr_all_test.go | 10 +-- internal/restorer/restorer.go | 4 +- 15 files changed, 108 insertions(+), 112 deletions(-) diff --git a/internal/fs/ea_windows_test.go b/internal/fs/ea_windows_test.go index 74afd7aa5..e474a3735 100644 --- a/internal/fs/ea_windows_test.go +++ b/internal/fs/ea_windows_test.go @@ -142,7 +142,7 @@ func TestSetGetFileEA(t *testing.T) { testFilePath, testFile := setupTestFile(t) testEAs := generateTestEAs(t, 3, testFilePath) fileHandle := openFile(t, testFilePath, windows.FILE_ATTRIBUTE_NORMAL) - defer closeFileHandle(t, testFilePath, testFile, fileHandle) + defer testCloseFileHandle(t, testFilePath, testFile, fileHandle) testSetGetEA(t, testFilePath, fileHandle, testEAs) } @@ -154,7 +154,7 @@ func TestSetGetFolderEA(t *testing.T) { testEAs := generateTestEAs(t, 3, testFolderPath) fileHandle := openFile(t, testFolderPath, windows.FILE_ATTRIBUTE_NORMAL|windows.FILE_FLAG_BACKUP_SEMANTICS) - defer closeFileHandle(t, testFolderPath, nil, fileHandle) + defer testCloseFileHandle(t, testFolderPath, nil, fileHandle) testSetGetEA(t, testFolderPath, fileHandle, testEAs) } @@ -212,7 +212,7 @@ func openFile(t *testing.T, path string, attributes uint32) windows.Handle { return fileHandle } -func closeFileHandle(t *testing.T, testfilePath string, testFile *os.File, handle windows.Handle) { +func testCloseFileHandle(t *testing.T, testfilePath string, testFile *os.File, handle windows.Handle) { if testFile != nil { err := testFile.Close() if err != nil { diff --git a/internal/restic/node.go b/internal/restic/node.go index 6afdff64a..c9c68cbee 100644 --- a/internal/restic/node.go +++ b/internal/restic/node.go @@ -150,7 +150,7 @@ func NodeFromFileInfo(path string, fi os.FileInfo, ignoreXattrListError bool) (* node.Size = uint64(fi.Size()) } - err := node.fillExtra(path, fi, ignoreXattrListError) + err := nodeFillExtra(node, path, fi, ignoreXattrListError) return node, err } @@ -187,33 +187,33 @@ func (node Node) GetExtendedAttribute(a string) []byte { return nil } -// CreateAt creates the node at the given path but does NOT restore node meta data. -func (node *Node) CreateAt(ctx context.Context, path string, repo BlobLoader) error { +// NodeCreateAt creates the node at the given path but does NOT restore node meta data. +func NodeCreateAt(ctx context.Context, node *Node, path string, repo BlobLoader) error { debug.Log("create node %v at %v", node.Name, path) switch node.Type { case "dir": - if err := node.createDirAt(path); err != nil { + if err := nodeCreateDirAt(node, path); err != nil { return err } case "file": - if err := node.createFileAt(ctx, path, repo); err != nil { + if err := nodeCreateFileAt(ctx, node, path, repo); err != nil { return err } case "symlink": - if err := node.createSymlinkAt(path); err != nil { + if err := nodeCreateSymlinkAt(node, path); err != nil { return err } case "dev": - if err := node.createDevAt(path); err != nil { + if err := nodeCreateDevAt(node, path); err != nil { return err } case "chardev": - if err := node.createCharDevAt(path); err != nil { + if err := nodeCreateCharDevAt(node, path); err != nil { return err } case "fifo": - if err := node.createFifoAt(path); err != nil { + if err := nodeCreateFifoAt(path); err != nil { return err } case "socket": @@ -225,9 +225,9 @@ func (node *Node) CreateAt(ctx context.Context, path string, repo BlobLoader) er return nil } -// RestoreMetadata restores node metadata -func (node Node) RestoreMetadata(path string, warn func(msg string)) error { - err := node.restoreMetadata(path, warn) +// NodeRestoreMetadata restores node metadata +func NodeRestoreMetadata(node *Node, path string, warn func(msg string)) error { + err := nodeRestoreMetadata(node, path, warn) if err != nil { // It is common to have permission errors for folders like /home // unless you're running as root, so ignore those. @@ -242,28 +242,28 @@ func (node Node) RestoreMetadata(path string, warn func(msg string)) error { return err } -func (node Node) restoreMetadata(path string, warn func(msg string)) error { +func nodeRestoreMetadata(node *Node, path string, warn func(msg string)) error { var firsterr error if err := lchown(path, int(node.UID), int(node.GID)); err != nil { firsterr = errors.WithStack(err) } - if err := node.restoreExtendedAttributes(path); err != nil { + if err := nodeRestoreExtendedAttributes(node, path); err != nil { debug.Log("error restoring extended attributes for %v: %v", path, err) if firsterr == nil { firsterr = err } } - if err := node.restoreGenericAttributes(path, warn); err != nil { + if err := nodeRestoreGenericAttributes(node, path, warn); err != nil { debug.Log("error restoring generic attributes for %v: %v", path, err) if firsterr == nil { firsterr = err } } - if err := node.RestoreTimestamps(path); err != nil { + if err := NodeRestoreTimestamps(node, path); err != nil { debug.Log("error restoring timestamps for %v: %v", path, err) if firsterr == nil { firsterr = err @@ -284,14 +284,14 @@ func (node Node) restoreMetadata(path string, warn func(msg string)) error { return firsterr } -func (node Node) RestoreTimestamps(path string) error { +func NodeRestoreTimestamps(node *Node, path string) error { var utimes = [...]syscall.Timespec{ syscall.NsecToTimespec(node.AccessTime.UnixNano()), syscall.NsecToTimespec(node.ModTime.UnixNano()), } if node.Type == "symlink" { - return node.restoreSymlinkTimestamps(path, utimes) + return nodeRestoreSymlinkTimestamps(path, utimes) } if err := syscall.UtimesNano(path, utimes[:]); err != nil { @@ -301,7 +301,7 @@ func (node Node) RestoreTimestamps(path string) error { return nil } -func (node Node) createDirAt(path string) error { +func nodeCreateDirAt(node *Node, path string) error { err := fs.Mkdir(path, node.Mode) if err != nil && !os.IsExist(err) { return errors.WithStack(err) @@ -310,13 +310,13 @@ func (node Node) createDirAt(path string) error { return nil } -func (node Node) createFileAt(ctx context.Context, path string, repo BlobLoader) error { +func nodeCreateFileAt(ctx context.Context, node *Node, path string, repo BlobLoader) error { f, err := fs.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) if err != nil { return errors.WithStack(err) } - err = node.writeNodeContent(ctx, repo, f) + err = nodeWriteNodeContent(ctx, node, repo, f) closeErr := f.Close() if err != nil { @@ -330,7 +330,7 @@ func (node Node) createFileAt(ctx context.Context, path string, repo BlobLoader) return nil } -func (node Node) writeNodeContent(ctx context.Context, repo BlobLoader, f *os.File) error { +func nodeWriteNodeContent(ctx context.Context, node *Node, repo BlobLoader, f *os.File) error { var buf []byte for _, id := range node.Content { buf, err := repo.LoadBlob(ctx, DataBlob, id, buf) @@ -347,7 +347,7 @@ func (node Node) writeNodeContent(ctx context.Context, repo BlobLoader, f *os.Fi return nil } -func (node Node) createSymlinkAt(path string) error { +func nodeCreateSymlinkAt(node *Node, path string) error { if err := fs.Symlink(node.LinkTarget, path); err != nil { return errors.WithStack(err) } @@ -355,15 +355,15 @@ func (node Node) createSymlinkAt(path string) error { return nil } -func (node *Node) createDevAt(path string) error { +func nodeCreateDevAt(node *Node, path string) error { return mknod(path, syscall.S_IFBLK|0600, node.Device) } -func (node *Node) createCharDevAt(path string) error { +func nodeCreateCharDevAt(node *Node, path string) error { return mknod(path, syscall.S_IFCHR|0600, node.Device) } -func (node *Node) createFifoAt(path string) error { +func nodeCreateFifoAt(path string) error { return mkfifo(path, 0600) } @@ -601,7 +601,7 @@ func deepEqual(map1, map2 map[GenericAttributeType]json.RawMessage) bool { return true } -func (node *Node) fillUser(stat *statT) { +func nodeFillUser(node *Node, stat *statT) { uid, gid := stat.uid(), stat.gid() node.UID, node.GID = uid, gid node.User = lookupUsername(uid) @@ -662,7 +662,7 @@ func lookupGroup(gid uint32) string { return group } -func (node *Node) fillExtra(path string, fi os.FileInfo, ignoreXattrListError bool) error { +func nodeFillExtra(node *Node, path string, fi os.FileInfo, ignoreXattrListError bool) error { stat, ok := toStatT(fi.Sys()) if !ok { // fill minimal info with current values for uid, gid @@ -675,9 +675,9 @@ func (node *Node) fillExtra(path string, fi os.FileInfo, ignoreXattrListError bo node.Inode = uint64(stat.ino()) node.DeviceID = uint64(stat.dev()) - node.fillTimes(stat) + nodeFillTimes(node, stat) - node.fillUser(stat) + nodeFillUser(node, stat) switch node.Type { case "file": @@ -703,10 +703,10 @@ func (node *Node) fillExtra(path string, fi os.FileInfo, ignoreXattrListError bo return errors.Errorf("unsupported file type %q", node.Type) } - allowExtended, err := node.fillGenericAttributes(path, fi, stat) + allowExtended, err := nodeFillGenericAttributes(node, path, fi, stat) if allowExtended { // Skip processing ExtendedAttributes if allowExtended is false. - err = errors.CombineErrors(err, node.fillExtendedAttributes(path, ignoreXattrListError)) + err = errors.CombineErrors(err, nodeFillExtendedAttributes(node, path, ignoreXattrListError)) } return err } @@ -715,7 +715,7 @@ func mkfifo(path string, mode uint32) (err error) { return mknod(path, mode|syscall.S_IFIFO, 0) } -func (node *Node) fillTimes(stat *statT) { +func nodeFillTimes(node *Node, stat *statT) { ctim := stat.ctim() atim := stat.atim() node.ChangeTime = time.Unix(ctim.Unix()) @@ -746,11 +746,11 @@ func handleUnknownGenericAttributeFound(genericAttributeType GenericAttributeTyp } } -// handleAllUnknownGenericAttributesFound performs validations for all generic attributes in the node. +// HandleAllUnknownGenericAttributesFound performs validations for all generic attributes of a node. // This is not used on windows currently because windows has handling for generic attributes. // nolint:unused -func (node Node) handleAllUnknownGenericAttributesFound(warn func(msg string)) error { - for name := range node.GenericAttributes { +func HandleAllUnknownGenericAttributesFound(attributes map[GenericAttributeType]json.RawMessage, warn func(msg string)) error { + for name := range attributes { handleUnknownGenericAttributeFound(name, warn) } return nil @@ -770,9 +770,8 @@ func checkGenericAttributeNameNotHandledAndPut(value GenericAttributeType) bool // The functions below are common helper functions which can be used for generic attributes support // across different OS. -// genericAttributesToOSAttrs gets the os specific attribute from the generic attribute using reflection -// nolint:unused -func genericAttributesToOSAttrs(attrs map[GenericAttributeType]json.RawMessage, attributeType reflect.Type, attributeValuePtr *reflect.Value, keyPrefix string) (unknownAttribs []GenericAttributeType, err error) { +// GenericAttributesToOSAttrs gets the os specific attribute from the generic attribute using reflection +func GenericAttributesToOSAttrs(attrs map[GenericAttributeType]json.RawMessage, attributeType reflect.Type, attributeValuePtr *reflect.Value, keyPrefix string) (unknownAttribs []GenericAttributeType, err error) { attributeValue := *attributeValuePtr for key, rawMsg := range attrs { @@ -796,20 +795,17 @@ func genericAttributesToOSAttrs(attrs map[GenericAttributeType]json.RawMessage, } // getFQKey gets the fully qualified key for the field -// nolint:unused func getFQKey(field reflect.StructField, keyPrefix string) GenericAttributeType { return GenericAttributeType(fmt.Sprintf("%s.%s", keyPrefix, field.Tag.Get("generic"))) } // getFQKeyByIndex gets the fully qualified key for the field index -// nolint:unused func getFQKeyByIndex(attributeType reflect.Type, index int, keyPrefix string) GenericAttributeType { return getFQKey(attributeType.Field(index), keyPrefix) } -// osAttrsToGenericAttributes gets the generic attribute from the os specific attribute using reflection -// nolint:unused -func osAttrsToGenericAttributes(attributeType reflect.Type, attributeValuePtr *reflect.Value, keyPrefix string) (attrs map[GenericAttributeType]json.RawMessage, err error) { +// OSAttrsToGenericAttributes gets the generic attribute from the os specific attribute using reflection +func OSAttrsToGenericAttributes(attributeType reflect.Type, attributeValuePtr *reflect.Value, keyPrefix string) (attrs map[GenericAttributeType]json.RawMessage, err error) { attributeValue := *attributeValuePtr attrs = make(map[GenericAttributeType]json.RawMessage) diff --git a/internal/restic/node_aix.go b/internal/restic/node_aix.go index 32f63af15..4cd279973 100644 --- a/internal/restic/node_aix.go +++ b/internal/restic/node_aix.go @@ -8,7 +8,7 @@ import ( "syscall" ) -func (node Node) restoreSymlinkTimestamps(_ string, _ [2]syscall.Timespec) error { +func nodeRestoreSymlinkTimestamps(_ string, _ [2]syscall.Timespec) error { return nil } @@ -23,13 +23,13 @@ func (s statT) atim() syscall.Timespec { return toTimespec(s.Atim) } func (s statT) mtim() syscall.Timespec { return toTimespec(s.Mtim) } func (s statT) ctim() syscall.Timespec { return toTimespec(s.Ctim) } -// restoreExtendedAttributes is a no-op on AIX. -func (node Node) restoreExtendedAttributes(_ string) error { +// nodeRestoreExtendedAttributes is a no-op on AIX. +func nodeRestoreExtendedAttributes(_ *Node, _ string) error { return nil } -// fillExtendedAttributes is a no-op on AIX. -func (node *Node) fillExtendedAttributes(_ string, _ bool) error { +// nodeFillExtendedAttributes is a no-op on AIX. +func nodeFillExtendedAttributes(_ *Node, _ string, _ bool) error { return nil } @@ -38,12 +38,12 @@ func IsListxattrPermissionError(_ error) bool { return false } -// restoreGenericAttributes is no-op on AIX. -func (node *Node) restoreGenericAttributes(_ string, warn func(msg string)) error { - return node.handleAllUnknownGenericAttributesFound(warn) +// nodeRestoreGenericAttributes is no-op on AIX. +func nodeRestoreGenericAttributes(node *Node, _ string, warn func(msg string)) error { + return HandleAllUnknownGenericAttributesFound(node.GenericAttributes, warn) } -// fillGenericAttributes is a no-op on AIX. -func (node *Node) fillGenericAttributes(_ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { +// nodeFillGenericAttributes is a no-op on AIX. +func nodeFillGenericAttributes(_ *Node, _ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { return true, nil } diff --git a/internal/restic/node_darwin.go b/internal/restic/node_darwin.go index 803aa68e5..099007e07 100644 --- a/internal/restic/node_darwin.go +++ b/internal/restic/node_darwin.go @@ -2,7 +2,7 @@ package restic import "syscall" -func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { +func nodeRestoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { return nil } diff --git a/internal/restic/node_freebsd.go b/internal/restic/node_freebsd.go index 34d5b272c..6d2dd1d98 100644 --- a/internal/restic/node_freebsd.go +++ b/internal/restic/node_freebsd.go @@ -5,7 +5,7 @@ package restic import "syscall" -func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { +func nodeRestoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { return nil } diff --git a/internal/restic/node_linux.go b/internal/restic/node_linux.go index 85a363830..6311a224b 100644 --- a/internal/restic/node_linux.go +++ b/internal/restic/node_linux.go @@ -10,7 +10,7 @@ import ( "github.com/restic/restic/internal/fs" ) -func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { +func nodeRestoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { dir, err := fs.Open(filepath.Dir(path)) if err != nil { return errors.WithStack(err) diff --git a/internal/restic/node_netbsd.go b/internal/restic/node_netbsd.go index 0fe46a3f2..a53412afb 100644 --- a/internal/restic/node_netbsd.go +++ b/internal/restic/node_netbsd.go @@ -5,7 +5,7 @@ import ( "syscall" ) -func (node Node) restoreSymlinkTimestamps(_ string, _ [2]syscall.Timespec) error { +func nodeRestoreSymlinkTimestamps(_ string, _ [2]syscall.Timespec) error { return nil } @@ -13,13 +13,13 @@ func (s statT) atim() syscall.Timespec { return s.Atimespec } func (s statT) mtim() syscall.Timespec { return s.Mtimespec } func (s statT) ctim() syscall.Timespec { return s.Ctimespec } -// restoreExtendedAttributes is a no-op on netbsd. -func (node Node) restoreExtendedAttributes(_ string) error { +// nodeRestoreExtendedAttributes is a no-op on netbsd. +func nodeRestoreExtendedAttributes(_ *Node, _ string) error { return nil } -// fillExtendedAttributes is a no-op on netbsd. -func (node *Node) fillExtendedAttributes(_ string, _ bool) error { +// nodeFillExtendedAttributes is a no-op on netbsd. +func nodeFillExtendedAttributes(_ *Node, _ string, _ bool) error { return nil } @@ -28,12 +28,12 @@ func IsListxattrPermissionError(_ error) bool { return false } -// restoreGenericAttributes is no-op on netbsd. -func (node *Node) restoreGenericAttributes(_ string, warn func(msg string)) error { - return node.handleAllUnknownGenericAttributesFound(warn) +// nodeRestoreGenericAttributes is no-op on netbsd. +func nodeRestoreGenericAttributes(node *Node, _ string, warn func(msg string)) error { + return HandleAllUnknownGenericAttributesFound(node.GenericAttributes, warn) } -// fillGenericAttributes is a no-op on netbsd. -func (node *Node) fillGenericAttributes(_ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { +// nodeFillGenericAttributes is a no-op on netbsd. +func nodeFillGenericAttributes(_ *Node, _ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { return true, nil } diff --git a/internal/restic/node_openbsd.go b/internal/restic/node_openbsd.go index 71841f59f..bbba89f2c 100644 --- a/internal/restic/node_openbsd.go +++ b/internal/restic/node_openbsd.go @@ -5,7 +5,7 @@ import ( "syscall" ) -func (node Node) restoreSymlinkTimestamps(_ string, _ [2]syscall.Timespec) error { +func nodeRestoreSymlinkTimestamps(_ string, _ [2]syscall.Timespec) error { return nil } @@ -13,13 +13,13 @@ func (s statT) atim() syscall.Timespec { return s.Atim } func (s statT) mtim() syscall.Timespec { return s.Mtim } func (s statT) ctim() syscall.Timespec { return s.Ctim } -// restoreExtendedAttributes is a no-op on openbsd. -func (node Node) restoreExtendedAttributes(_ string) error { +// nodeRestoreExtendedAttributes is a no-op on openbsd. +func nodeRestoreExtendedAttributes(_ *Node, _ string) error { return nil } -// fillExtendedAttributes is a no-op on openbsd. -func (node *Node) fillExtendedAttributes(_ string, _ bool) error { +// nodeFillExtendedAttributes is a no-op on openbsd. +func nodeFillExtendedAttributes(_ *Node, _ string, _ bool) error { return nil } @@ -28,12 +28,12 @@ func IsListxattrPermissionError(_ error) bool { return false } -// restoreGenericAttributes is no-op on openbsd. -func (node *Node) restoreGenericAttributes(_ string, warn func(msg string)) error { - return node.handleAllUnknownGenericAttributesFound(warn) +// nodeRestoreGenericAttributes is no-op on openbsd. +func nodeRestoreGenericAttributes(node *Node, _ string, warn func(msg string)) error { + return HandleAllUnknownGenericAttributesFound(node.GenericAttributes, warn) } // fillGenericAttributes is a no-op on openbsd. -func (node *Node) fillGenericAttributes(_ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { +func nodeFillGenericAttributes(_ *Node, _ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { return true, nil } diff --git a/internal/restic/node_solaris.go b/internal/restic/node_solaris.go index c9d03f9c2..114d11766 100644 --- a/internal/restic/node_solaris.go +++ b/internal/restic/node_solaris.go @@ -2,7 +2,7 @@ package restic import "syscall" -func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { +func nodeRestoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { return nil } diff --git a/internal/restic/node_test.go b/internal/restic/node_test.go index ab7f66e5b..7258b5429 100644 --- a/internal/restic/node_test.go +++ b/internal/restic/node_test.go @@ -245,8 +245,8 @@ func TestNodeRestoreAt(t *testing.T) { } else { nodePath = filepath.Join(tempdir, test.Name) } - rtest.OK(t, test.CreateAt(context.TODO(), nodePath, nil)) - rtest.OK(t, test.RestoreMetadata(nodePath, func(msg string) { rtest.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", nodePath, msg)) })) + rtest.OK(t, NodeCreateAt(context.TODO(), &test, nodePath, nil)) + rtest.OK(t, NodeRestoreMetadata(&test, nodePath, func(msg string) { rtest.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", nodePath, msg)) })) fi, err := os.Lstat(nodePath) rtest.OK(t, err) @@ -402,10 +402,10 @@ func TestSymlinkSerializationFormat(t *testing.T) { func TestNodeRestoreMetadataError(t *testing.T) { tempdir := t.TempDir() - node := nodeTests[0] + node := &nodeTests[0] nodePath := filepath.Join(tempdir, node.Name) // This will fail because the target file does not exist - err := node.RestoreMetadata(nodePath, func(msg string) { rtest.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", nodePath, msg)) }) + err := NodeRestoreMetadata(node, nodePath, func(msg string) { rtest.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", nodePath, msg)) }) test.Assert(t, errors.Is(err, os.ErrNotExist), "failed for an unexpected reason") } diff --git a/internal/restic/node_windows.go b/internal/restic/node_windows.go index bce01ccad..3f836ae61 100644 --- a/internal/restic/node_windows.go +++ b/internal/restic/node_windows.go @@ -56,7 +56,7 @@ func lchown(_ string, _ int, _ int) (err error) { } // restoreSymlinkTimestamps restores timestamps for symlinks -func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { +func nodeRestoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { // tweaked version of UtimesNano from go/src/syscall/syscall_windows.go pathp, e := syscall.UTF16PtrFromString(path) if e != nil { @@ -82,7 +82,7 @@ func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespe } // restore extended attributes for windows -func (node Node) restoreExtendedAttributes(path string) (err error) { +func nodeRestoreExtendedAttributes(node *Node, path string) (err error) { count := len(node.ExtendedAttributes) if count > 0 { eas := make([]fs.ExtendedAttribute, count) @@ -97,7 +97,7 @@ func (node Node) restoreExtendedAttributes(path string) (err error) { } // fill extended attributes in the node. This also includes the Generic attributes for windows. -func (node *Node) fillExtendedAttributes(path string, _ bool) (err error) { +func nodeFillExtendedAttributes(node *Node, path string, _ bool) (err error) { var fileHandle windows.Handle if fileHandle, err = fs.OpenHandleForEA(node.Type, path, false); fileHandle == 0 { return nil @@ -210,7 +210,7 @@ func (s statT) ctim() syscall.Timespec { } // restoreGenericAttributes restores generic attributes for Windows -func (node Node) restoreGenericAttributes(path string, warn func(msg string)) (err error) { +func nodeRestoreGenericAttributes(node *Node, path string, warn func(msg string)) (err error) { if len(node.GenericAttributes) == 0 { return nil } @@ -242,7 +242,7 @@ func (node Node) restoreGenericAttributes(path string, warn func(msg string)) (e // genericAttributesToWindowsAttrs converts the generic attributes map to a WindowsAttributes and also returns a string of unknown attributes that it could not convert. func genericAttributesToWindowsAttrs(attrs map[GenericAttributeType]json.RawMessage) (windowsAttributes WindowsAttributes, unknownAttribs []GenericAttributeType, err error) { waValue := reflect.ValueOf(&windowsAttributes).Elem() - unknownAttribs, err = genericAttributesToOSAttrs(attrs, reflect.TypeOf(windowsAttributes), &waValue, "windows") + unknownAttribs, err = GenericAttributesToOSAttrs(attrs, reflect.TypeOf(windowsAttributes), &waValue, "windows") return windowsAttributes, unknownAttribs, err } @@ -361,11 +361,11 @@ func decryptFile(pathPointer *uint16) error { return nil } -// fillGenericAttributes fills in the generic attributes for windows like File Attributes, +// nodeFillGenericAttributes fills in the generic attributes for windows like File Attributes, // Created time and Security Descriptors. // It also checks if the volume supports extended attributes and stores the result in a map // so that it does not have to be checked again for subsequent calls for paths in the same volume. -func (node *Node) fillGenericAttributes(path string, fi os.FileInfo, stat *statT) (allowExtended bool, err error) { +func nodeFillGenericAttributes(node *Node, path string, fi os.FileInfo, stat *statT) (allowExtended bool, err error) { if strings.Contains(filepath.Base(path), ":") { // Do not process for Alternate Data Streams in Windows // Also do not allow processing of extended attributes for ADS. @@ -499,7 +499,7 @@ func prepareVolumeName(path string) (volumeName string, err error) { func WindowsAttrsToGenericAttributes(windowsAttributes WindowsAttributes) (attrs map[GenericAttributeType]json.RawMessage, err error) { // Get the value of the WindowsAttributes windowsAttributesValue := reflect.ValueOf(windowsAttributes) - return osAttrsToGenericAttributes(reflect.TypeOf(windowsAttributes), &windowsAttributesValue, runtime.GOOS) + return OSAttrsToGenericAttributes(reflect.TypeOf(windowsAttributes), &windowsAttributesValue, runtime.GOOS) } // getCreationTime gets the value for the WindowsAttribute CreationTime in a windows specific time format. diff --git a/internal/restic/node_windows_test.go b/internal/restic/node_windows_test.go index 6ba25559b..e78c8cb96 100644 --- a/internal/restic/node_windows_test.go +++ b/internal/restic/node_windows_test.go @@ -42,7 +42,7 @@ func testRestoreSecurityDescriptor(t *testing.T, sd string, tempDir, fileType, f expectedNode := getNode(fileName, fileType, genericAttributes) // Restore the file/dir and restore the meta data including the security descriptors. - testPath, node := restoreAndGetNode(t, tempDir, expectedNode, false) + testPath, node := restoreAndGetNode(t, tempDir, &expectedNode, false) // Get the security descriptor from the node constructed from the file info of the restored path. sdByteFromRestoredNode := getWindowsAttr(t, testPath, node).SecurityDescriptor @@ -186,7 +186,7 @@ func runGenericAttributesTest(t *testing.T, tempDir string, genericAttributeName func runGenericAttributesTestForNodes(t *testing.T, expectedNodes []Node, tempDir string, genericAttr GenericAttributeType, genericAttributeExpected WindowsAttributes, warningExpected bool) { for _, testNode := range expectedNodes { - testPath, node := restoreAndGetNode(t, tempDir, testNode, warningExpected) + testPath, node := restoreAndGetNode(t, tempDir, &testNode, warningExpected) rawMessage := node.GenericAttributes[genericAttr] genericAttrsExpected, err := WindowsAttrsToGenericAttributes(genericAttributeExpected) test.OK(t, err) @@ -195,7 +195,7 @@ func runGenericAttributesTestForNodes(t *testing.T, expectedNodes []Node, tempDi } } -func restoreAndGetNode(t *testing.T, tempDir string, testNode Node, warningExpected bool) (string, *Node) { +func restoreAndGetNode(t *testing.T, tempDir string, testNode *Node, warningExpected bool) (string, *Node) { testPath := filepath.Join(tempDir, "001", testNode.Name) err := os.MkdirAll(filepath.Dir(testPath), testNode.Mode) test.OK(t, errors.Wrapf(err, "Failed to create parent directories for: %s", testPath)) @@ -211,7 +211,7 @@ func restoreAndGetNode(t *testing.T, tempDir string, testNode Node, warningExpec test.OK(t, errors.Wrapf(err, "Failed to create test directory: %s", testPath)) } - err = testNode.RestoreMetadata(testPath, func(msg string) { + err = NodeRestoreMetadata(testNode, testPath, func(msg string) { if warningExpected { test.Assert(t, warningExpected, "Warning triggered as expected: %s", msg) } else { @@ -260,7 +260,7 @@ func TestNewGenericAttributeType(t *testing.T) { }, } for _, testNode := range expectedNodes { - testPath, node := restoreAndGetNode(t, tempDir, testNode, true) + testPath, node := restoreAndGetNode(t, tempDir, &testNode, true) _, ua, err := genericAttributesToWindowsAttrs(node.GenericAttributes) test.OK(t, err) // Since this GenericAttribute is unknown to this version of the software, it will not get set on the file. @@ -296,7 +296,7 @@ func TestRestoreExtendedAttributes(t *testing.T) { }, } for _, testNode := range expectedNodes { - testPath, node := restoreAndGetNode(t, tempDir, testNode, false) + testPath, node := restoreAndGetNode(t, tempDir, &testNode, false) var handle windows.Handle var err error diff --git a/internal/restic/node_xattr.go b/internal/restic/node_xattr.go index 5a5a253d9..062ef4345 100644 --- a/internal/restic/node_xattr.go +++ b/internal/restic/node_xattr.go @@ -64,17 +64,17 @@ func handleXattrErr(err error) error { } } -// restoreGenericAttributes is no-op. -func (node *Node) restoreGenericAttributes(_ string, warn func(msg string)) error { - return node.handleAllUnknownGenericAttributesFound(warn) +// nodeRestoreGenericAttributes is no-op. +func nodeRestoreGenericAttributes(node *Node, _ string, warn func(msg string)) error { + return HandleAllUnknownGenericAttributesFound(node.GenericAttributes, warn) } -// fillGenericAttributes is a no-op. -func (node *Node) fillGenericAttributes(_ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { +// nodeFillGenericAttributes is a no-op. +func nodeFillGenericAttributes(_ *Node, _ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { return true, nil } -func (node Node) restoreExtendedAttributes(path string) error { +func nodeRestoreExtendedAttributes(node *Node, path string) error { expectedAttrs := map[string]struct{}{} for _, attr := range node.ExtendedAttributes { err := setxattr(path, attr.Name, attr.Value) @@ -101,7 +101,7 @@ func (node Node) restoreExtendedAttributes(path string) error { return nil } -func (node *Node) fillExtendedAttributes(path string, ignoreListError bool) error { +func nodeFillExtendedAttributes(node *Node, path string, ignoreListError bool) error { xattrs, err := listxattr(path) debug.Log("fillExtendedAttributes(%v) %v %v", path, xattrs, err) if err != nil { diff --git a/internal/restic/node_xattr_all_test.go b/internal/restic/node_xattr_all_test.go index 56ce5e286..30d29a6ed 100644 --- a/internal/restic/node_xattr_all_test.go +++ b/internal/restic/node_xattr_all_test.go @@ -21,18 +21,18 @@ func setAndVerifyXattr(t *testing.T, file string, attrs []ExtendedAttribute) { } } - node := Node{ + node := &Node{ Type: "file", ExtendedAttributes: attrs, } - rtest.OK(t, node.restoreExtendedAttributes(file)) + rtest.OK(t, nodeRestoreExtendedAttributes(node, file)) - nodeActual := Node{ + nodeActual := &Node{ Type: "file", } - rtest.OK(t, nodeActual.fillExtendedAttributes(file, false)) + rtest.OK(t, nodeFillExtendedAttributes(nodeActual, file, false)) - rtest.Assert(t, nodeActual.sameExtendedAttributes(node), "xattr mismatch got %v expected %v", nodeActual.ExtendedAttributes, node.ExtendedAttributes) + rtest.Assert(t, nodeActual.Equals(*node), "xattr mismatch got %v expected %v", nodeActual.ExtendedAttributes, node.ExtendedAttributes) } func TestOverwriteXattr(t *testing.T) { diff --git a/internal/restorer/restorer.go b/internal/restorer/restorer.go index 0e30b82f8..31157c979 100644 --- a/internal/restorer/restorer.go +++ b/internal/restorer/restorer.go @@ -272,7 +272,7 @@ func (res *Restorer) restoreNodeTo(ctx context.Context, node *restic.Node, targe return errors.Wrap(err, "RemoveNode") } - err := node.CreateAt(ctx, target, res.repo) + err := restic.NodeCreateAt(ctx, node, target, res.repo) if err != nil { debug.Log("node.CreateAt(%s) error %v", target, err) return err @@ -288,7 +288,7 @@ func (res *Restorer) restoreNodeMetadataTo(node *restic.Node, target, location s return nil } debug.Log("restoreNodeMetadata %v %v %v", node.Name, target, location) - err := node.RestoreMetadata(target, res.Warn) + err := restic.NodeRestoreMetadata(node, target, res.Warn) if err != nil { debug.Log("node.RestoreMetadata(%s) error %v", target, err) } From a2e54eac6427791c81f27acbe1daded84ba70b3b Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Mon, 26 Aug 2024 22:41:06 +0200 Subject: [PATCH 2/8] restic: simplify nodeCreateFileAt The code to write the file content is never used. --- internal/restic/node.go | 35 +++++------------------------------ internal/restic/node_test.go | 3 +-- internal/restorer/restorer.go | 6 +++--- 3 files changed, 9 insertions(+), 35 deletions(-) diff --git a/internal/restic/node.go b/internal/restic/node.go index c9c68cbee..e23d39f1b 100644 --- a/internal/restic/node.go +++ b/internal/restic/node.go @@ -1,7 +1,6 @@ package restic import ( - "context" "encoding/json" "fmt" "os" @@ -188,7 +187,7 @@ func (node Node) GetExtendedAttribute(a string) []byte { } // NodeCreateAt creates the node at the given path but does NOT restore node meta data. -func NodeCreateAt(ctx context.Context, node *Node, path string, repo BlobLoader) error { +func NodeCreateAt(node *Node, path string) error { debug.Log("create node %v at %v", node.Name, path) switch node.Type { @@ -197,7 +196,7 @@ func NodeCreateAt(ctx context.Context, node *Node, path string, repo BlobLoader) return err } case "file": - if err := nodeCreateFileAt(ctx, node, path, repo); err != nil { + if err := nodeCreateFileAt(path); err != nil { return err } case "symlink": @@ -310,38 +309,14 @@ func nodeCreateDirAt(node *Node, path string) error { return nil } -func nodeCreateFileAt(ctx context.Context, node *Node, path string, repo BlobLoader) error { +func nodeCreateFileAt(path string) error { f, err := fs.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) if err != nil { return errors.WithStack(err) } - err = nodeWriteNodeContent(ctx, node, repo, f) - closeErr := f.Close() - - if err != nil { - return err - } - - if closeErr != nil { - return errors.WithStack(closeErr) - } - - return nil -} - -func nodeWriteNodeContent(ctx context.Context, node *Node, repo BlobLoader, f *os.File) error { - var buf []byte - for _, id := range node.Content { - buf, err := repo.LoadBlob(ctx, DataBlob, id, buf) - if err != nil { - return err - } - - _, err = f.Write(buf) - if err != nil { - return errors.WithStack(err) - } + if err := f.Close(); err != nil { + return errors.WithStack(err) } return nil diff --git a/internal/restic/node_test.go b/internal/restic/node_test.go index 7258b5429..075dd5cc5 100644 --- a/internal/restic/node_test.go +++ b/internal/restic/node_test.go @@ -1,7 +1,6 @@ package restic import ( - "context" "encoding/json" "fmt" "os" @@ -245,7 +244,7 @@ func TestNodeRestoreAt(t *testing.T) { } else { nodePath = filepath.Join(tempdir, test.Name) } - rtest.OK(t, NodeCreateAt(context.TODO(), &test, nodePath, nil)) + rtest.OK(t, NodeCreateAt(&test, nodePath)) rtest.OK(t, NodeRestoreMetadata(&test, nodePath, func(msg string) { rtest.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", nodePath, msg)) })) fi, err := os.Lstat(nodePath) diff --git a/internal/restorer/restorer.go b/internal/restorer/restorer.go index 31157c979..83644c7ac 100644 --- a/internal/restorer/restorer.go +++ b/internal/restorer/restorer.go @@ -265,14 +265,14 @@ func (res *Restorer) traverseTreeInner(ctx context.Context, target, location str return filenames, hasRestored, nil } -func (res *Restorer) restoreNodeTo(ctx context.Context, node *restic.Node, target, location string) error { +func (res *Restorer) restoreNodeTo(node *restic.Node, target, location string) error { if !res.opts.DryRun { debug.Log("restoreNode %v %v %v", node.Name, target, location) if err := fs.Remove(target); err != nil && !errors.Is(err, os.ErrNotExist) { return errors.Wrap(err, "RemoveNode") } - err := restic.NodeCreateAt(ctx, node, target, res.repo) + err := restic.NodeCreateAt(node, target) if err != nil { debug.Log("node.CreateAt(%s) error %v", target, err) return err @@ -435,7 +435,7 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) (uint64, error) debug.Log("second pass, visitNode: restore node %q", location) if node.Type != "file" { _, err := res.withOverwriteCheck(ctx, node, target, location, false, nil, func(_ bool, _ *fileState) error { - return res.restoreNodeTo(ctx, node, target, location) + return res.restoreNodeTo(node, target, location) }) return err } From b9b32e5647a26ceb3a3ebe8a927ba35be6153323 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Mon, 26 Aug 2024 23:03:25 +0200 Subject: [PATCH 3/8] restic: extract Node filesystem code to fs package --- internal/archiver/archiver.go | 2 +- internal/archiver/archiver_test.go | 4 +- internal/archiver/archiver_unix_test.go | 2 +- internal/archiver/file_saver_test.go | 2 +- internal/{restic => fs}/mknod_unix.go | 2 +- internal/fs/node.go | 334 ++++++++++++++++++ internal/{restic => fs}/node_aix.go | 14 +- internal/{restic => fs}/node_darwin.go | 2 +- internal/{restic => fs}/node_freebsd.go | 2 +- internal/{restic => fs}/node_linux.go | 5 +- internal/{restic => fs}/node_netbsd.go | 14 +- internal/{restic => fs}/node_openbsd.go | 14 +- internal/{restic => fs}/node_solaris.go | 2 +- internal/fs/node_test.go | 324 +++++++++++++++++ internal/{restic => fs}/node_unix.go | 2 +- internal/{restic => fs}/node_unix_test.go | 7 +- internal/{restic => fs}/node_windows.go | 66 ++-- internal/{restic => fs}/node_windows_test.go | 56 +-- internal/{restic => fs}/node_xattr.go | 17 +- .../{restic => fs}/node_xattr_all_test.go | 13 +- internal/{restic => fs}/node_xattr_test.go | 2 +- internal/restic/node.go | 323 ----------------- internal/restic/node_test.go | 314 ---------------- internal/restic/tree_test.go | 5 +- internal/restorer/restorer.go | 4 +- internal/restorer/restorer_windows_test.go | 3 +- 26 files changed, 783 insertions(+), 752 deletions(-) rename internal/{restic => fs}/mknod_unix.go (93%) create mode 100644 internal/fs/node.go rename internal/{restic => fs}/node_aix.go (68%) rename internal/{restic => fs}/node_darwin.go (95%) rename internal/{restic => fs}/node_freebsd.go (96%) rename internal/{restic => fs}/node_linux.go (88%) rename internal/{restic => fs}/node_netbsd.go (58%) rename internal/{restic => fs}/node_openbsd.go (57%) rename internal/{restic => fs}/node_solaris.go (95%) create mode 100644 internal/fs/node_test.go rename internal/{restic => fs}/node_unix.go (97%) rename internal/{restic => fs}/node_unix_test.go (94%) rename internal/{restic => fs}/node_windows.go (88%) rename internal/{restic => fs}/node_windows_test.go (90%) rename internal/{restic => fs}/node_xattr.go (80%) rename internal/{restic => fs}/node_xattr_all_test.go (77%) rename internal/{restic => fs}/node_xattr_test.go (98%) diff --git a/internal/archiver/archiver.go b/internal/archiver/archiver.go index e7c346d3a..c576d047c 100644 --- a/internal/archiver/archiver.go +++ b/internal/archiver/archiver.go @@ -248,7 +248,7 @@ func (arch *Archiver) trackItem(item string, previous, current *restic.Node, s I // nodeFromFileInfo returns the restic node from an os.FileInfo. func (arch *Archiver) nodeFromFileInfo(snPath, filename string, fi os.FileInfo, ignoreXattrListError bool) (*restic.Node, error) { - node, err := restic.NodeFromFileInfo(filename, fi, ignoreXattrListError) + node, err := fs.NodeFromFileInfo(filename, fi, ignoreXattrListError) if !arch.WithAtime { node.AccessTime = node.ModTime } diff --git a/internal/archiver/archiver_test.go b/internal/archiver/archiver_test.go index c54f9ea33..18151eb02 100644 --- a/internal/archiver/archiver_test.go +++ b/internal/archiver/archiver_test.go @@ -557,7 +557,7 @@ func rename(t testing.TB, oldname, newname string) { } func nodeFromFI(t testing.TB, filename string, fi os.FileInfo) *restic.Node { - node, err := restic.NodeFromFileInfo(filename, fi, false) + node, err := fs.NodeFromFileInfo(filename, fi, false) if err != nil { t.Fatal(err) } @@ -2291,7 +2291,7 @@ func TestMetadataChanged(t *testing.T) { // get metadata fi := lstat(t, "testfile") - want, err := restic.NodeFromFileInfo("testfile", fi, false) + want, err := fs.NodeFromFileInfo("testfile", fi, false) if err != nil { t.Fatal(err) } diff --git a/internal/archiver/archiver_unix_test.go b/internal/archiver/archiver_unix_test.go index 4a380dff8..d91d993dd 100644 --- a/internal/archiver/archiver_unix_test.go +++ b/internal/archiver/archiver_unix_test.go @@ -48,7 +48,7 @@ func wrapFileInfo(fi os.FileInfo) os.FileInfo { func statAndSnapshot(t *testing.T, repo archiverRepo, name string) (*restic.Node, *restic.Node) { fi := lstat(t, name) - want, err := restic.NodeFromFileInfo(name, fi, false) + want, err := fs.NodeFromFileInfo(name, fi, false) rtest.OK(t, err) _, node := snapshot(t, repo, fs.Local{}, nil, name) diff --git a/internal/archiver/file_saver_test.go b/internal/archiver/file_saver_test.go index 409bdedd0..4a4327572 100644 --- a/internal/archiver/file_saver_test.go +++ b/internal/archiver/file_saver_test.go @@ -50,7 +50,7 @@ func startFileSaver(ctx context.Context, t testing.TB) (*FileSaver, context.Cont s := NewFileSaver(ctx, wg, saveBlob, pol, workers, workers) s.NodeFromFileInfo = func(snPath, filename string, fi os.FileInfo, ignoreXattrListError bool) (*restic.Node, error) { - return restic.NodeFromFileInfo(filename, fi, ignoreXattrListError) + return fs.NodeFromFileInfo(filename, fi, ignoreXattrListError) } return s, ctx, wg diff --git a/internal/restic/mknod_unix.go b/internal/fs/mknod_unix.go similarity index 93% rename from internal/restic/mknod_unix.go rename to internal/fs/mknod_unix.go index 7dd6c60d0..6127599f7 100644 --- a/internal/restic/mknod_unix.go +++ b/internal/fs/mknod_unix.go @@ -1,7 +1,7 @@ //go:build !freebsd && !windows // +build !freebsd,!windows -package restic +package fs import "golang.org/x/sys/unix" diff --git a/internal/fs/node.go b/internal/fs/node.go new file mode 100644 index 000000000..9bd507ba5 --- /dev/null +++ b/internal/fs/node.go @@ -0,0 +1,334 @@ +package fs + +import ( + "os" + "os/user" + "strconv" + "sync" + "syscall" + "time" + + "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/restic" +) + +// NodeFromFileInfo returns a new node from the given path and FileInfo. It +// returns the first error that is encountered, together with a node. +func NodeFromFileInfo(path string, fi os.FileInfo, ignoreXattrListError bool) (*restic.Node, error) { + mask := os.ModePerm | os.ModeType | os.ModeSetuid | os.ModeSetgid | os.ModeSticky + node := &restic.Node{ + Path: path, + Name: fi.Name(), + Mode: fi.Mode() & mask, + ModTime: fi.ModTime(), + } + + node.Type = nodeTypeFromFileInfo(fi) + if node.Type == "file" { + node.Size = uint64(fi.Size()) + } + + err := nodeFillExtra(node, path, fi, ignoreXattrListError) + return node, err +} + +func nodeTypeFromFileInfo(fi os.FileInfo) string { + switch fi.Mode() & os.ModeType { + case 0: + return "file" + case os.ModeDir: + return "dir" + case os.ModeSymlink: + return "symlink" + case os.ModeDevice | os.ModeCharDevice: + return "chardev" + case os.ModeDevice: + return "dev" + case os.ModeNamedPipe: + return "fifo" + case os.ModeSocket: + return "socket" + case os.ModeIrregular: + return "irregular" + } + + return "" +} + +func nodeFillExtra(node *restic.Node, path string, fi os.FileInfo, ignoreXattrListError bool) error { + stat, ok := toStatT(fi.Sys()) + if !ok { + // fill minimal info with current values for uid, gid + node.UID = uint32(os.Getuid()) + node.GID = uint32(os.Getgid()) + node.ChangeTime = node.ModTime + return nil + } + + node.Inode = uint64(stat.ino()) + node.DeviceID = uint64(stat.dev()) + + nodeFillTimes(node, stat) + + nodeFillUser(node, stat) + + switch node.Type { + case "file": + node.Size = uint64(stat.size()) + node.Links = uint64(stat.nlink()) + case "dir": + case "symlink": + var err error + node.LinkTarget, err = Readlink(path) + node.Links = uint64(stat.nlink()) + if err != nil { + return errors.WithStack(err) + } + case "dev": + node.Device = uint64(stat.rdev()) + node.Links = uint64(stat.nlink()) + case "chardev": + node.Device = uint64(stat.rdev()) + node.Links = uint64(stat.nlink()) + case "fifo": + case "socket": + default: + return errors.Errorf("unsupported file type %q", node.Type) + } + + allowExtended, err := nodeFillGenericAttributes(node, path, fi, stat) + if allowExtended { + // Skip processing ExtendedAttributes if allowExtended is false. + err = errors.CombineErrors(err, nodeFillExtendedAttributes(node, path, ignoreXattrListError)) + } + return err +} + +func nodeFillTimes(node *restic.Node, stat *statT) { + ctim := stat.ctim() + atim := stat.atim() + node.ChangeTime = time.Unix(ctim.Unix()) + node.AccessTime = time.Unix(atim.Unix()) +} + +func nodeFillUser(node *restic.Node, stat *statT) { + uid, gid := stat.uid(), stat.gid() + node.UID, node.GID = uid, gid + node.User = lookupUsername(uid) + node.Group = lookupGroup(gid) +} + +var ( + uidLookupCache = make(map[uint32]string) + uidLookupCacheMutex = sync.RWMutex{} +) + +// Cached user name lookup by uid. Returns "" when no name can be found. +func lookupUsername(uid uint32) string { + uidLookupCacheMutex.RLock() + username, ok := uidLookupCache[uid] + uidLookupCacheMutex.RUnlock() + + if ok { + return username + } + + u, err := user.LookupId(strconv.Itoa(int(uid))) + if err == nil { + username = u.Username + } + + uidLookupCacheMutex.Lock() + uidLookupCache[uid] = username + uidLookupCacheMutex.Unlock() + + return username +} + +var ( + gidLookupCache = make(map[uint32]string) + gidLookupCacheMutex = sync.RWMutex{} +) + +// Cached group name lookup by gid. Returns "" when no name can be found. +func lookupGroup(gid uint32) string { + gidLookupCacheMutex.RLock() + group, ok := gidLookupCache[gid] + gidLookupCacheMutex.RUnlock() + + if ok { + return group + } + + g, err := user.LookupGroupId(strconv.Itoa(int(gid))) + if err == nil { + group = g.Name + } + + gidLookupCacheMutex.Lock() + gidLookupCache[gid] = group + gidLookupCacheMutex.Unlock() + + return group +} + +// NodeCreateAt creates the node at the given path but does NOT restore node meta data. +func NodeCreateAt(node *restic.Node, path string) error { + debug.Log("create node %v at %v", node.Name, path) + + switch node.Type { + case "dir": + if err := nodeCreateDirAt(node, path); err != nil { + return err + } + case "file": + if err := nodeCreateFileAt(path); err != nil { + return err + } + case "symlink": + if err := nodeCreateSymlinkAt(node, path); err != nil { + return err + } + case "dev": + if err := nodeCreateDevAt(node, path); err != nil { + return err + } + case "chardev": + if err := nodeCreateCharDevAt(node, path); err != nil { + return err + } + case "fifo": + if err := nodeCreateFifoAt(path); err != nil { + return err + } + case "socket": + return nil + default: + return errors.Errorf("filetype %q not implemented", node.Type) + } + + return nil +} + +func nodeCreateDirAt(node *restic.Node, path string) error { + err := Mkdir(path, node.Mode) + if err != nil && !os.IsExist(err) { + return errors.WithStack(err) + } + + return nil +} + +func nodeCreateFileAt(path string) error { + f, err := OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) + if err != nil { + return errors.WithStack(err) + } + + if err := f.Close(); err != nil { + return errors.WithStack(err) + } + + return nil +} + +func nodeCreateSymlinkAt(node *restic.Node, path string) error { + if err := Symlink(node.LinkTarget, path); err != nil { + return errors.WithStack(err) + } + + return nil +} + +func nodeCreateDevAt(node *restic.Node, path string) error { + return mknod(path, syscall.S_IFBLK|0600, node.Device) +} + +func nodeCreateCharDevAt(node *restic.Node, path string) error { + return mknod(path, syscall.S_IFCHR|0600, node.Device) +} + +func nodeCreateFifoAt(path string) error { + return mkfifo(path, 0600) +} + +func mkfifo(path string, mode uint32) (err error) { + return mknod(path, mode|syscall.S_IFIFO, 0) +} + +// NodeRestoreMetadata restores node metadata +func NodeRestoreMetadata(node *restic.Node, path string, warn func(msg string)) error { + err := nodeRestoreMetadata(node, path, warn) + if err != nil { + // It is common to have permission errors for folders like /home + // unless you're running as root, so ignore those. + if os.Geteuid() > 0 && errors.Is(err, os.ErrPermission) { + debug.Log("not running as root, ignoring permission error for %v: %v", + path, err) + return nil + } + debug.Log("restoreMetadata(%s) error %v", path, err) + } + + return err +} + +func nodeRestoreMetadata(node *restic.Node, path string, warn func(msg string)) error { + var firsterr error + + if err := lchown(path, int(node.UID), int(node.GID)); err != nil { + firsterr = errors.WithStack(err) + } + + if err := nodeRestoreExtendedAttributes(node, path); err != nil { + debug.Log("error restoring extended attributes for %v: %v", path, err) + if firsterr == nil { + firsterr = err + } + } + + if err := nodeRestoreGenericAttributes(node, path, warn); err != nil { + debug.Log("error restoring generic attributes for %v: %v", path, err) + if firsterr == nil { + firsterr = err + } + } + + if err := NodeRestoreTimestamps(node, path); err != nil { + debug.Log("error restoring timestamps for %v: %v", path, err) + if firsterr == nil { + firsterr = err + } + } + + // Moving RestoreTimestamps and restoreExtendedAttributes calls above as for readonly files in windows + // calling Chmod below will no longer allow any modifications to be made on the file and the + // calls above would fail. + if node.Type != "symlink" { + if err := Chmod(path, node.Mode); err != nil { + if firsterr == nil { + firsterr = errors.WithStack(err) + } + } + } + + return firsterr +} + +func NodeRestoreTimestamps(node *restic.Node, path string) error { + var utimes = [...]syscall.Timespec{ + syscall.NsecToTimespec(node.AccessTime.UnixNano()), + syscall.NsecToTimespec(node.ModTime.UnixNano()), + } + + if node.Type == "symlink" { + return nodeRestoreSymlinkTimestamps(path, utimes) + } + + if err := syscall.UtimesNano(path, utimes[:]); err != nil { + return errors.Wrap(err, "UtimesNano") + } + + return nil +} diff --git a/internal/restic/node_aix.go b/internal/fs/node_aix.go similarity index 68% rename from internal/restic/node_aix.go rename to internal/fs/node_aix.go index 4cd279973..4e6944425 100644 --- a/internal/restic/node_aix.go +++ b/internal/fs/node_aix.go @@ -1,11 +1,13 @@ //go:build aix // +build aix -package restic +package fs import ( "os" "syscall" + + "github.com/restic/restic/internal/restic" ) func nodeRestoreSymlinkTimestamps(_ string, _ [2]syscall.Timespec) error { @@ -24,12 +26,12 @@ func (s statT) mtim() syscall.Timespec { return toTimespec(s.Mtim) } func (s statT) ctim() syscall.Timespec { return toTimespec(s.Ctim) } // nodeRestoreExtendedAttributes is a no-op on AIX. -func nodeRestoreExtendedAttributes(_ *Node, _ string) error { +func nodeRestoreExtendedAttributes(_ *restic.Node, _ string) error { return nil } // nodeFillExtendedAttributes is a no-op on AIX. -func nodeFillExtendedAttributes(_ *Node, _ string, _ bool) error { +func nodeFillExtendedAttributes(_ *restic.Node, _ string, _ bool) error { return nil } @@ -39,11 +41,11 @@ func IsListxattrPermissionError(_ error) bool { } // nodeRestoreGenericAttributes is no-op on AIX. -func nodeRestoreGenericAttributes(node *Node, _ string, warn func(msg string)) error { - return HandleAllUnknownGenericAttributesFound(node.GenericAttributes, warn) +func nodeRestoreGenericAttributes(node *restic.Node, _ string, warn func(msg string)) error { + return restic.HandleAllUnknownGenericAttributesFound(node.GenericAttributes, warn) } // nodeFillGenericAttributes is a no-op on AIX. -func nodeFillGenericAttributes(_ *Node, _ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { +func nodeFillGenericAttributes(_ *restic.Node, _ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { return true, nil } diff --git a/internal/restic/node_darwin.go b/internal/fs/node_darwin.go similarity index 95% rename from internal/restic/node_darwin.go rename to internal/fs/node_darwin.go index 099007e07..1ca7ce480 100644 --- a/internal/restic/node_darwin.go +++ b/internal/fs/node_darwin.go @@ -1,4 +1,4 @@ -package restic +package fs import "syscall" diff --git a/internal/restic/node_freebsd.go b/internal/fs/node_freebsd.go similarity index 96% rename from internal/restic/node_freebsd.go rename to internal/fs/node_freebsd.go index 6d2dd1d98..8796358b0 100644 --- a/internal/restic/node_freebsd.go +++ b/internal/fs/node_freebsd.go @@ -1,7 +1,7 @@ //go:build freebsd // +build freebsd -package restic +package fs import "syscall" diff --git a/internal/restic/node_linux.go b/internal/fs/node_linux.go similarity index 88% rename from internal/restic/node_linux.go rename to internal/fs/node_linux.go index 6311a224b..1cb4ee1ae 100644 --- a/internal/restic/node_linux.go +++ b/internal/fs/node_linux.go @@ -1,4 +1,4 @@ -package restic +package fs import ( "path/filepath" @@ -7,11 +7,10 @@ import ( "golang.org/x/sys/unix" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/fs" ) func nodeRestoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { - dir, err := fs.Open(filepath.Dir(path)) + dir, err := Open(filepath.Dir(path)) if err != nil { return errors.WithStack(err) } diff --git a/internal/restic/node_netbsd.go b/internal/fs/node_netbsd.go similarity index 58% rename from internal/restic/node_netbsd.go rename to internal/fs/node_netbsd.go index a53412afb..c71e4bdf5 100644 --- a/internal/restic/node_netbsd.go +++ b/internal/fs/node_netbsd.go @@ -1,8 +1,10 @@ -package restic +package fs import ( "os" "syscall" + + "github.com/restic/restic/internal/restic" ) func nodeRestoreSymlinkTimestamps(_ string, _ [2]syscall.Timespec) error { @@ -14,12 +16,12 @@ func (s statT) mtim() syscall.Timespec { return s.Mtimespec } func (s statT) ctim() syscall.Timespec { return s.Ctimespec } // nodeRestoreExtendedAttributes is a no-op on netbsd. -func nodeRestoreExtendedAttributes(_ *Node, _ string) error { +func nodeRestoreExtendedAttributes(_ *restic.Node, _ string) error { return nil } // nodeFillExtendedAttributes is a no-op on netbsd. -func nodeFillExtendedAttributes(_ *Node, _ string, _ bool) error { +func nodeFillExtendedAttributes(_ *restic.Node, _ string, _ bool) error { return nil } @@ -29,11 +31,11 @@ func IsListxattrPermissionError(_ error) bool { } // nodeRestoreGenericAttributes is no-op on netbsd. -func nodeRestoreGenericAttributes(node *Node, _ string, warn func(msg string)) error { - return HandleAllUnknownGenericAttributesFound(node.GenericAttributes, warn) +func nodeRestoreGenericAttributes(node *restic.Node, _ string, warn func(msg string)) error { + return restic.HandleAllUnknownGenericAttributesFound(node.GenericAttributes, warn) } // nodeFillGenericAttributes is a no-op on netbsd. -func nodeFillGenericAttributes(_ *Node, _ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { +func nodeFillGenericAttributes(_ *restic.Node, _ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { return true, nil } diff --git a/internal/restic/node_openbsd.go b/internal/fs/node_openbsd.go similarity index 57% rename from internal/restic/node_openbsd.go rename to internal/fs/node_openbsd.go index bbba89f2c..f74f2ae00 100644 --- a/internal/restic/node_openbsd.go +++ b/internal/fs/node_openbsd.go @@ -1,8 +1,10 @@ -package restic +package fs import ( "os" "syscall" + + "github.com/restic/restic/internal/restic" ) func nodeRestoreSymlinkTimestamps(_ string, _ [2]syscall.Timespec) error { @@ -14,12 +16,12 @@ func (s statT) mtim() syscall.Timespec { return s.Mtim } func (s statT) ctim() syscall.Timespec { return s.Ctim } // nodeRestoreExtendedAttributes is a no-op on openbsd. -func nodeRestoreExtendedAttributes(_ *Node, _ string) error { +func nodeRestoreExtendedAttributes(_ *restic.Node, _ string) error { return nil } // nodeFillExtendedAttributes is a no-op on openbsd. -func nodeFillExtendedAttributes(_ *Node, _ string, _ bool) error { +func nodeFillExtendedAttributes(_ *restic.Node, _ string, _ bool) error { return nil } @@ -29,11 +31,11 @@ func IsListxattrPermissionError(_ error) bool { } // nodeRestoreGenericAttributes is no-op on openbsd. -func nodeRestoreGenericAttributes(node *Node, _ string, warn func(msg string)) error { - return HandleAllUnknownGenericAttributesFound(node.GenericAttributes, warn) +func nodeRestoreGenericAttributes(node *restic.Node, _ string, warn func(msg string)) error { + return restic.HandleAllUnknownGenericAttributesFound(node.GenericAttributes, warn) } // fillGenericAttributes is a no-op on openbsd. -func nodeFillGenericAttributes(_ *Node, _ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { +func nodeFillGenericAttributes(_ *restic.Node, _ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { return true, nil } diff --git a/internal/restic/node_solaris.go b/internal/fs/node_solaris.go similarity index 95% rename from internal/restic/node_solaris.go rename to internal/fs/node_solaris.go index 114d11766..3f025b334 100644 --- a/internal/restic/node_solaris.go +++ b/internal/fs/node_solaris.go @@ -1,4 +1,4 @@ -package restic +package fs import "syscall" diff --git a/internal/fs/node_test.go b/internal/fs/node_test.go new file mode 100644 index 000000000..e7f608352 --- /dev/null +++ b/internal/fs/node_test.go @@ -0,0 +1,324 @@ +package fs + +import ( + "fmt" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/test" + rtest "github.com/restic/restic/internal/test" +) + +func BenchmarkNodeFillUser(t *testing.B) { + tempfile, err := os.CreateTemp("", "restic-test-temp-") + if err != nil { + t.Fatal(err) + } + + fi, err := tempfile.Stat() + if err != nil { + t.Fatal(err) + } + + path := tempfile.Name() + + t.ResetTimer() + + for i := 0; i < t.N; i++ { + _, err := NodeFromFileInfo(path, fi, false) + rtest.OK(t, err) + } + + rtest.OK(t, tempfile.Close()) + rtest.RemoveAll(t, tempfile.Name()) +} + +func BenchmarkNodeFromFileInfo(t *testing.B) { + tempfile, err := os.CreateTemp("", "restic-test-temp-") + if err != nil { + t.Fatal(err) + } + + fi, err := tempfile.Stat() + if err != nil { + t.Fatal(err) + } + + path := tempfile.Name() + + t.ResetTimer() + + for i := 0; i < t.N; i++ { + _, err := NodeFromFileInfo(path, fi, false) + if err != nil { + t.Fatal(err) + } + } + + rtest.OK(t, tempfile.Close()) + rtest.RemoveAll(t, tempfile.Name()) +} + +func parseTime(s string) time.Time { + t, err := time.Parse("2006-01-02 15:04:05.999", s) + if err != nil { + panic(err) + } + + return t.Local() +} + +var nodeTests = []restic.Node{ + { + Name: "testFile", + Type: "file", + Content: restic.IDs{}, + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Mode: 0604, + ModTime: parseTime("2015-05-14 21:07:23.111"), + AccessTime: parseTime("2015-05-14 21:07:24.222"), + ChangeTime: parseTime("2015-05-14 21:07:25.333"), + }, + { + Name: "testSuidFile", + Type: "file", + Content: restic.IDs{}, + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Mode: 0755 | os.ModeSetuid, + ModTime: parseTime("2015-05-14 21:07:23.111"), + AccessTime: parseTime("2015-05-14 21:07:24.222"), + ChangeTime: parseTime("2015-05-14 21:07:25.333"), + }, + { + Name: "testSuidFile2", + Type: "file", + Content: restic.IDs{}, + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Mode: 0755 | os.ModeSetgid, + ModTime: parseTime("2015-05-14 21:07:23.111"), + AccessTime: parseTime("2015-05-14 21:07:24.222"), + ChangeTime: parseTime("2015-05-14 21:07:25.333"), + }, + { + Name: "testSticky", + Type: "file", + Content: restic.IDs{}, + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Mode: 0755 | os.ModeSticky, + ModTime: parseTime("2015-05-14 21:07:23.111"), + AccessTime: parseTime("2015-05-14 21:07:24.222"), + ChangeTime: parseTime("2015-05-14 21:07:25.333"), + }, + { + Name: "testDir", + Type: "dir", + Subtree: nil, + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Mode: 0750 | os.ModeDir, + ModTime: parseTime("2015-05-14 21:07:23.111"), + AccessTime: parseTime("2015-05-14 21:07:24.222"), + ChangeTime: parseTime("2015-05-14 21:07:25.333"), + }, + { + Name: "testSymlink", + Type: "symlink", + LinkTarget: "invalid", + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Mode: 0777 | os.ModeSymlink, + ModTime: parseTime("2015-05-14 21:07:23.111"), + AccessTime: parseTime("2015-05-14 21:07:24.222"), + ChangeTime: parseTime("2015-05-14 21:07:25.333"), + }, + + // include "testFile" and "testDir" again with slightly different + // metadata, so we can test if CreateAt works with pre-existing files. + { + Name: "testFile", + Type: "file", + Content: restic.IDs{}, + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Mode: 0604, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + }, + { + Name: "testDir", + Type: "dir", + Subtree: nil, + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Mode: 0750 | os.ModeDir, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + }, + { + Name: "testXattrFile", + Type: "file", + Content: restic.IDs{}, + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Mode: 0604, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + ExtendedAttributes: []restic.ExtendedAttribute{ + {Name: "user.foo", Value: []byte("bar")}, + }, + }, + { + Name: "testXattrDir", + Type: "dir", + Subtree: nil, + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Mode: 0750 | os.ModeDir, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + ExtendedAttributes: []restic.ExtendedAttribute{ + {Name: "user.foo", Value: []byte("bar")}, + }, + }, + { + Name: "testXattrFileMacOSResourceFork", + Type: "file", + Content: restic.IDs{}, + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Mode: 0604, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + ExtendedAttributes: []restic.ExtendedAttribute{ + {Name: "com.apple.ResourceFork", Value: []byte("bar")}, + }, + }, +} + +func TestNodeRestoreAt(t *testing.T) { + tempdir := t.TempDir() + + for _, test := range nodeTests { + t.Run("", func(t *testing.T) { + var nodePath string + if test.ExtendedAttributes != nil { + if runtime.GOOS == "windows" { + // In windows extended attributes are case insensitive and windows returns + // the extended attributes in UPPER case. + // Update the tests to use UPPER case xattr names for windows. + extAttrArr := test.ExtendedAttributes + // Iterate through the array using pointers + for i := 0; i < len(extAttrArr); i++ { + extAttrArr[i].Name = strings.ToUpper(extAttrArr[i].Name) + } + } + for _, attr := range test.ExtendedAttributes { + if strings.HasPrefix(attr.Name, "com.apple.") && runtime.GOOS != "darwin" { + t.Skipf("attr %v only relevant on macOS", attr.Name) + } + } + + // tempdir might be backed by a filesystem that does not support + // extended attributes + nodePath = test.Name + defer func() { + _ = os.Remove(nodePath) + }() + } else { + nodePath = filepath.Join(tempdir, test.Name) + } + rtest.OK(t, NodeCreateAt(&test, nodePath)) + rtest.OK(t, NodeRestoreMetadata(&test, nodePath, func(msg string) { rtest.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", nodePath, msg)) })) + + fi, err := os.Lstat(nodePath) + rtest.OK(t, err) + + n2, err := NodeFromFileInfo(nodePath, fi, false) + rtest.OK(t, err) + n3, err := NodeFromFileInfo(nodePath, fi, true) + rtest.OK(t, err) + rtest.Assert(t, n2.Equals(*n3), "unexpected node info mismatch %v", cmp.Diff(n2, n3)) + + rtest.Assert(t, test.Name == n2.Name, + "%v: name doesn't match (%v != %v)", test.Type, test.Name, n2.Name) + rtest.Assert(t, test.Type == n2.Type, + "%v: type doesn't match (%v != %v)", test.Type, test.Type, n2.Type) + rtest.Assert(t, test.Size == n2.Size, + "%v: size doesn't match (%v != %v)", test.Size, test.Size, n2.Size) + + if runtime.GOOS != "windows" { + rtest.Assert(t, test.UID == n2.UID, + "%v: UID doesn't match (%v != %v)", test.Type, test.UID, n2.UID) + rtest.Assert(t, test.GID == n2.GID, + "%v: GID doesn't match (%v != %v)", test.Type, test.GID, n2.GID) + if test.Type != "symlink" { + // On OpenBSD only root can set sticky bit (see sticky(8)). + if runtime.GOOS != "openbsd" && runtime.GOOS != "netbsd" && runtime.GOOS != "solaris" && test.Name == "testSticky" { + rtest.Assert(t, test.Mode == n2.Mode, + "%v: mode doesn't match (0%o != 0%o)", test.Type, test.Mode, n2.Mode) + } + } + } + + AssertFsTimeEqual(t, "AccessTime", test.Type, test.AccessTime, n2.AccessTime) + AssertFsTimeEqual(t, "ModTime", test.Type, test.ModTime, n2.ModTime) + if len(n2.ExtendedAttributes) == 0 { + n2.ExtendedAttributes = nil + } + rtest.Assert(t, reflect.DeepEqual(test.ExtendedAttributes, n2.ExtendedAttributes), + "%v: xattrs don't match (%v != %v)", test.Name, test.ExtendedAttributes, n2.ExtendedAttributes) + }) + } +} + +func AssertFsTimeEqual(t *testing.T, label string, nodeType string, t1 time.Time, t2 time.Time) { + var equal bool + + // Go currently doesn't support setting timestamps of symbolic links on darwin and bsd + if nodeType == "symlink" { + switch runtime.GOOS { + case "darwin", "freebsd", "openbsd", "netbsd", "solaris": + return + } + } + + switch runtime.GOOS { + case "darwin": + // HFS+ timestamps don't support sub-second precision, + // see https://en.wikipedia.org/wiki/Comparison_of_file_systems + diff := int(t1.Sub(t2).Seconds()) + equal = diff == 0 + default: + equal = t1.Equal(t2) + } + + rtest.Assert(t, equal, "%s: %s doesn't match (%v != %v)", label, nodeType, t1, t2) +} + +func TestNodeRestoreMetadataError(t *testing.T) { + tempdir := t.TempDir() + + node := &nodeTests[0] + nodePath := filepath.Join(tempdir, node.Name) + + // This will fail because the target file does not exist + err := NodeRestoreMetadata(node, nodePath, func(msg string) { rtest.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", nodePath, msg)) }) + test.Assert(t, errors.Is(err, os.ErrNotExist), "failed for an unexpected reason") +} diff --git a/internal/restic/node_unix.go b/internal/fs/node_unix.go similarity index 97% rename from internal/restic/node_unix.go rename to internal/fs/node_unix.go index 976cd7b03..fb247ac99 100644 --- a/internal/restic/node_unix.go +++ b/internal/fs/node_unix.go @@ -1,7 +1,7 @@ //go:build !windows // +build !windows -package restic +package fs import ( "os" diff --git a/internal/restic/node_unix_test.go b/internal/fs/node_unix_test.go similarity index 94% rename from internal/restic/node_unix_test.go rename to internal/fs/node_unix_test.go index 9ea7b1725..b505357f2 100644 --- a/internal/restic/node_unix_test.go +++ b/internal/fs/node_unix_test.go @@ -1,7 +1,7 @@ //go:build !windows // +build !windows -package restic +package fs import ( "os" @@ -11,6 +11,7 @@ import ( "testing" "time" + "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -27,7 +28,7 @@ func stat(t testing.TB, filename string) (fi os.FileInfo, ok bool) { return fi, true } -func checkFile(t testing.TB, stat *syscall.Stat_t, node *Node) { +func checkFile(t testing.TB, stat *syscall.Stat_t, node *restic.Node) { t.Helper() if uint32(node.Mode.Perm()) != uint32(stat.Mode&0777) { t.Errorf("Mode does not match, want %v, got %v", stat.Mode&0777, node.Mode) @@ -80,7 +81,7 @@ func checkFile(t testing.TB, stat *syscall.Stat_t, node *Node) { } -func checkDevice(t testing.TB, stat *syscall.Stat_t, node *Node) { +func checkDevice(t testing.TB, stat *syscall.Stat_t, node *restic.Node) { if node.Device != uint64(stat.Rdev) { t.Errorf("Rdev does not match, want %v, got %v", stat.Rdev, node.Device) } diff --git a/internal/restic/node_windows.go b/internal/fs/node_windows.go similarity index 88% rename from internal/restic/node_windows.go rename to internal/fs/node_windows.go index 3f836ae61..90fa3462c 100644 --- a/internal/restic/node_windows.go +++ b/internal/fs/node_windows.go @@ -1,4 +1,4 @@ -package restic +package fs import ( "encoding/json" @@ -14,7 +14,7 @@ import ( "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/fs" + "github.com/restic/restic/internal/restic" "golang.org/x/sys/windows" ) @@ -82,12 +82,12 @@ func nodeRestoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error } // restore extended attributes for windows -func nodeRestoreExtendedAttributes(node *Node, path string) (err error) { +func nodeRestoreExtendedAttributes(node *restic.Node, path string) (err error) { count := len(node.ExtendedAttributes) if count > 0 { - eas := make([]fs.ExtendedAttribute, count) + eas := make([]ExtendedAttribute, count) for i, attr := range node.ExtendedAttributes { - eas[i] = fs.ExtendedAttribute{Name: attr.Name, Value: attr.Value} + eas[i] = ExtendedAttribute{Name: attr.Name, Value: attr.Value} } if errExt := restoreExtendedAttributes(node.Type, path, eas); errExt != nil { return errExt @@ -97,9 +97,9 @@ func nodeRestoreExtendedAttributes(node *Node, path string) (err error) { } // fill extended attributes in the node. This also includes the Generic attributes for windows. -func nodeFillExtendedAttributes(node *Node, path string, _ bool) (err error) { +func nodeFillExtendedAttributes(node *restic.Node, path string, _ bool) (err error) { var fileHandle windows.Handle - if fileHandle, err = fs.OpenHandleForEA(node.Type, path, false); fileHandle == 0 { + if fileHandle, err = OpenHandleForEA(node.Type, path, false); fileHandle == 0 { return nil } if err != nil { @@ -107,8 +107,8 @@ func nodeFillExtendedAttributes(node *Node, path string, _ bool) (err error) { } defer closeFileHandle(fileHandle, path) // Replaced inline defer with named function call //Get the windows Extended Attributes using the file handle - var extAtts []fs.ExtendedAttribute - extAtts, err = fs.GetFileEA(fileHandle) + var extAtts []ExtendedAttribute + extAtts, err = GetFileEA(fileHandle) debug.Log("fillExtendedAttributes(%v) %v", path, extAtts) if err != nil { return errors.Errorf("get EA failed for path %v, with: %v", path, err) @@ -119,7 +119,7 @@ func nodeFillExtendedAttributes(node *Node, path string, _ bool) (err error) { //Fill the ExtendedAttributes in the node using the name/value pairs in the windows EA for _, attr := range extAtts { - extendedAttr := ExtendedAttribute{ + extendedAttr := restic.ExtendedAttribute{ Name: attr.Name, Value: attr.Value, } @@ -139,9 +139,9 @@ func closeFileHandle(fileHandle windows.Handle, path string) { // restoreExtendedAttributes handles restore of the Windows Extended Attributes to the specified path. // The Windows API requires setting of all the Extended Attributes in one call. -func restoreExtendedAttributes(nodeType, path string, eas []fs.ExtendedAttribute) (err error) { +func restoreExtendedAttributes(nodeType, path string, eas []ExtendedAttribute) (err error) { var fileHandle windows.Handle - if fileHandle, err = fs.OpenHandleForEA(nodeType, path, true); fileHandle == 0 { + if fileHandle, err = OpenHandleForEA(nodeType, path, true); fileHandle == 0 { return nil } if err != nil { @@ -150,7 +150,7 @@ func restoreExtendedAttributes(nodeType, path string, eas []fs.ExtendedAttribute defer closeFileHandle(fileHandle, path) // Replaced inline defer with named function call // clear old unexpected xattrs by setting them to an empty value - oldEAs, err := fs.GetFileEA(fileHandle) + oldEAs, err := GetFileEA(fileHandle) if err != nil { return err } @@ -165,11 +165,11 @@ func restoreExtendedAttributes(nodeType, path string, eas []fs.ExtendedAttribute } if !found { - eas = append(eas, fs.ExtendedAttribute{Name: oldEA.Name, Value: nil}) + eas = append(eas, ExtendedAttribute{Name: oldEA.Name, Value: nil}) } } - if err = fs.SetFileEA(fileHandle, eas); err != nil { + if err = SetFileEA(fileHandle, eas); err != nil { return errors.Errorf("set EA failed for path %v, with: %v", path, err) } return nil @@ -210,7 +210,7 @@ func (s statT) ctim() syscall.Timespec { } // restoreGenericAttributes restores generic attributes for Windows -func nodeRestoreGenericAttributes(node *Node, path string, warn func(msg string)) (err error) { +func nodeRestoreGenericAttributes(node *restic.Node, path string, warn func(msg string)) (err error) { if len(node.GenericAttributes) == 0 { return nil } @@ -230,19 +230,19 @@ func nodeRestoreGenericAttributes(node *Node, path string, warn func(msg string) } } if windowsAttributes.SecurityDescriptor != nil { - if err := fs.SetSecurityDescriptor(path, windowsAttributes.SecurityDescriptor); err != nil { + if err := SetSecurityDescriptor(path, windowsAttributes.SecurityDescriptor); err != nil { errs = append(errs, fmt.Errorf("error restoring security descriptor for: %s : %v", path, err)) } } - HandleUnknownGenericAttributesFound(unknownAttribs, warn) + restic.HandleUnknownGenericAttributesFound(unknownAttribs, warn) return errors.CombineErrors(errs...) } // genericAttributesToWindowsAttrs converts the generic attributes map to a WindowsAttributes and also returns a string of unknown attributes that it could not convert. -func genericAttributesToWindowsAttrs(attrs map[GenericAttributeType]json.RawMessage) (windowsAttributes WindowsAttributes, unknownAttribs []GenericAttributeType, err error) { +func genericAttributesToWindowsAttrs(attrs map[restic.GenericAttributeType]json.RawMessage) (windowsAttributes WindowsAttributes, unknownAttribs []restic.GenericAttributeType, err error) { waValue := reflect.ValueOf(&windowsAttributes).Elem() - unknownAttribs, err = GenericAttributesToOSAttrs(attrs, reflect.TypeOf(windowsAttributes), &waValue, "windows") + unknownAttribs, err = restic.GenericAttributesToOSAttrs(attrs, reflect.TypeOf(windowsAttributes), &waValue, "windows") return windowsAttributes, unknownAttribs, err } @@ -289,14 +289,14 @@ func fixEncryptionAttribute(path string, attrs *uint32, pathPointer *uint16) (er // File should be encrypted. err = encryptFile(pathPointer) if err != nil { - if fs.IsAccessDenied(err) || errors.Is(err, windows.ERROR_FILE_READ_ONLY) { + if IsAccessDenied(err) || errors.Is(err, windows.ERROR_FILE_READ_ONLY) { // If existing file already has readonly or system flag, encrypt file call fails. // The readonly and system flags will be set again at the end of this func if they are needed. - err = fs.ResetPermissions(path) + err = ResetPermissions(path) if err != nil { return fmt.Errorf("failed to encrypt file: failed to reset permissions: %s : %v", path, err) } - err = fs.ClearSystem(path) + err = ClearSystem(path) if err != nil { return fmt.Errorf("failed to encrypt file: failed to clear system flag: %s : %v", path, err) } @@ -317,14 +317,14 @@ func fixEncryptionAttribute(path string, attrs *uint32, pathPointer *uint16) (er // File should not be encrypted, but its already encrypted. Decrypt it. err = decryptFile(pathPointer) if err != nil { - if fs.IsAccessDenied(err) || errors.Is(err, windows.ERROR_FILE_READ_ONLY) { + if IsAccessDenied(err) || errors.Is(err, windows.ERROR_FILE_READ_ONLY) { // If existing file already has readonly or system flag, decrypt file call fails. // The readonly and system flags will be set again after this func if they are needed. - err = fs.ResetPermissions(path) + err = ResetPermissions(path) if err != nil { return fmt.Errorf("failed to encrypt file: failed to reset permissions: %s : %v", path, err) } - err = fs.ClearSystem(path) + err = ClearSystem(path) if err != nil { return fmt.Errorf("failed to decrypt file: failed to clear system flag: %s : %v", path, err) } @@ -365,7 +365,7 @@ func decryptFile(pathPointer *uint16) error { // Created time and Security Descriptors. // It also checks if the volume supports extended attributes and stores the result in a map // so that it does not have to be checked again for subsequent calls for paths in the same volume. -func nodeFillGenericAttributes(node *Node, path string, fi os.FileInfo, stat *statT) (allowExtended bool, err error) { +func nodeFillGenericAttributes(node *restic.Node, path string, fi os.FileInfo, stat *statT) (allowExtended bool, err error) { if strings.Contains(filepath.Base(path), ":") { // Do not process for Alternate Data Streams in Windows // Also do not allow processing of extended attributes for ADS. @@ -392,7 +392,7 @@ func nodeFillGenericAttributes(node *Node, path string, fi os.FileInfo, stat *st if err != nil { return false, err } - if sd, err = fs.GetSecurityDescriptor(path); err != nil { + if sd, err = GetSecurityDescriptor(path); err != nil { return allowExtended, err } } @@ -422,7 +422,7 @@ func checkAndStoreEASupport(path string) (isEASupportedVolume bool, err error) { return eaSupportedValue.(bool), nil } // If not found, check if EA is supported with manually prepared volume name - isEASupportedVolume, err = fs.PathSupportsExtendedAttributes(volumeName + `\`) + isEASupportedVolume, err = PathSupportsExtendedAttributes(volumeName + `\`) // If the prepared volume name is not valid, we will fetch the actual volume name next. if err != nil && !errors.Is(err, windows.DNS_ERROR_INVALID_NAME) { debug.Log("Error checking if extended attributes are supported for prepared volume name %s: %v", volumeName, err) @@ -432,7 +432,7 @@ func checkAndStoreEASupport(path string) (isEASupportedVolume bool, err error) { } } // If an entry is not found, get the actual volume name using the GetVolumePathName function - volumeNameActual, err := fs.GetVolumePathName(path) + volumeNameActual, err := GetVolumePathName(path) if err != nil { debug.Log("Error getting actual volume name %s for path %s: %v", volumeName, path, err) // There can be multiple errors like path does not exist, bad network path, etc. @@ -447,7 +447,7 @@ func checkAndStoreEASupport(path string) (isEASupportedVolume bool, err error) { return eaSupportedValue.(bool), nil } // If the actual volume name is different and is not in the map, again check if the new volume supports extended attributes with the actual volume name - isEASupportedVolume, err = fs.PathSupportsExtendedAttributes(volumeNameActual + `\`) + isEASupportedVolume, err = PathSupportsExtendedAttributes(volumeNameActual + `\`) // Debug log for cases where the prepared volume name is not valid if err != nil { debug.Log("Error checking if extended attributes are supported for actual volume name %s: %v", volumeNameActual, err) @@ -496,10 +496,10 @@ func prepareVolumeName(path string) (volumeName string, err error) { } // windowsAttrsToGenericAttributes converts the WindowsAttributes to a generic attributes map using reflection -func WindowsAttrsToGenericAttributes(windowsAttributes WindowsAttributes) (attrs map[GenericAttributeType]json.RawMessage, err error) { +func WindowsAttrsToGenericAttributes(windowsAttributes WindowsAttributes) (attrs map[restic.GenericAttributeType]json.RawMessage, err error) { // Get the value of the WindowsAttributes windowsAttributesValue := reflect.ValueOf(windowsAttributes) - return OSAttrsToGenericAttributes(reflect.TypeOf(windowsAttributes), &windowsAttributesValue, runtime.GOOS) + return restic.OSAttrsToGenericAttributes(reflect.TypeOf(windowsAttributes), &windowsAttributesValue, runtime.GOOS) } // getCreationTime gets the value for the WindowsAttribute CreationTime in a windows specific time format. diff --git a/internal/restic/node_windows_test.go b/internal/fs/node_windows_test.go similarity index 90% rename from internal/restic/node_windows_test.go rename to internal/fs/node_windows_test.go index e78c8cb96..046c1984c 100644 --- a/internal/restic/node_windows_test.go +++ b/internal/fs/node_windows_test.go @@ -1,7 +1,7 @@ //go:build windows // +build windows -package restic +package fs import ( "encoding/base64" @@ -15,7 +15,7 @@ import ( "time" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/fs" + "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" "golang.org/x/sys/windows" ) @@ -23,10 +23,10 @@ import ( func TestRestoreSecurityDescriptors(t *testing.T) { t.Parallel() tempDir := t.TempDir() - for i, sd := range fs.TestFileSDs { + for i, sd := range TestFileSDs { testRestoreSecurityDescriptor(t, sd, tempDir, "file", fmt.Sprintf("testfile%d", i)) } - for i, sd := range fs.TestDirSDs { + for i, sd := range TestDirSDs { testRestoreSecurityDescriptor(t, sd, tempDir, "dir", fmt.Sprintf("testdir%d", i)) } } @@ -47,17 +47,17 @@ func testRestoreSecurityDescriptor(t *testing.T, sd string, tempDir, fileType, f sdByteFromRestoredNode := getWindowsAttr(t, testPath, node).SecurityDescriptor // Get the security descriptor for the test path after the restore. - sdBytesFromRestoredPath, err := fs.GetSecurityDescriptor(testPath) + sdBytesFromRestoredPath, err := GetSecurityDescriptor(testPath) test.OK(t, errors.Wrapf(err, "Error while getting the security descriptor for: %s", testPath)) // Compare the input SD and the SD got from the restored file. - fs.CompareSecurityDescriptors(t, testPath, sdInputBytes, *sdBytesFromRestoredPath) + CompareSecurityDescriptors(t, testPath, sdInputBytes, *sdBytesFromRestoredPath) // Compare the SD got from node constructed from the restored file info and the SD got directly from the restored file. - fs.CompareSecurityDescriptors(t, testPath, *sdByteFromRestoredNode, *sdBytesFromRestoredPath) + CompareSecurityDescriptors(t, testPath, *sdByteFromRestoredNode, *sdBytesFromRestoredPath) } -func getNode(name string, fileType string, genericAttributes map[GenericAttributeType]json.RawMessage) Node { - return Node{ +func getNode(name string, fileType string, genericAttributes map[restic.GenericAttributeType]json.RawMessage) restic.Node { + return restic.Node{ Name: name, Type: fileType, Mode: 0644, @@ -68,7 +68,7 @@ func getNode(name string, fileType string, genericAttributes map[GenericAttribut } } -func getWindowsAttr(t *testing.T, testPath string, node *Node) WindowsAttributes { +func getWindowsAttr(t *testing.T, testPath string, node *restic.Node) WindowsAttributes { windowsAttributes, unknownAttribs, err := genericAttributesToWindowsAttrs(node.GenericAttributes) test.OK(t, errors.Wrapf(err, "Error getting windows attr from generic attr: %s", testPath)) test.Assert(t, len(unknownAttribs) == 0, "Unknown attribs found: %s for: %s", unknownAttribs, testPath) @@ -83,12 +83,12 @@ func TestRestoreCreationTime(t *testing.T) { creationTimeAttribute := getCreationTime(fi, path) test.OK(t, errors.Wrapf(err, "Could not get creation time for path: %s", path)) //Using the temp dir creation time as the test creation time for the test file and folder - runGenericAttributesTest(t, path, TypeCreationTime, WindowsAttributes{CreationTime: creationTimeAttribute}, false) + runGenericAttributesTest(t, path, restic.TypeCreationTime, WindowsAttributes{CreationTime: creationTimeAttribute}, false) } func TestRestoreFileAttributes(t *testing.T) { t.Parallel() - genericAttributeName := TypeFileAttributes + genericAttributeName := restic.TypeFileAttributes tempDir := t.TempDir() normal := uint32(syscall.FILE_ATTRIBUTE_NORMAL) hidden := uint32(syscall.FILE_ATTRIBUTE_HIDDEN) @@ -110,7 +110,7 @@ func TestRestoreFileAttributes(t *testing.T) { for i, fileAttr := range fileAttributes { genericAttrs, err := WindowsAttrsToGenericAttributes(fileAttr) test.OK(t, err) - expectedNodes := []Node{ + expectedNodes := []restic.Node{ { Name: fmt.Sprintf("testfile%d", i), Type: "file", @@ -143,7 +143,7 @@ func TestRestoreFileAttributes(t *testing.T) { for i, folderAttr := range folderAttributes { genericAttrs, err := WindowsAttrsToGenericAttributes(folderAttr) test.OK(t, err) - expectedNodes := []Node{ + expectedNodes := []restic.Node{ { Name: fmt.Sprintf("testdirectory%d", i), Type: "dir", @@ -158,10 +158,10 @@ func TestRestoreFileAttributes(t *testing.T) { } } -func runGenericAttributesTest(t *testing.T, tempDir string, genericAttributeName GenericAttributeType, genericAttributeExpected WindowsAttributes, warningExpected bool) { +func runGenericAttributesTest(t *testing.T, tempDir string, genericAttributeName restic.GenericAttributeType, genericAttributeExpected WindowsAttributes, warningExpected bool) { genericAttributes, err := WindowsAttrsToGenericAttributes(genericAttributeExpected) test.OK(t, err) - expectedNodes := []Node{ + expectedNodes := []restic.Node{ { Name: "testfile", Type: "file", @@ -183,7 +183,7 @@ func runGenericAttributesTest(t *testing.T, tempDir string, genericAttributeName } runGenericAttributesTestForNodes(t, expectedNodes, tempDir, genericAttributeName, genericAttributeExpected, warningExpected) } -func runGenericAttributesTestForNodes(t *testing.T, expectedNodes []Node, tempDir string, genericAttr GenericAttributeType, genericAttributeExpected WindowsAttributes, warningExpected bool) { +func runGenericAttributesTestForNodes(t *testing.T, expectedNodes []restic.Node, tempDir string, genericAttr restic.GenericAttributeType, genericAttributeExpected WindowsAttributes, warningExpected bool) { for _, testNode := range expectedNodes { testPath, node := restoreAndGetNode(t, tempDir, &testNode, warningExpected) @@ -195,7 +195,7 @@ func runGenericAttributesTestForNodes(t *testing.T, expectedNodes []Node, tempDi } } -func restoreAndGetNode(t *testing.T, tempDir string, testNode *Node, warningExpected bool) (string, *Node) { +func restoreAndGetNode(t *testing.T, tempDir string, testNode *restic.Node, warningExpected bool) (string, *restic.Node) { testPath := filepath.Join(tempDir, "001", testNode.Name) err := os.MkdirAll(filepath.Dir(testPath), testNode.Mode) test.OK(t, errors.Wrapf(err, "Failed to create parent directories for: %s", testPath)) @@ -230,16 +230,16 @@ func restoreAndGetNode(t *testing.T, tempDir string, testNode *Node, warningExpe return testPath, nodeFromFileInfo } -const TypeSomeNewAttribute GenericAttributeType = "MockAttributes.SomeNewAttribute" +const TypeSomeNewAttribute restic.GenericAttributeType = "MockAttributes.SomeNewAttribute" func TestNewGenericAttributeType(t *testing.T) { t.Parallel() - newGenericAttribute := map[GenericAttributeType]json.RawMessage{} + newGenericAttribute := map[restic.GenericAttributeType]json.RawMessage{} newGenericAttribute[TypeSomeNewAttribute] = []byte("any value") tempDir := t.TempDir() - expectedNodes := []Node{ + expectedNodes := []restic.Node{ { Name: "testfile", Type: "file", @@ -271,7 +271,7 @@ func TestNewGenericAttributeType(t *testing.T) { func TestRestoreExtendedAttributes(t *testing.T) { t.Parallel() tempDir := t.TempDir() - expectedNodes := []Node{ + expectedNodes := []restic.Node{ { Name: "testfile", Type: "file", @@ -279,7 +279,7 @@ func TestRestoreExtendedAttributes(t *testing.T) { ModTime: parseTime("2005-05-14 21:07:03.111"), AccessTime: parseTime("2005-05-14 21:07:04.222"), ChangeTime: parseTime("2005-05-14 21:07:05.333"), - ExtendedAttributes: []ExtendedAttribute{ + ExtendedAttributes: []restic.ExtendedAttribute{ {"user.foo", []byte("bar")}, }, }, @@ -290,7 +290,7 @@ func TestRestoreExtendedAttributes(t *testing.T) { ModTime: parseTime("2005-05-14 21:07:03.111"), AccessTime: parseTime("2005-05-14 21:07:04.222"), ChangeTime: parseTime("2005-05-14 21:07:05.333"), - ExtendedAttributes: []ExtendedAttribute{ + ExtendedAttributes: []restic.ExtendedAttribute{ {"user.foo", []byte("bar")}, }, }, @@ -312,12 +312,12 @@ func TestRestoreExtendedAttributes(t *testing.T) { test.OK(t, errors.Wrapf(err, "Error closing file for: %s", testPath)) }() - extAttr, err := fs.GetFileEA(handle) + extAttr, err := GetFileEA(handle) test.OK(t, errors.Wrapf(err, "Error getting extended attributes for: %s", testPath)) test.Equals(t, len(node.ExtendedAttributes), len(extAttr)) for _, expectedExtAttr := range node.ExtendedAttributes { - var foundExtAttr *fs.ExtendedAttribute + var foundExtAttr *ExtendedAttribute for _, ea := range extAttr { if strings.EqualFold(ea.Name, expectedExtAttr.Name) { foundExtAttr = &ea @@ -491,13 +491,13 @@ func TestPrepareVolumeName(t *testing.T) { test.Equals(t, tc.expectedVolume, volume) if tc.isRealPath { - isEASupportedVolume, err := fs.PathSupportsExtendedAttributes(volume + `\`) + isEASupportedVolume, err := PathSupportsExtendedAttributes(volume + `\`) // If the prepared volume name is not valid, we will next fetch the actual volume name. test.OK(t, err) test.Equals(t, tc.expectedEASupported, isEASupportedVolume) - actualVolume, err := fs.GetVolumePathName(tc.path) + actualVolume, err := GetVolumePathName(tc.path) test.OK(t, err) test.Equals(t, tc.expectedVolume, actualVolume) } diff --git a/internal/restic/node_xattr.go b/internal/fs/node_xattr.go similarity index 80% rename from internal/restic/node_xattr.go rename to internal/fs/node_xattr.go index 062ef4345..11bdf382b 100644 --- a/internal/restic/node_xattr.go +++ b/internal/fs/node_xattr.go @@ -1,7 +1,7 @@ //go:build darwin || freebsd || linux || solaris // +build darwin freebsd linux solaris -package restic +package fs import ( "fmt" @@ -10,6 +10,7 @@ import ( "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/restic" "github.com/pkg/xattr" ) @@ -65,16 +66,16 @@ func handleXattrErr(err error) error { } // nodeRestoreGenericAttributes is no-op. -func nodeRestoreGenericAttributes(node *Node, _ string, warn func(msg string)) error { - return HandleAllUnknownGenericAttributesFound(node.GenericAttributes, warn) +func nodeRestoreGenericAttributes(node *restic.Node, _ string, warn func(msg string)) error { + return restic.HandleAllUnknownGenericAttributesFound(node.GenericAttributes, warn) } // nodeFillGenericAttributes is a no-op. -func nodeFillGenericAttributes(_ *Node, _ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { +func nodeFillGenericAttributes(_ *restic.Node, _ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { return true, nil } -func nodeRestoreExtendedAttributes(node *Node, path string) error { +func nodeRestoreExtendedAttributes(node *restic.Node, path string) error { expectedAttrs := map[string]struct{}{} for _, attr := range node.ExtendedAttributes { err := setxattr(path, attr.Name, attr.Value) @@ -101,7 +102,7 @@ func nodeRestoreExtendedAttributes(node *Node, path string) error { return nil } -func nodeFillExtendedAttributes(node *Node, path string, ignoreListError bool) error { +func nodeFillExtendedAttributes(node *restic.Node, path string, ignoreListError bool) error { xattrs, err := listxattr(path) debug.Log("fillExtendedAttributes(%v) %v %v", path, xattrs, err) if err != nil { @@ -111,14 +112,14 @@ func nodeFillExtendedAttributes(node *Node, path string, ignoreListError bool) e return err } - node.ExtendedAttributes = make([]ExtendedAttribute, 0, len(xattrs)) + node.ExtendedAttributes = make([]restic.ExtendedAttribute, 0, len(xattrs)) for _, attr := range xattrs { attrVal, err := getxattr(path, attr) if err != nil { fmt.Fprintf(os.Stderr, "can not obtain extended attribute %v for %v:\n", attr, path) continue } - attr := ExtendedAttribute{ + attr := restic.ExtendedAttribute{ Name: attr, Value: attrVal, } diff --git a/internal/restic/node_xattr_all_test.go b/internal/fs/node_xattr_all_test.go similarity index 77% rename from internal/restic/node_xattr_all_test.go rename to internal/fs/node_xattr_all_test.go index 30d29a6ed..39670d6e1 100644 --- a/internal/restic/node_xattr_all_test.go +++ b/internal/fs/node_xattr_all_test.go @@ -1,7 +1,7 @@ //go:build darwin || freebsd || linux || solaris || windows // +build darwin freebsd linux solaris windows -package restic +package fs import ( "os" @@ -10,10 +10,11 @@ import ( "strings" "testing" + "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) -func setAndVerifyXattr(t *testing.T, file string, attrs []ExtendedAttribute) { +func setAndVerifyXattr(t *testing.T, file string, attrs []restic.ExtendedAttribute) { if runtime.GOOS == "windows" { // windows seems to convert the xattr name to upper case for i := range attrs { @@ -21,13 +22,13 @@ func setAndVerifyXattr(t *testing.T, file string, attrs []ExtendedAttribute) { } } - node := &Node{ + node := &restic.Node{ Type: "file", ExtendedAttributes: attrs, } rtest.OK(t, nodeRestoreExtendedAttributes(node, file)) - nodeActual := &Node{ + nodeActual := &restic.Node{ Type: "file", } rtest.OK(t, nodeFillExtendedAttributes(nodeActual, file, false)) @@ -40,14 +41,14 @@ func TestOverwriteXattr(t *testing.T) { file := filepath.Join(dir, "file") rtest.OK(t, os.WriteFile(file, []byte("hello world"), 0o600)) - setAndVerifyXattr(t, file, []ExtendedAttribute{ + setAndVerifyXattr(t, file, []restic.ExtendedAttribute{ { Name: "user.foo", Value: []byte("bar"), }, }) - setAndVerifyXattr(t, file, []ExtendedAttribute{ + setAndVerifyXattr(t, file, []restic.ExtendedAttribute{ { Name: "user.other", Value: []byte("some"), diff --git a/internal/restic/node_xattr_test.go b/internal/fs/node_xattr_test.go similarity index 98% rename from internal/restic/node_xattr_test.go rename to internal/fs/node_xattr_test.go index 5ce77bd28..d948e3b31 100644 --- a/internal/restic/node_xattr_test.go +++ b/internal/fs/node_xattr_test.go @@ -1,7 +1,7 @@ //go:build darwin || freebsd || linux || solaris // +build darwin freebsd linux solaris -package restic +package fs import ( "os" diff --git a/internal/restic/node.go b/internal/restic/node.go index e23d39f1b..8bf97e59c 100644 --- a/internal/restic/node.go +++ b/internal/restic/node.go @@ -4,12 +4,10 @@ import ( "encoding/json" "fmt" "os" - "os/user" "reflect" "strconv" "strings" "sync" - "syscall" "time" "unicode/utf8" @@ -18,7 +16,6 @@ import ( "bytes" "github.com/restic/restic/internal/debug" - "github.com/restic/restic/internal/fs" ) // ExtendedAttribute is a tuple storing the xattr name and value for various filesystems. @@ -133,49 +130,6 @@ func (node Node) String() string { mode|node.Mode, node.UID, node.GID, node.Size, node.ModTime, node.Name) } -// NodeFromFileInfo returns a new node from the given path and FileInfo. It -// returns the first error that is encountered, together with a node. -func NodeFromFileInfo(path string, fi os.FileInfo, ignoreXattrListError bool) (*Node, error) { - mask := os.ModePerm | os.ModeType | os.ModeSetuid | os.ModeSetgid | os.ModeSticky - node := &Node{ - Path: path, - Name: fi.Name(), - Mode: fi.Mode() & mask, - ModTime: fi.ModTime(), - } - - node.Type = nodeTypeFromFileInfo(fi) - if node.Type == "file" { - node.Size = uint64(fi.Size()) - } - - err := nodeFillExtra(node, path, fi, ignoreXattrListError) - return node, err -} - -func nodeTypeFromFileInfo(fi os.FileInfo) string { - switch fi.Mode() & os.ModeType { - case 0: - return "file" - case os.ModeDir: - return "dir" - case os.ModeSymlink: - return "symlink" - case os.ModeDevice | os.ModeCharDevice: - return "chardev" - case os.ModeDevice: - return "dev" - case os.ModeNamedPipe: - return "fifo" - case os.ModeSocket: - return "socket" - case os.ModeIrregular: - return "irregular" - } - - return "" -} - // GetExtendedAttribute gets the extended attribute. func (node Node) GetExtendedAttribute(a string) []byte { for _, attr := range node.ExtendedAttributes { @@ -186,162 +140,6 @@ func (node Node) GetExtendedAttribute(a string) []byte { return nil } -// NodeCreateAt creates the node at the given path but does NOT restore node meta data. -func NodeCreateAt(node *Node, path string) error { - debug.Log("create node %v at %v", node.Name, path) - - switch node.Type { - case "dir": - if err := nodeCreateDirAt(node, path); err != nil { - return err - } - case "file": - if err := nodeCreateFileAt(path); err != nil { - return err - } - case "symlink": - if err := nodeCreateSymlinkAt(node, path); err != nil { - return err - } - case "dev": - if err := nodeCreateDevAt(node, path); err != nil { - return err - } - case "chardev": - if err := nodeCreateCharDevAt(node, path); err != nil { - return err - } - case "fifo": - if err := nodeCreateFifoAt(path); err != nil { - return err - } - case "socket": - return nil - default: - return errors.Errorf("filetype %q not implemented", node.Type) - } - - return nil -} - -// NodeRestoreMetadata restores node metadata -func NodeRestoreMetadata(node *Node, path string, warn func(msg string)) error { - err := nodeRestoreMetadata(node, path, warn) - if err != nil { - // It is common to have permission errors for folders like /home - // unless you're running as root, so ignore those. - if os.Geteuid() > 0 && errors.Is(err, os.ErrPermission) { - debug.Log("not running as root, ignoring permission error for %v: %v", - path, err) - return nil - } - debug.Log("restoreMetadata(%s) error %v", path, err) - } - - return err -} - -func nodeRestoreMetadata(node *Node, path string, warn func(msg string)) error { - var firsterr error - - if err := lchown(path, int(node.UID), int(node.GID)); err != nil { - firsterr = errors.WithStack(err) - } - - if err := nodeRestoreExtendedAttributes(node, path); err != nil { - debug.Log("error restoring extended attributes for %v: %v", path, err) - if firsterr == nil { - firsterr = err - } - } - - if err := nodeRestoreGenericAttributes(node, path, warn); err != nil { - debug.Log("error restoring generic attributes for %v: %v", path, err) - if firsterr == nil { - firsterr = err - } - } - - if err := NodeRestoreTimestamps(node, path); err != nil { - debug.Log("error restoring timestamps for %v: %v", path, err) - if firsterr == nil { - firsterr = err - } - } - - // Moving RestoreTimestamps and restoreExtendedAttributes calls above as for readonly files in windows - // calling Chmod below will no longer allow any modifications to be made on the file and the - // calls above would fail. - if node.Type != "symlink" { - if err := fs.Chmod(path, node.Mode); err != nil { - if firsterr == nil { - firsterr = errors.WithStack(err) - } - } - } - - return firsterr -} - -func NodeRestoreTimestamps(node *Node, path string) error { - var utimes = [...]syscall.Timespec{ - syscall.NsecToTimespec(node.AccessTime.UnixNano()), - syscall.NsecToTimespec(node.ModTime.UnixNano()), - } - - if node.Type == "symlink" { - return nodeRestoreSymlinkTimestamps(path, utimes) - } - - if err := syscall.UtimesNano(path, utimes[:]); err != nil { - return errors.Wrap(err, "UtimesNano") - } - - return nil -} - -func nodeCreateDirAt(node *Node, path string) error { - err := fs.Mkdir(path, node.Mode) - if err != nil && !os.IsExist(err) { - return errors.WithStack(err) - } - - return nil -} - -func nodeCreateFileAt(path string) error { - f, err := fs.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) - if err != nil { - return errors.WithStack(err) - } - - if err := f.Close(); err != nil { - return errors.WithStack(err) - } - - return nil -} - -func nodeCreateSymlinkAt(node *Node, path string) error { - if err := fs.Symlink(node.LinkTarget, path); err != nil { - return errors.WithStack(err) - } - - return nil -} - -func nodeCreateDevAt(node *Node, path string) error { - return mknod(path, syscall.S_IFBLK|0600, node.Device) -} - -func nodeCreateCharDevAt(node *Node, path string) error { - return mknod(path, syscall.S_IFCHR|0600, node.Device) -} - -func nodeCreateFifoAt(path string) error { - return mkfifo(path, 0600) -} - // FixTime returns a time.Time which can safely be used to marshal as JSON. If // the timestamp is earlier than year zero, the year is set to zero. In the same // way, if the year is larger than 9999, the year is set to 9999. Other than @@ -576,127 +374,6 @@ func deepEqual(map1, map2 map[GenericAttributeType]json.RawMessage) bool { return true } -func nodeFillUser(node *Node, stat *statT) { - uid, gid := stat.uid(), stat.gid() - node.UID, node.GID = uid, gid - node.User = lookupUsername(uid) - node.Group = lookupGroup(gid) -} - -var ( - uidLookupCache = make(map[uint32]string) - uidLookupCacheMutex = sync.RWMutex{} -) - -// Cached user name lookup by uid. Returns "" when no name can be found. -func lookupUsername(uid uint32) string { - uidLookupCacheMutex.RLock() - username, ok := uidLookupCache[uid] - uidLookupCacheMutex.RUnlock() - - if ok { - return username - } - - u, err := user.LookupId(strconv.Itoa(int(uid))) - if err == nil { - username = u.Username - } - - uidLookupCacheMutex.Lock() - uidLookupCache[uid] = username - uidLookupCacheMutex.Unlock() - - return username -} - -var ( - gidLookupCache = make(map[uint32]string) - gidLookupCacheMutex = sync.RWMutex{} -) - -// Cached group name lookup by gid. Returns "" when no name can be found. -func lookupGroup(gid uint32) string { - gidLookupCacheMutex.RLock() - group, ok := gidLookupCache[gid] - gidLookupCacheMutex.RUnlock() - - if ok { - return group - } - - g, err := user.LookupGroupId(strconv.Itoa(int(gid))) - if err == nil { - group = g.Name - } - - gidLookupCacheMutex.Lock() - gidLookupCache[gid] = group - gidLookupCacheMutex.Unlock() - - return group -} - -func nodeFillExtra(node *Node, path string, fi os.FileInfo, ignoreXattrListError bool) error { - stat, ok := toStatT(fi.Sys()) - if !ok { - // fill minimal info with current values for uid, gid - node.UID = uint32(os.Getuid()) - node.GID = uint32(os.Getgid()) - node.ChangeTime = node.ModTime - return nil - } - - node.Inode = uint64(stat.ino()) - node.DeviceID = uint64(stat.dev()) - - nodeFillTimes(node, stat) - - nodeFillUser(node, stat) - - switch node.Type { - case "file": - node.Size = uint64(stat.size()) - node.Links = uint64(stat.nlink()) - case "dir": - case "symlink": - var err error - node.LinkTarget, err = fs.Readlink(path) - node.Links = uint64(stat.nlink()) - if err != nil { - return errors.WithStack(err) - } - case "dev": - node.Device = uint64(stat.rdev()) - node.Links = uint64(stat.nlink()) - case "chardev": - node.Device = uint64(stat.rdev()) - node.Links = uint64(stat.nlink()) - case "fifo": - case "socket": - default: - return errors.Errorf("unsupported file type %q", node.Type) - } - - allowExtended, err := nodeFillGenericAttributes(node, path, fi, stat) - if allowExtended { - // Skip processing ExtendedAttributes if allowExtended is false. - err = errors.CombineErrors(err, nodeFillExtendedAttributes(node, path, ignoreXattrListError)) - } - return err -} - -func mkfifo(path string, mode uint32) (err error) { - return mknod(path, mode|syscall.S_IFIFO, 0) -} - -func nodeFillTimes(node *Node, stat *statT) { - ctim := stat.ctim() - atim := stat.atim() - node.ChangeTime = time.Unix(ctim.Unix()) - node.AccessTime = time.Unix(atim.Unix()) -} - // HandleUnknownGenericAttributesFound is used for handling and distinguing between scenarios related to future versions and cross-OS repositories func HandleUnknownGenericAttributesFound(unknownAttribs []GenericAttributeType, warn func(msg string)) { for _, unknownAttrib := range unknownAttribs { diff --git a/internal/restic/node_test.go b/internal/restic/node_test.go index 075dd5cc5..38a17cb09 100644 --- a/internal/restic/node_test.go +++ b/internal/restic/node_test.go @@ -3,315 +3,12 @@ package restic import ( "encoding/json" "fmt" - "os" - "path/filepath" - "reflect" - "runtime" - "strings" "testing" "time" - "github.com/google/go-cmp/cmp" - "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/test" - rtest "github.com/restic/restic/internal/test" ) -func BenchmarkNodeFillUser(t *testing.B) { - tempfile, err := os.CreateTemp("", "restic-test-temp-") - if err != nil { - t.Fatal(err) - } - - fi, err := tempfile.Stat() - if err != nil { - t.Fatal(err) - } - - path := tempfile.Name() - - t.ResetTimer() - - for i := 0; i < t.N; i++ { - _, err := NodeFromFileInfo(path, fi, false) - rtest.OK(t, err) - } - - rtest.OK(t, tempfile.Close()) - rtest.RemoveAll(t, tempfile.Name()) -} - -func BenchmarkNodeFromFileInfo(t *testing.B) { - tempfile, err := os.CreateTemp("", "restic-test-temp-") - if err != nil { - t.Fatal(err) - } - - fi, err := tempfile.Stat() - if err != nil { - t.Fatal(err) - } - - path := tempfile.Name() - - t.ResetTimer() - - for i := 0; i < t.N; i++ { - _, err := NodeFromFileInfo(path, fi, false) - if err != nil { - t.Fatal(err) - } - } - - rtest.OK(t, tempfile.Close()) - rtest.RemoveAll(t, tempfile.Name()) -} - -func parseTime(s string) time.Time { - t, err := time.Parse("2006-01-02 15:04:05.999", s) - if err != nil { - panic(err) - } - - return t.Local() -} - -var nodeTests = []Node{ - { - Name: "testFile", - Type: "file", - Content: IDs{}, - UID: uint32(os.Getuid()), - GID: uint32(os.Getgid()), - Mode: 0604, - ModTime: parseTime("2015-05-14 21:07:23.111"), - AccessTime: parseTime("2015-05-14 21:07:24.222"), - ChangeTime: parseTime("2015-05-14 21:07:25.333"), - }, - { - Name: "testSuidFile", - Type: "file", - Content: IDs{}, - UID: uint32(os.Getuid()), - GID: uint32(os.Getgid()), - Mode: 0755 | os.ModeSetuid, - ModTime: parseTime("2015-05-14 21:07:23.111"), - AccessTime: parseTime("2015-05-14 21:07:24.222"), - ChangeTime: parseTime("2015-05-14 21:07:25.333"), - }, - { - Name: "testSuidFile2", - Type: "file", - Content: IDs{}, - UID: uint32(os.Getuid()), - GID: uint32(os.Getgid()), - Mode: 0755 | os.ModeSetgid, - ModTime: parseTime("2015-05-14 21:07:23.111"), - AccessTime: parseTime("2015-05-14 21:07:24.222"), - ChangeTime: parseTime("2015-05-14 21:07:25.333"), - }, - { - Name: "testSticky", - Type: "file", - Content: IDs{}, - UID: uint32(os.Getuid()), - GID: uint32(os.Getgid()), - Mode: 0755 | os.ModeSticky, - ModTime: parseTime("2015-05-14 21:07:23.111"), - AccessTime: parseTime("2015-05-14 21:07:24.222"), - ChangeTime: parseTime("2015-05-14 21:07:25.333"), - }, - { - Name: "testDir", - Type: "dir", - Subtree: nil, - UID: uint32(os.Getuid()), - GID: uint32(os.Getgid()), - Mode: 0750 | os.ModeDir, - ModTime: parseTime("2015-05-14 21:07:23.111"), - AccessTime: parseTime("2015-05-14 21:07:24.222"), - ChangeTime: parseTime("2015-05-14 21:07:25.333"), - }, - { - Name: "testSymlink", - Type: "symlink", - LinkTarget: "invalid", - UID: uint32(os.Getuid()), - GID: uint32(os.Getgid()), - Mode: 0777 | os.ModeSymlink, - ModTime: parseTime("2015-05-14 21:07:23.111"), - AccessTime: parseTime("2015-05-14 21:07:24.222"), - ChangeTime: parseTime("2015-05-14 21:07:25.333"), - }, - - // include "testFile" and "testDir" again with slightly different - // metadata, so we can test if CreateAt works with pre-existing files. - { - Name: "testFile", - Type: "file", - Content: IDs{}, - UID: uint32(os.Getuid()), - GID: uint32(os.Getgid()), - Mode: 0604, - ModTime: parseTime("2005-05-14 21:07:03.111"), - AccessTime: parseTime("2005-05-14 21:07:04.222"), - ChangeTime: parseTime("2005-05-14 21:07:05.333"), - }, - { - Name: "testDir", - Type: "dir", - Subtree: nil, - UID: uint32(os.Getuid()), - GID: uint32(os.Getgid()), - Mode: 0750 | os.ModeDir, - ModTime: parseTime("2005-05-14 21:07:03.111"), - AccessTime: parseTime("2005-05-14 21:07:04.222"), - ChangeTime: parseTime("2005-05-14 21:07:05.333"), - }, - { - Name: "testXattrFile", - Type: "file", - Content: IDs{}, - UID: uint32(os.Getuid()), - GID: uint32(os.Getgid()), - Mode: 0604, - ModTime: parseTime("2005-05-14 21:07:03.111"), - AccessTime: parseTime("2005-05-14 21:07:04.222"), - ChangeTime: parseTime("2005-05-14 21:07:05.333"), - ExtendedAttributes: []ExtendedAttribute{ - {"user.foo", []byte("bar")}, - }, - }, - { - Name: "testXattrDir", - Type: "dir", - Subtree: nil, - UID: uint32(os.Getuid()), - GID: uint32(os.Getgid()), - Mode: 0750 | os.ModeDir, - ModTime: parseTime("2005-05-14 21:07:03.111"), - AccessTime: parseTime("2005-05-14 21:07:04.222"), - ChangeTime: parseTime("2005-05-14 21:07:05.333"), - ExtendedAttributes: []ExtendedAttribute{ - {"user.foo", []byte("bar")}, - }, - }, - { - Name: "testXattrFileMacOSResourceFork", - Type: "file", - Content: IDs{}, - UID: uint32(os.Getuid()), - GID: uint32(os.Getgid()), - Mode: 0604, - ModTime: parseTime("2005-05-14 21:07:03.111"), - AccessTime: parseTime("2005-05-14 21:07:04.222"), - ChangeTime: parseTime("2005-05-14 21:07:05.333"), - ExtendedAttributes: []ExtendedAttribute{ - {"com.apple.ResourceFork", []byte("bar")}, - }, - }, -} - -func TestNodeRestoreAt(t *testing.T) { - tempdir := t.TempDir() - - for _, test := range nodeTests { - t.Run("", func(t *testing.T) { - var nodePath string - if test.ExtendedAttributes != nil { - if runtime.GOOS == "windows" { - // In windows extended attributes are case insensitive and windows returns - // the extended attributes in UPPER case. - // Update the tests to use UPPER case xattr names for windows. - extAttrArr := test.ExtendedAttributes - // Iterate through the array using pointers - for i := 0; i < len(extAttrArr); i++ { - extAttrArr[i].Name = strings.ToUpper(extAttrArr[i].Name) - } - } - for _, attr := range test.ExtendedAttributes { - if strings.HasPrefix(attr.Name, "com.apple.") && runtime.GOOS != "darwin" { - t.Skipf("attr %v only relevant on macOS", attr.Name) - } - } - - // tempdir might be backed by a filesystem that does not support - // extended attributes - nodePath = test.Name - defer func() { - _ = os.Remove(nodePath) - }() - } else { - nodePath = filepath.Join(tempdir, test.Name) - } - rtest.OK(t, NodeCreateAt(&test, nodePath)) - rtest.OK(t, NodeRestoreMetadata(&test, nodePath, func(msg string) { rtest.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", nodePath, msg)) })) - - fi, err := os.Lstat(nodePath) - rtest.OK(t, err) - - n2, err := NodeFromFileInfo(nodePath, fi, false) - rtest.OK(t, err) - n3, err := NodeFromFileInfo(nodePath, fi, true) - rtest.OK(t, err) - rtest.Assert(t, n2.Equals(*n3), "unexpected node info mismatch %v", cmp.Diff(n2, n3)) - - rtest.Assert(t, test.Name == n2.Name, - "%v: name doesn't match (%v != %v)", test.Type, test.Name, n2.Name) - rtest.Assert(t, test.Type == n2.Type, - "%v: type doesn't match (%v != %v)", test.Type, test.Type, n2.Type) - rtest.Assert(t, test.Size == n2.Size, - "%v: size doesn't match (%v != %v)", test.Size, test.Size, n2.Size) - - if runtime.GOOS != "windows" { - rtest.Assert(t, test.UID == n2.UID, - "%v: UID doesn't match (%v != %v)", test.Type, test.UID, n2.UID) - rtest.Assert(t, test.GID == n2.GID, - "%v: GID doesn't match (%v != %v)", test.Type, test.GID, n2.GID) - if test.Type != "symlink" { - // On OpenBSD only root can set sticky bit (see sticky(8)). - if runtime.GOOS != "openbsd" && runtime.GOOS != "netbsd" && runtime.GOOS != "solaris" && test.Name == "testSticky" { - rtest.Assert(t, test.Mode == n2.Mode, - "%v: mode doesn't match (0%o != 0%o)", test.Type, test.Mode, n2.Mode) - } - } - } - - AssertFsTimeEqual(t, "AccessTime", test.Type, test.AccessTime, n2.AccessTime) - AssertFsTimeEqual(t, "ModTime", test.Type, test.ModTime, n2.ModTime) - if len(n2.ExtendedAttributes) == 0 { - n2.ExtendedAttributes = nil - } - rtest.Assert(t, reflect.DeepEqual(test.ExtendedAttributes, n2.ExtendedAttributes), - "%v: xattrs don't match (%v != %v)", test.Name, test.ExtendedAttributes, n2.ExtendedAttributes) - }) - } -} - -func AssertFsTimeEqual(t *testing.T, label string, nodeType string, t1 time.Time, t2 time.Time) { - var equal bool - - // Go currently doesn't support setting timestamps of symbolic links on darwin and bsd - if nodeType == "symlink" { - switch runtime.GOOS { - case "darwin", "freebsd", "openbsd", "netbsd", "solaris": - return - } - } - - switch runtime.GOOS { - case "darwin": - // HFS+ timestamps don't support sub-second precision, - // see https://en.wikipedia.org/wiki/Comparison_of_file_systems - diff := int(t1.Sub(t2).Seconds()) - equal = diff == 0 - default: - equal = t1.Equal(t2) - } - - rtest.Assert(t, equal, "%s: %s doesn't match (%v != %v)", label, nodeType, t1, t2) -} - func parseTimeNano(t testing.TB, s string) time.Time { // 2006-01-02T15:04:05.999999999Z07:00 ts, err := time.Parse(time.RFC3339Nano, s) @@ -397,14 +94,3 @@ func TestSymlinkSerializationFormat(t *testing.T) { test.Assert(t, n2.LinkTargetRaw == nil, "quoted link target is just a helper field and must be unset after decoding") } } - -func TestNodeRestoreMetadataError(t *testing.T) { - tempdir := t.TempDir() - - node := &nodeTests[0] - nodePath := filepath.Join(tempdir, node.Name) - - // This will fail because the target file does not exist - err := NodeRestoreMetadata(node, nodePath, func(msg string) { rtest.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", nodePath, msg)) }) - test.Assert(t, errors.Is(err, os.ErrNotExist), "failed for an unexpected reason") -} diff --git a/internal/restic/tree_test.go b/internal/restic/tree_test.go index 8e0b3587a..cdd6b3c18 100644 --- a/internal/restic/tree_test.go +++ b/internal/restic/tree_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/restic/restic/internal/archiver" + "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" @@ -86,7 +87,7 @@ func TestNodeComparison(t *testing.T) { fi, err := os.Lstat("tree_test.go") rtest.OK(t, err) - node, err := restic.NodeFromFileInfo("tree_test.go", fi, false) + node, err := fs.NodeFromFileInfo("tree_test.go", fi, false) rtest.OK(t, err) n2 := *node @@ -127,7 +128,7 @@ func TestTreeEqualSerialization(t *testing.T) { for _, fn := range files[:i] { fi, err := os.Lstat(fn) rtest.OK(t, err) - node, err := restic.NodeFromFileInfo(fn, fi, false) + node, err := fs.NodeFromFileInfo(fn, fi, false) rtest.OK(t, err) rtest.OK(t, tree.Insert(node)) diff --git a/internal/restorer/restorer.go b/internal/restorer/restorer.go index 83644c7ac..26b6f3474 100644 --- a/internal/restorer/restorer.go +++ b/internal/restorer/restorer.go @@ -272,7 +272,7 @@ func (res *Restorer) restoreNodeTo(node *restic.Node, target, location string) e return errors.Wrap(err, "RemoveNode") } - err := restic.NodeCreateAt(node, target) + err := fs.NodeCreateAt(node, target) if err != nil { debug.Log("node.CreateAt(%s) error %v", target, err) return err @@ -288,7 +288,7 @@ func (res *Restorer) restoreNodeMetadataTo(node *restic.Node, target, location s return nil } debug.Log("restoreNodeMetadata %v %v %v", node.Name, target, location) - err := restic.NodeRestoreMetadata(node, target, res.Warn) + err := fs.NodeRestoreMetadata(node, target, res.Warn) if err != nil { debug.Log("node.RestoreMetadata(%s) error %v", target, err) } diff --git a/internal/restorer/restorer_windows_test.go b/internal/restorer/restorer_windows_test.go index 4764bed2d..9fcdfc48d 100644 --- a/internal/restorer/restorer_windows_test.go +++ b/internal/restorer/restorer_windows_test.go @@ -16,6 +16,7 @@ import ( "unsafe" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" @@ -263,7 +264,7 @@ func setup(t *testing.T, nodesMap map[string]Node) *Restorer { //If the node is a directory add FILE_ATTRIBUTE_DIRECTORY to attributes fileattr |= windows.FILE_ATTRIBUTE_DIRECTORY } - attrs, err := restic.WindowsAttrsToGenericAttributes(restic.WindowsAttributes{FileAttributes: &fileattr}) + attrs, err := fs.WindowsAttrsToGenericAttributes(fs.WindowsAttributes{FileAttributes: &fileattr}) test.OK(t, err) return attrs } From fc549c94621920ed71995e529443bc0b108a7103 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 10 Jul 2024 22:28:48 +0200 Subject: [PATCH 4/8] cleanup imports --- internal/restorer/fileswriter.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/internal/restorer/fileswriter.go b/internal/restorer/fileswriter.go index 962f66619..d6f78f2d7 100644 --- a/internal/restorer/fileswriter.go +++ b/internal/restorer/fileswriter.go @@ -2,7 +2,6 @@ package restorer import ( "fmt" - stdfs "io/fs" "os" "sync" "syscall" @@ -82,7 +81,7 @@ func createFile(path string, createSize int64, sparse bool, allowRecursiveDelete return nil, err } - var fi stdfs.FileInfo + var fi os.FileInfo if f != nil { // stat to check that we've opened a regular file fi, err = f.Stat() @@ -135,7 +134,7 @@ func createFile(path string, createSize int64, sparse bool, allowRecursiveDelete return ensureSize(f, fi, createSize, sparse) } -func ensureSize(f *os.File, fi stdfs.FileInfo, createSize int64, sparse bool) (*os.File, error) { +func ensureSize(f *os.File, fi os.FileInfo, createSize int64, sparse bool) (*os.File, error) { if sparse { err := truncateSparse(f, createSize) if err != nil { From 0ddb4441d7e74eab98389dd2e917e99b5327e9a1 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 21 Jul 2024 14:40:33 +0200 Subject: [PATCH 5/8] fs: clean up helper functions --- internal/archiver/archiver.go | 4 ++-- internal/archiver/testing.go | 4 ++-- internal/archiver/testing_test.go | 2 +- internal/fs/file.go | 20 -------------------- internal/fs/helpers.go | 13 ------------- 5 files changed, 5 insertions(+), 38 deletions(-) delete mode 100644 internal/fs/helpers.go diff --git a/internal/archiver/archiver.go b/internal/archiver/archiver.go index c576d047c..4f0990843 100644 --- a/internal/archiver/archiver.go +++ b/internal/archiver/archiver.go @@ -446,7 +446,7 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous } switch { - case fs.IsRegularFile(fi): + case fi.Mode().IsRegular(): debug.Log(" %v regular file", target) // check if the file has not changed before performing a fopen operation (more expensive, specially @@ -505,7 +505,7 @@ func (arch *Archiver) save(ctx context.Context, snPath, target string, previous } // make sure it's still a file - if !fs.IsRegularFile(fi) { + if !fi.Mode().IsRegular() { err = errors.Errorf("file %v changed type, refusing to archive", fi.Name()) _ = file.Close() err = arch.error(abstarget, err) diff --git a/internal/archiver/testing.go b/internal/archiver/testing.go index 106e68445..8bd854904 100644 --- a/internal/archiver/testing.go +++ b/internal/archiver/testing.go @@ -169,7 +169,7 @@ func TestEnsureFiles(t testing.TB, target string, dir TestDir) { } return nil case TestFile: - if !fs.IsRegularFile(fi) { + if !fi.Mode().IsRegular() { t.Errorf("is not a regular file: %v", path) return nil } @@ -208,7 +208,7 @@ func TestEnsureFiles(t testing.TB, target string, dir TestDir) { }) // then, traverse the directory again, looking for additional files - err := fs.Walk(target, func(path string, fi os.FileInfo, err error) error { + err := filepath.Walk(target, func(path string, fi os.FileInfo, err error) error { if err != nil { return err } diff --git a/internal/archiver/testing_test.go b/internal/archiver/testing_test.go index ff3bd3668..bb4b63a82 100644 --- a/internal/archiver/testing_test.go +++ b/internal/archiver/testing_test.go @@ -122,7 +122,7 @@ func TestTestCreateFiles(t *testing.T) { switch node := item.(type) { case TestFile: - if !fs.IsRegularFile(fi) { + if !fi.Mode().IsRegular() { t.Errorf("is not regular file: %v", name) continue } diff --git a/internal/fs/file.go b/internal/fs/file.go index 85b202dc8..1071f4e87 100644 --- a/internal/fs/file.go +++ b/internal/fs/file.go @@ -3,7 +3,6 @@ package fs import ( "fmt" "os" - "path/filepath" "time" ) @@ -75,15 +74,6 @@ func Lstat(name string) (os.FileInfo, error) { return os.Lstat(fixpath(name)) } -// Create creates the named file with mode 0666 (before umask), truncating -// it if it already exists. If successful, methods on the returned -// File can be used for I/O; the associated file descriptor has mode -// O_RDWR. -// If there is an error, it will be of type *PathError. -func Create(name string) (*os.File, error) { - return os.Create(fixpath(name)) -} - // Open opens a file for reading. func Open(name string) (File, error) { return os.Open(fixpath(name)) @@ -98,16 +88,6 @@ func OpenFile(name string, flag int, perm os.FileMode) (*os.File, error) { return os.OpenFile(fixpath(name), flag, perm) } -// Walk walks the file tree rooted at root, calling walkFn for each file or -// directory in the tree, including root. All errors that arise visiting files -// and directories are filtered by walkFn. The files are walked in lexical -// order, which makes the output deterministic but means that for very -// large directories Walk can be inefficient. -// Walk does not follow symbolic links. -func Walk(root string, walkFn filepath.WalkFunc) error { - return filepath.Walk(fixpath(root), walkFn) -} - // RemoveIfExists removes a file, returning no error if it does not exist. func RemoveIfExists(filename string) error { err := os.Remove(filename) diff --git a/internal/fs/helpers.go b/internal/fs/helpers.go deleted file mode 100644 index 4dd1e0e73..000000000 --- a/internal/fs/helpers.go +++ /dev/null @@ -1,13 +0,0 @@ -package fs - -import "os" - -// IsRegularFile returns true if fi belongs to a normal file. If fi is nil, -// false is returned. -func IsRegularFile(fi os.FileInfo) bool { - if fi == nil { - return false - } - - return fi.Mode()&os.ModeType == 0 -} From 80ed863aab455e3843136c69c3639d23cb4fd720 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 21 Jul 2024 15:00:34 +0200 Subject: [PATCH 6/8] repository: remove redundant cleanup code The temp files used by the packer manager are either delete after creation (unix) or marked as delete on close (windows). Thus, no explicit cleanup is necessary. --- internal/fs/file.go | 9 --------- internal/repository/packer_manager.go | 9 --------- 2 files changed, 18 deletions(-) diff --git a/internal/fs/file.go b/internal/fs/file.go index 1071f4e87..356b466c3 100644 --- a/internal/fs/file.go +++ b/internal/fs/file.go @@ -88,15 +88,6 @@ func OpenFile(name string, flag int, perm os.FileMode) (*os.File, error) { return os.OpenFile(fixpath(name), flag, perm) } -// RemoveIfExists removes a file, returning no error if it does not exist. -func RemoveIfExists(filename string) error { - err := os.Remove(filename) - if err != nil && os.IsNotExist(err) { - err = nil - } - return err -} - // Chtimes changes the access and modification times of the named file, // similar to the Unix utime() or utimes() functions. // diff --git a/internal/repository/packer_manager.go b/internal/repository/packer_manager.go index 8dd8f71c1..731ad9a6a 100644 --- a/internal/repository/packer_manager.go +++ b/internal/repository/packer_manager.go @@ -6,7 +6,6 @@ import ( "crypto/sha256" "io" "os" - "runtime" "sync" "github.com/restic/restic/internal/backend" @@ -186,14 +185,6 @@ func (r *Repository) savePacker(ctx context.Context, t restic.BlobType, p *packe return errors.Wrap(err, "close tempfile") } - // on windows the tempfile is automatically deleted on close - if runtime.GOOS != "windows" { - err = fs.RemoveIfExists(p.tmpfile.Name()) - if err != nil { - return errors.WithStack(err) - } - } - // update blobs in the index debug.Log(" updating blobs %v to pack %v", p.Packer.Blobs(), id) r.idx.StorePack(id, p.Packer.Blobs()) From 263709da8c1a711cf4979cdcbba2d90d110f9779 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 21 Jul 2024 15:03:17 +0200 Subject: [PATCH 7/8] fs: unexport isListxattrPermissionError --- internal/fs/node_aix.go | 4 ++-- internal/fs/node_netbsd.go | 4 ++-- internal/fs/node_openbsd.go | 4 ++-- internal/fs/node_xattr.go | 4 ++-- internal/fs/node_xattr_test.go | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/internal/fs/node_aix.go b/internal/fs/node_aix.go index 4e6944425..123985c2d 100644 --- a/internal/fs/node_aix.go +++ b/internal/fs/node_aix.go @@ -35,8 +35,8 @@ func nodeFillExtendedAttributes(_ *restic.Node, _ string, _ bool) error { return nil } -// IsListxattrPermissionError is a no-op on AIX. -func IsListxattrPermissionError(_ error) bool { +// isListxattrPermissionError is a no-op on AIX. +func isListxattrPermissionError(_ error) bool { return false } diff --git a/internal/fs/node_netbsd.go b/internal/fs/node_netbsd.go index c71e4bdf5..996125851 100644 --- a/internal/fs/node_netbsd.go +++ b/internal/fs/node_netbsd.go @@ -25,8 +25,8 @@ func nodeFillExtendedAttributes(_ *restic.Node, _ string, _ bool) error { return nil } -// IsListxattrPermissionError is a no-op on netbsd. -func IsListxattrPermissionError(_ error) bool { +// isListxattrPermissionError is a no-op on netbsd. +func isListxattrPermissionError(_ error) bool { return false } diff --git a/internal/fs/node_openbsd.go b/internal/fs/node_openbsd.go index f74f2ae00..62eb78618 100644 --- a/internal/fs/node_openbsd.go +++ b/internal/fs/node_openbsd.go @@ -25,8 +25,8 @@ func nodeFillExtendedAttributes(_ *restic.Node, _ string, _ bool) error { return nil } -// IsListxattrPermissionError is a no-op on openbsd. -func IsListxattrPermissionError(_ error) bool { +// isListxattrPermissionError is a no-op on openbsd. +func isListxattrPermissionError(_ error) bool { return false } diff --git a/internal/fs/node_xattr.go b/internal/fs/node_xattr.go index 11bdf382b..55376ba58 100644 --- a/internal/fs/node_xattr.go +++ b/internal/fs/node_xattr.go @@ -28,7 +28,7 @@ func listxattr(path string) ([]string, error) { return l, handleXattrErr(err) } -func IsListxattrPermissionError(err error) bool { +func isListxattrPermissionError(err error) bool { var xerr *xattr.Error if errors.As(err, &xerr) { return xerr.Op == "xattr.list" && errors.Is(xerr.Err, os.ErrPermission) @@ -106,7 +106,7 @@ func nodeFillExtendedAttributes(node *restic.Node, path string, ignoreListError xattrs, err := listxattr(path) debug.Log("fillExtendedAttributes(%v) %v %v", path, xattrs, err) if err != nil { - if ignoreListError && IsListxattrPermissionError(err) { + if ignoreListError && isListxattrPermissionError(err) { return nil } return err diff --git a/internal/fs/node_xattr_test.go b/internal/fs/node_xattr_test.go index d948e3b31..3784dba45 100644 --- a/internal/fs/node_xattr_test.go +++ b/internal/fs/node_xattr_test.go @@ -19,10 +19,10 @@ func TestIsListxattrPermissionError(t *testing.T) { } err := handleXattrErr(xerr) rtest.Assert(t, err != nil, "missing error") - rtest.Assert(t, IsListxattrPermissionError(err), "expected IsListxattrPermissionError to return true for %v", err) + rtest.Assert(t, isListxattrPermissionError(err), "expected IsListxattrPermissionError to return true for %v", err) xerr.Err = os.ErrNotExist err = handleXattrErr(xerr) rtest.Assert(t, err != nil, "missing error") - rtest.Assert(t, !IsListxattrPermissionError(err), "expected IsListxattrPermissionError to return false for %v", err) + rtest.Assert(t, !isListxattrPermissionError(err), "expected IsListxattrPermissionError to return false for %v", err) } From 507842b614bb80c07a37872c304274d5b2378ea8 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 21 Jul 2024 16:30:49 +0200 Subject: [PATCH 8/8] fs: remove Open method from FS interface --- internal/archiver/archiver_test.go | 17 ----------- internal/archiver/file_saver_test.go | 2 +- internal/fs/fs_local.go | 10 ------- internal/fs/fs_local_vss.go | 5 ---- internal/fs/fs_reader.go | 45 ++++++++++------------------ internal/fs/fs_reader_test.go | 35 +++------------------- internal/fs/fs_track.go | 10 ------- internal/fs/interface.go | 1 - 8 files changed, 21 insertions(+), 104 deletions(-) diff --git a/internal/archiver/archiver_test.go b/internal/archiver/archiver_test.go index 18151eb02..d67b5b06a 100644 --- a/internal/archiver/archiver_test.go +++ b/internal/archiver/archiver_test.go @@ -1664,15 +1664,6 @@ type MockFS struct { bytesRead map[string]int // tracks bytes read from all opened files } -func (m *MockFS) Open(name string) (fs.File, error) { - f, err := m.FS.Open(name) - if err != nil { - return f, err - } - - return MockFile{File: f, fs: m, filename: name}, nil -} - func (m *MockFS) OpenFile(name string, flag int, perm os.FileMode) (fs.File, error) { f, err := m.FS.OpenFile(name, flag, perm) if err != nil { @@ -2061,14 +2052,6 @@ type TrackFS struct { m sync.Mutex } -func (m *TrackFS) Open(name string) (fs.File, error) { - m.m.Lock() - m.opened[name]++ - m.m.Unlock() - - return m.FS.Open(name) -} - func (m *TrackFS) OpenFile(name string, flag int, perm os.FileMode) (fs.File, error) { m.m.Lock() m.opened[name]++ diff --git a/internal/archiver/file_saver_test.go b/internal/archiver/file_saver_test.go index 4a4327572..948d7ce3c 100644 --- a/internal/archiver/file_saver_test.go +++ b/internal/archiver/file_saver_test.go @@ -72,7 +72,7 @@ func TestFileSaver(t *testing.T) { var results []FutureNode for _, filename := range files { - f, err := testFs.Open(filename) + f, err := testFs.OpenFile(filename, os.O_RDONLY, 0) if err != nil { t.Fatal(err) } diff --git a/internal/fs/fs_local.go b/internal/fs/fs_local.go index 48c40dc90..0bcbf7f3a 100644 --- a/internal/fs/fs_local.go +++ b/internal/fs/fs_local.go @@ -18,16 +18,6 @@ func (fs Local) VolumeName(path string) string { return filepath.VolumeName(path) } -// Open opens a file for reading. -func (fs Local) Open(name string) (File, error) { - f, err := os.Open(fixpath(name)) - if err != nil { - return nil, err - } - _ = setFlags(f) - return f, nil -} - // OpenFile is the generalized open call; most users will use Open // or Create instead. It opens the named file with specified flag // (O_RDONLY etc.) and perm, (0666 etc.) if applicable. If successful, diff --git a/internal/fs/fs_local_vss.go b/internal/fs/fs_local_vss.go index 718dfc46d..46b40d013 100644 --- a/internal/fs/fs_local_vss.go +++ b/internal/fs/fs_local_vss.go @@ -125,11 +125,6 @@ func (fs *LocalVss) DeleteSnapshots() { fs.snapshots = activeSnapshots } -// Open wraps the Open method of the underlying file system. -func (fs *LocalVss) Open(name string) (File, error) { - return os.Open(fs.snapshotPath(name)) -} - // OpenFile wraps the Open method of the underlying file system. func (fs *LocalVss) OpenFile(name string, flag int, perm os.FileMode) (File, error) { return os.OpenFile(fs.snapshotPath(name), flag, perm) diff --git a/internal/fs/fs_reader.go b/internal/fs/fs_reader.go index 47af74245..93a42f9eb 100644 --- a/internal/fs/fs_reader.go +++ b/internal/fs/fs_reader.go @@ -39,29 +39,6 @@ func (fs *Reader) VolumeName(_ string) string { return "" } -// Open opens a file for reading. -func (fs *Reader) Open(name string) (f File, err error) { - switch name { - case fs.Name: - fs.open.Do(func() { - f = newReaderFile(fs.ReadCloser, fs.fi(), fs.AllowEmptyFile) - }) - - if f == nil { - return nil, pathError("open", name, syscall.EIO) - } - - return f, nil - case "/", ".": - f = fakeDir{ - entries: []os.FileInfo{fs.fi()}, - } - return f, nil - } - - return nil, pathError("open", name, syscall.ENOENT) -} - func (fs *Reader) fi() os.FileInfo { return fakeFileInfo{ name: fs.Name, @@ -82,15 +59,25 @@ func (fs *Reader) OpenFile(name string, flag int, _ os.FileMode) (f File, err er fmt.Errorf("invalid combination of flags 0x%x", flag)) } - fs.open.Do(func() { - f = newReaderFile(fs.ReadCloser, fs.fi(), fs.AllowEmptyFile) - }) + switch name { + case fs.Name: + fs.open.Do(func() { + f = newReaderFile(fs.ReadCloser, fs.fi(), fs.AllowEmptyFile) + }) - if f == nil { - return nil, pathError("open", name, syscall.EIO) + if f == nil { + return nil, pathError("open", name, syscall.EIO) + } + + return f, nil + case "/", ".": + f = fakeDir{ + entries: []os.FileInfo{fs.fi()}, + } + return f, nil } - return f, nil + return nil, pathError("open", name, syscall.ENOENT) } // Stat returns a FileInfo describing the named file. If there is an error, it diff --git a/internal/fs/fs_reader_test.go b/internal/fs/fs_reader_test.go index d3ef5608a..9fa67b5ac 100644 --- a/internal/fs/fs_reader_test.go +++ b/internal/fs/fs_reader_test.go @@ -15,27 +15,6 @@ import ( "github.com/restic/restic/internal/test" ) -func verifyFileContentOpen(t testing.TB, fs FS, filename string, want []byte) { - f, err := fs.Open(filename) - if err != nil { - t.Fatal(err) - } - - buf, err := io.ReadAll(f) - if err != nil { - t.Fatal(err) - } - - err = f.Close() - if err != nil { - t.Fatal(err) - } - - if !cmp.Equal(want, buf) { - t.Error(cmp.Diff(want, buf)) - } -} - func verifyFileContentOpenFile(t testing.TB, fs FS, filename string, want []byte) { f, err := fs.OpenFile(filename, O_RDONLY, 0) if err != nil { @@ -58,7 +37,7 @@ func verifyFileContentOpenFile(t testing.TB, fs FS, filename string, want []byte } func verifyDirectoryContents(t testing.TB, fs FS, dir string, want []string) { - f, err := fs.Open(dir) + f, err := fs.OpenFile(dir, os.O_RDONLY, 0) if err != nil { t.Fatal(err) } @@ -96,7 +75,7 @@ func (s fiSlice) Swap(i, j int) { } func verifyDirectoryContentsFI(t testing.TB, fs FS, dir string, want []os.FileInfo) { - f, err := fs.Open(dir) + f, err := fs.OpenFile(dir, os.O_RDONLY, 0) if err != nil { t.Fatal(err) } @@ -219,12 +198,6 @@ func TestFSReader(t *testing.T) { verifyDirectoryContentsFI(t, fs, ".", []os.FileInfo{fi}) }, }, - { - name: "file/Open", - f: func(t *testing.T, fs FS) { - verifyFileContentOpen(t, fs, filename, data) - }, - }, { name: "file/OpenFile", f: func(t *testing.T, fs FS) { @@ -245,7 +218,7 @@ func TestFSReader(t *testing.T) { { name: "file/Stat", f: func(t *testing.T, fs FS) { - f, err := fs.Open(filename) + f, err := fs.OpenFile(filename, os.O_RDONLY, 0) if err != nil { t.Fatal(err) } @@ -417,7 +390,7 @@ func TestFSReaderMinFileSize(t *testing.T) { AllowEmptyFile: test.allowEmpty, } - f, err := fs.Open("testfile") + f, err := fs.OpenFile("testfile", os.O_RDONLY, 0) if err != nil { t.Fatal(err) } diff --git a/internal/fs/fs_track.go b/internal/fs/fs_track.go index 0c65a8564..366bbee76 100644 --- a/internal/fs/fs_track.go +++ b/internal/fs/fs_track.go @@ -15,16 +15,6 @@ type Track struct { FS } -// Open wraps the Open method of the underlying file system. -func (fs Track) Open(name string) (File, error) { - f, err := fs.FS.Open(fixpath(name)) - if err != nil { - return nil, err - } - - return newTrackFile(debug.Stack(), name, f), nil -} - // OpenFile wraps the OpenFile method of the underlying file system. func (fs Track) OpenFile(name string, flag int, perm os.FileMode) (File, error) { f, err := fs.FS.OpenFile(fixpath(name), flag, perm) diff --git a/internal/fs/interface.go b/internal/fs/interface.go index b26c56944..e1f4ef2d9 100644 --- a/internal/fs/interface.go +++ b/internal/fs/interface.go @@ -7,7 +7,6 @@ import ( // FS bundles all methods needed for a file system. type FS interface { - Open(name string) (File, error) OpenFile(name string, flag int, perm os.FileMode) (File, error) Stat(name string) (os.FileInfo, error) Lstat(name string) (os.FileInfo, error)