2
2
mirror of https://github.com/octoleo/restic.git synced 2024-05-28 22:50:48 +00:00
restic/internal/restic/node.go

883 lines
24 KiB
Go
Raw Normal View History

2015-02-03 20:07:55 +00:00
package restic
import (
2017-06-04 09:16:55 +00:00
"context"
2015-02-03 20:07:55 +00:00
"encoding/json"
"fmt"
"os"
2015-04-29 03:45:24 +00:00
"os/user"
"reflect"
2015-02-03 20:07:55 +00:00
"strconv"
"strings"
2015-05-01 21:26:40 +00:00
"sync"
2015-02-03 20:07:55 +00:00
"syscall"
"time"
"unicode/utf8"
2015-02-03 20:07:55 +00:00
2017-07-23 12:21:03 +00:00
"github.com/restic/restic/internal/errors"
"bytes"
2017-07-23 12:21:03 +00:00
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/fs"
2015-02-03 20:07:55 +00:00
)
// ExtendedAttribute is a tuple storing the xattr name and value for various filesystems.
type ExtendedAttribute struct {
Name string `json:"name"`
Value []byte `json:"value"`
}
// GenericAttributeType can be used for OS specific functionalities by defining specific types
// in node.go to be used by the specific node_xx files.
// OS specific attribute types should follow the convention <OS>Attributes.
// GenericAttributeTypes should follow the convention <OS specific attribute type>.<attribute name>
// The attributes in OS specific attribute types must be pointers as we want to distinguish nil values
// and not create GenericAttributes for them.
type GenericAttributeType string
// OSType is the type created to represent each specific OS
type OSType string
const (
// When new GenericAttributeType are defined, they must be added in the init function as well.
// Below are windows specific attributes.
// TypeCreationTime is the GenericAttributeType used for storing creation time for windows files within the generic attributes map.
TypeCreationTime GenericAttributeType = "windows.creation_time"
// TypeFileAttributes is the GenericAttributeType used for storing file attributes for windows files within the generic attributes map.
TypeFileAttributes GenericAttributeType = "windows.file_attributes"
// Generic Attributes for other OS types should be defined here.
)
// init is called when the package is initialized. Any new GenericAttributeTypes being created must be added here as well.
func init() {
storeGenericAttributeType(TypeCreationTime, TypeFileAttributes)
}
// genericAttributesForOS maintains a map of known genericAttributesForOS to the OSType
var genericAttributesForOS = map[GenericAttributeType]OSType{}
// storeGenericAttributeType adds and entry in genericAttributesForOS map
func storeGenericAttributeType(attributeTypes ...GenericAttributeType) {
for _, attributeType := range attributeTypes {
// Get the OS attribute type from the GenericAttributeType
osAttributeName := strings.Split(string(attributeType), ".")[0]
genericAttributesForOS[attributeType] = OSType(osAttributeName)
}
}
2015-05-02 14:12:19 +00:00
// Node is a file, directory or other item in a backup.
2015-02-03 20:07:55 +00:00
type Node struct {
Name string `json:"name"`
Type string `json:"type"`
Mode os.FileMode `json:"mode,omitempty"`
ModTime time.Time `json:"mtime,omitempty"`
AccessTime time.Time `json:"atime,omitempty"`
ChangeTime time.Time `json:"ctime,omitempty"`
UID uint32 `json:"uid"`
GID uint32 `json:"gid"`
User string `json:"user,omitempty"`
Group string `json:"group,omitempty"`
Inode uint64 `json:"inode,omitempty"`
DeviceID uint64 `json:"device_id,omitempty"` // device id of the file, stat.st_dev
Size uint64 `json:"size,omitempty"`
Links uint64 `json:"links,omitempty"`
LinkTarget string `json:"linktarget,omitempty"`
// implicitly base64-encoded field. Only used while encoding, `linktarget_raw` will overwrite LinkTarget if present.
// This allows storing arbitrary byte-sequences, which are possible as symlink targets on unix systems,
// as LinkTarget without breaking backwards-compatibility.
// Must only be set of the linktarget cannot be encoded as valid utf8.
LinkTargetRaw []byte `json:"linktarget_raw,omitempty"`
ExtendedAttributes []ExtendedAttribute `json:"extended_attributes,omitempty"`
GenericAttributes map[GenericAttributeType]json.RawMessage `json:"generic_attributes,omitempty"`
Device uint64 `json:"device,omitempty"` // in case of Type == "dev", stat.st_rdev
Content IDs `json:"content"`
Subtree *ID `json:"subtree,omitempty"`
2015-02-03 20:07:55 +00:00
Error string `json:"error,omitempty"`
2016-08-31 20:39:36 +00:00
Path string `json:"-"`
2015-02-03 20:07:55 +00:00
}
2017-06-10 20:55:12 +00:00
// Nodes is a slice of nodes that can be sorted.
type Nodes []*Node
func (n Nodes) Len() int { return len(n) }
func (n Nodes) Less(i, j int) bool { return n[i].Name < n[j].Name }
func (n Nodes) Swap(i, j int) { n[i], n[j] = n[j], n[i] }
2015-04-28 06:11:01 +00:00
func (node Node) String() string {
2017-12-03 16:34:37 +00:00
var mode os.FileMode
2016-09-01 19:20:03 +00:00
switch node.Type {
2015-02-03 20:07:55 +00:00
case "file":
2017-12-03 16:34:37 +00:00
mode = 0
2015-02-03 20:07:55 +00:00
case "dir":
2017-12-03 16:34:37 +00:00
mode = os.ModeDir
case "symlink":
mode = os.ModeSymlink
case "dev":
mode = os.ModeDevice
case "chardev":
mode = os.ModeDevice | os.ModeCharDevice
case "fifo":
mode = os.ModeNamedPipe
case "socket":
mode = os.ModeSocket
2015-02-03 20:07:55 +00:00
}
2017-12-03 16:34:37 +00:00
return fmt.Sprintf("%s %5d %5d %6d %s %s",
mode|node.Mode, node.UID, node.GID, node.Size, node.ModTime, node.Name)
2015-02-03 20:07:55 +00:00
}
// NodeFromFileInfo returns a new node from the given path and FileInfo. It
// returns the first error that is encountered, together with a node.
2015-02-03 20:07:55 +00:00
func NodeFromFileInfo(path string, fi os.FileInfo) (*Node, error) {
mask := os.ModePerm | os.ModeType | os.ModeSetuid | os.ModeSetgid | os.ModeSticky
Remove pools for nodes and IDs This removes the allocation pools for nodes and IDs. I feel they aren't really needed. Benchmarks: benchmark old ns/op new ns/op delta BenchmarkChunkEncrypt 197890867 198616293 +0.37% BenchmarkChunkEncryptParallel 196127004 198819818 +1.37% BenchmarkArchiveDirectory 1098848419 1087237723 -1.06% BenchmarkPreload 30464455 29910239 -1.82% BenchmarkLoadTree 3265092 3088543 -5.41% BenchmarkEncryptWriter 37213511 37134683 -0.21% BenchmarkEncrypt 36037879 36166546 +0.36% BenchmarkDecryptReader 38165659 38556734 +1.02% BenchmarkEncryptDecryptReader 77027044 77194987 +0.22% BenchmarkDecrypt 36017602 35937888 -0.22% BenchmarkSaveJSON 47906 50270 +4.93% BenchmarkSaveFrom 49775973 50520969 +1.50% BenchmarkLoadJSONID 105290245 107281849 +1.89% BenchmarkChunkerWithSHA256 151501430 148264078 -2.14% BenchmarkChunkerWithMD5 93606346 94036392 +0.46% BenchmarkChunker 74285431 75933882 +2.22% BenchmarkPipelineWalker 387689 346467 -10.63% benchmark old MB/s new MB/s speedup BenchmarkChunkEncrypt 52.99 52.79 1.00x BenchmarkChunkEncryptParallel 53.46 52.74 0.99x BenchmarkEncryptWriter 225.42 225.90 1.00x BenchmarkEncrypt 232.77 231.94 1.00x BenchmarkDecryptReader 219.79 217.57 0.99x BenchmarkEncryptDecryptReader 108.90 108.67 1.00x BenchmarkDecrypt 232.90 233.42 1.00x BenchmarkSaveFrom 84.26 83.02 0.99x BenchmarkChunkerWithSHA256 69.21 70.72 1.02x BenchmarkChunkerWithMD5 112.02 111.51 1.00x BenchmarkChunker 141.15 138.09 0.98x benchmark old allocs new allocs delta BenchmarkChunkEncrypt 110 110 +0.00% BenchmarkChunkEncryptParallel 100 100 +0.00% BenchmarkArchiveDirectory 475591 476635 +0.22% BenchmarkPreload 28059 24182 -13.82% BenchmarkLoadTree 3124 2889 -7.52% BenchmarkEncryptWriter 19 19 +0.00% BenchmarkEncrypt 13 13 +0.00% BenchmarkDecryptReader 16 15 -6.25% BenchmarkEncryptDecryptReader 39 39 +0.00% BenchmarkDecrypt 11 11 +0.00% BenchmarkSaveJSON 74 74 +0.00% BenchmarkSaveFrom 109 112 +2.75% BenchmarkLoadJSONID 103630 97849 -5.58% BenchmarkChunkerWithSHA256 13 13 +0.00% BenchmarkChunkerWithMD5 12 12 +0.00% BenchmarkChunker 6 6 +0.00% BenchmarkPipelineWalker 212 165 -22.17% benchmark old bytes new bytes delta BenchmarkChunkEncrypt 64697 64697 +0.00% BenchmarkChunkEncryptParallel 64681 64681 +0.00% BenchmarkArchiveDirectory 193385504 193790864 +0.21% BenchmarkPreload 4064701 3942000 -3.02% BenchmarkLoadTree 344954 325396 -5.67% BenchmarkEncryptWriter 12793 12793 +0.00% BenchmarkEncrypt 1950 1950 +0.00% BenchmarkDecryptReader 3120 2774 -11.09% BenchmarkEncryptDecryptReader 1528036 1528036 +0.00% BenchmarkDecrypt 1919 1919 +0.00% BenchmarkSaveJSON 5524 5524 +0.00% BenchmarkSaveFrom 31353 40804 +30.14% BenchmarkLoadJSONID 12872020 16010968 +24.39% BenchmarkChunkerWithSHA256 26821 26821 +0.00% BenchmarkChunkerWithMD5 13554 13554 +0.00% BenchmarkChunker 13458 13458 +0.00% BenchmarkPipelineWalker 58584 55560 -5.16%
2015-03-22 15:38:03 +00:00
node := &Node{
2016-08-31 20:39:36 +00:00
Path: path,
Remove pools for nodes and IDs This removes the allocation pools for nodes and IDs. I feel they aren't really needed. Benchmarks: benchmark old ns/op new ns/op delta BenchmarkChunkEncrypt 197890867 198616293 +0.37% BenchmarkChunkEncryptParallel 196127004 198819818 +1.37% BenchmarkArchiveDirectory 1098848419 1087237723 -1.06% BenchmarkPreload 30464455 29910239 -1.82% BenchmarkLoadTree 3265092 3088543 -5.41% BenchmarkEncryptWriter 37213511 37134683 -0.21% BenchmarkEncrypt 36037879 36166546 +0.36% BenchmarkDecryptReader 38165659 38556734 +1.02% BenchmarkEncryptDecryptReader 77027044 77194987 +0.22% BenchmarkDecrypt 36017602 35937888 -0.22% BenchmarkSaveJSON 47906 50270 +4.93% BenchmarkSaveFrom 49775973 50520969 +1.50% BenchmarkLoadJSONID 105290245 107281849 +1.89% BenchmarkChunkerWithSHA256 151501430 148264078 -2.14% BenchmarkChunkerWithMD5 93606346 94036392 +0.46% BenchmarkChunker 74285431 75933882 +2.22% BenchmarkPipelineWalker 387689 346467 -10.63% benchmark old MB/s new MB/s speedup BenchmarkChunkEncrypt 52.99 52.79 1.00x BenchmarkChunkEncryptParallel 53.46 52.74 0.99x BenchmarkEncryptWriter 225.42 225.90 1.00x BenchmarkEncrypt 232.77 231.94 1.00x BenchmarkDecryptReader 219.79 217.57 0.99x BenchmarkEncryptDecryptReader 108.90 108.67 1.00x BenchmarkDecrypt 232.90 233.42 1.00x BenchmarkSaveFrom 84.26 83.02 0.99x BenchmarkChunkerWithSHA256 69.21 70.72 1.02x BenchmarkChunkerWithMD5 112.02 111.51 1.00x BenchmarkChunker 141.15 138.09 0.98x benchmark old allocs new allocs delta BenchmarkChunkEncrypt 110 110 +0.00% BenchmarkChunkEncryptParallel 100 100 +0.00% BenchmarkArchiveDirectory 475591 476635 +0.22% BenchmarkPreload 28059 24182 -13.82% BenchmarkLoadTree 3124 2889 -7.52% BenchmarkEncryptWriter 19 19 +0.00% BenchmarkEncrypt 13 13 +0.00% BenchmarkDecryptReader 16 15 -6.25% BenchmarkEncryptDecryptReader 39 39 +0.00% BenchmarkDecrypt 11 11 +0.00% BenchmarkSaveJSON 74 74 +0.00% BenchmarkSaveFrom 109 112 +2.75% BenchmarkLoadJSONID 103630 97849 -5.58% BenchmarkChunkerWithSHA256 13 13 +0.00% BenchmarkChunkerWithMD5 12 12 +0.00% BenchmarkChunker 6 6 +0.00% BenchmarkPipelineWalker 212 165 -22.17% benchmark old bytes new bytes delta BenchmarkChunkEncrypt 64697 64697 +0.00% BenchmarkChunkEncryptParallel 64681 64681 +0.00% BenchmarkArchiveDirectory 193385504 193790864 +0.21% BenchmarkPreload 4064701 3942000 -3.02% BenchmarkLoadTree 344954 325396 -5.67% BenchmarkEncryptWriter 12793 12793 +0.00% BenchmarkEncrypt 1950 1950 +0.00% BenchmarkDecryptReader 3120 2774 -11.09% BenchmarkEncryptDecryptReader 1528036 1528036 +0.00% BenchmarkDecrypt 1919 1919 +0.00% BenchmarkSaveJSON 5524 5524 +0.00% BenchmarkSaveFrom 31353 40804 +30.14% BenchmarkLoadJSONID 12872020 16010968 +24.39% BenchmarkChunkerWithSHA256 26821 26821 +0.00% BenchmarkChunkerWithMD5 13554 13554 +0.00% BenchmarkChunker 13458 13458 +0.00% BenchmarkPipelineWalker 58584 55560 -5.16%
2015-03-22 15:38:03 +00:00
Name: fi.Name(),
Mode: fi.Mode() & mask,
Remove pools for nodes and IDs This removes the allocation pools for nodes and IDs. I feel they aren't really needed. Benchmarks: benchmark old ns/op new ns/op delta BenchmarkChunkEncrypt 197890867 198616293 +0.37% BenchmarkChunkEncryptParallel 196127004 198819818 +1.37% BenchmarkArchiveDirectory 1098848419 1087237723 -1.06% BenchmarkPreload 30464455 29910239 -1.82% BenchmarkLoadTree 3265092 3088543 -5.41% BenchmarkEncryptWriter 37213511 37134683 -0.21% BenchmarkEncrypt 36037879 36166546 +0.36% BenchmarkDecryptReader 38165659 38556734 +1.02% BenchmarkEncryptDecryptReader 77027044 77194987 +0.22% BenchmarkDecrypt 36017602 35937888 -0.22% BenchmarkSaveJSON 47906 50270 +4.93% BenchmarkSaveFrom 49775973 50520969 +1.50% BenchmarkLoadJSONID 105290245 107281849 +1.89% BenchmarkChunkerWithSHA256 151501430 148264078 -2.14% BenchmarkChunkerWithMD5 93606346 94036392 +0.46% BenchmarkChunker 74285431 75933882 +2.22% BenchmarkPipelineWalker 387689 346467 -10.63% benchmark old MB/s new MB/s speedup BenchmarkChunkEncrypt 52.99 52.79 1.00x BenchmarkChunkEncryptParallel 53.46 52.74 0.99x BenchmarkEncryptWriter 225.42 225.90 1.00x BenchmarkEncrypt 232.77 231.94 1.00x BenchmarkDecryptReader 219.79 217.57 0.99x BenchmarkEncryptDecryptReader 108.90 108.67 1.00x BenchmarkDecrypt 232.90 233.42 1.00x BenchmarkSaveFrom 84.26 83.02 0.99x BenchmarkChunkerWithSHA256 69.21 70.72 1.02x BenchmarkChunkerWithMD5 112.02 111.51 1.00x BenchmarkChunker 141.15 138.09 0.98x benchmark old allocs new allocs delta BenchmarkChunkEncrypt 110 110 +0.00% BenchmarkChunkEncryptParallel 100 100 +0.00% BenchmarkArchiveDirectory 475591 476635 +0.22% BenchmarkPreload 28059 24182 -13.82% BenchmarkLoadTree 3124 2889 -7.52% BenchmarkEncryptWriter 19 19 +0.00% BenchmarkEncrypt 13 13 +0.00% BenchmarkDecryptReader 16 15 -6.25% BenchmarkEncryptDecryptReader 39 39 +0.00% BenchmarkDecrypt 11 11 +0.00% BenchmarkSaveJSON 74 74 +0.00% BenchmarkSaveFrom 109 112 +2.75% BenchmarkLoadJSONID 103630 97849 -5.58% BenchmarkChunkerWithSHA256 13 13 +0.00% BenchmarkChunkerWithMD5 12 12 +0.00% BenchmarkChunker 6 6 +0.00% BenchmarkPipelineWalker 212 165 -22.17% benchmark old bytes new bytes delta BenchmarkChunkEncrypt 64697 64697 +0.00% BenchmarkChunkEncryptParallel 64681 64681 +0.00% BenchmarkArchiveDirectory 193385504 193790864 +0.21% BenchmarkPreload 4064701 3942000 -3.02% BenchmarkLoadTree 344954 325396 -5.67% BenchmarkEncryptWriter 12793 12793 +0.00% BenchmarkEncrypt 1950 1950 +0.00% BenchmarkDecryptReader 3120 2774 -11.09% BenchmarkEncryptDecryptReader 1528036 1528036 +0.00% BenchmarkDecrypt 1919 1919 +0.00% BenchmarkSaveJSON 5524 5524 +0.00% BenchmarkSaveFrom 31353 40804 +30.14% BenchmarkLoadJSONID 12872020 16010968 +24.39% BenchmarkChunkerWithSHA256 26821 26821 +0.00% BenchmarkChunkerWithMD5 13554 13554 +0.00% BenchmarkChunker 13458 13458 +0.00% BenchmarkPipelineWalker 58584 55560 -5.16%
2015-03-22 15:38:03 +00:00
ModTime: fi.ModTime(),
}
2015-02-03 20:07:55 +00:00
2016-09-01 19:20:03 +00:00
node.Type = nodeTypeFromFileInfo(fi)
if node.Type == "file" {
2015-03-07 10:53:32 +00:00
node.Size = uint64(fi.Size())
}
2015-04-24 23:39:32 +00:00
err := node.fillExtra(path, fi)
2015-03-07 10:53:32 +00:00
return node, err
}
2015-04-29 01:24:43 +00:00
func nodeTypeFromFileInfo(fi os.FileInfo) string {
switch fi.Mode() & os.ModeType {
2015-02-03 20:07:55 +00:00
case 0:
2015-03-07 10:53:32 +00:00
return "file"
2015-02-03 20:07:55 +00:00
case os.ModeDir:
2015-03-07 10:53:32 +00:00
return "dir"
2015-02-03 20:07:55 +00:00
case os.ModeSymlink:
2015-03-07 10:53:32 +00:00
return "symlink"
2015-02-03 20:07:55 +00:00
case os.ModeDevice | os.ModeCharDevice:
2015-03-07 10:53:32 +00:00
return "chardev"
2015-02-03 20:07:55 +00:00
case os.ModeDevice:
2015-03-07 10:53:32 +00:00
return "dev"
2015-02-03 20:07:55 +00:00
case os.ModeNamedPipe:
2015-03-07 10:53:32 +00:00
return "fifo"
2015-02-03 20:07:55 +00:00
case os.ModeSocket:
2015-03-07 10:53:32 +00:00
return "socket"
case os.ModeIrregular:
return "irregular"
2015-02-03 20:07:55 +00:00
}
2015-03-07 10:53:32 +00:00
return ""
2015-02-03 20:07:55 +00:00
}
// GetExtendedAttribute gets the extended attribute.
func (node Node) GetExtendedAttribute(a string) []byte {
for _, attr := range node.ExtendedAttributes {
if attr.Name == a {
return attr.Value
}
}
return nil
}
// CreateAt creates the node at the given path but does NOT restore node meta data.
func (node *Node) CreateAt(ctx context.Context, path string, repo BlobLoader) error {
2016-09-27 20:35:08 +00:00
debug.Log("create node %v at %v", node.Name, path)
2016-09-01 19:20:03 +00:00
switch node.Type {
2015-02-03 20:07:55 +00:00
case "dir":
2015-04-29 01:24:43 +00:00
if err := node.createDirAt(path); err != nil {
2016-08-21 15:24:13 +00:00
return err
2015-02-03 20:07:55 +00:00
}
case "file":
if err := node.createFileAt(ctx, path, repo); err != nil {
2016-08-21 15:24:13 +00:00
return err
2015-02-03 20:07:55 +00:00
}
case "symlink":
2015-04-29 01:24:43 +00:00
if err := node.createSymlinkAt(path); err != nil {
2016-08-21 15:24:13 +00:00
return err
2015-02-03 20:07:55 +00:00
}
case "dev":
2015-04-29 01:24:43 +00:00
if err := node.createDevAt(path); err != nil {
2016-08-21 15:24:13 +00:00
return err
2015-02-03 20:07:55 +00:00
}
case "chardev":
2015-04-29 01:24:43 +00:00
if err := node.createCharDevAt(path); err != nil {
2016-08-21 15:24:13 +00:00
return err
2015-02-03 20:07:55 +00:00
}
case "fifo":
2015-04-29 01:24:43 +00:00
if err := node.createFifoAt(path); err != nil {
2016-08-21 15:24:13 +00:00
return err
2015-02-03 20:07:55 +00:00
}
case "socket":
return nil
default:
return errors.Errorf("filetype %q not implemented", node.Type)
2015-02-03 20:07:55 +00:00
}
return nil
}
// RestoreMetadata restores node metadata
func (node Node) RestoreMetadata(path string, warn func(msg string)) error {
err := node.restoreMetadata(path, warn)
if err != nil {
2016-09-27 20:35:08 +00:00
debug.Log("restoreMetadata(%s) error %v", path, err)
}
return err
2015-04-29 01:24:43 +00:00
}
func (node Node) restoreMetadata(path string, warn func(msg string)) error {
var firsterr error
2015-04-29 01:24:43 +00:00
if err := lchown(path, int(node.UID), int(node.GID)); err != nil {
// Like "cp -a" and "rsync -a" do, we only report lchown permission errors
// if we run as root.
if os.Geteuid() > 0 && os.IsPermission(err) {
debug.Log("not running as root, ignoring lchown permission error for %v: %v",
path, err)
} else {
firsterr = errors.WithStack(err)
}
2015-04-29 01:24:43 +00:00
}
if err := node.RestoreTimestamps(path); err != nil {
debug.Log("error restoring timestamps for dir %v: %v", path, err)
if firsterr != nil {
firsterr = err
2015-05-14 03:11:31 +00:00
}
}
if err := node.restoreExtendedAttributes(path); err != nil {
debug.Log("error restoring extended attributes for %v: %v", path, err)
if firsterr != nil {
firsterr = err
}
}
if err := node.restoreGenericAttributes(path, warn); err != nil {
debug.Log("error restoring generic attributes for %v: %v", path, err)
if firsterr != nil {
firsterr = err
}
}
// Moving RestoreTimestamps and restoreExtendedAttributes calls above as for readonly files in windows
// calling Chmod below will no longer allow any modifications to be made on the file and the
// calls above would fail.
if node.Type != "symlink" {
if err := fs.Chmod(path, node.Mode); err != nil {
if firsterr != nil {
firsterr = errors.WithStack(err)
}
}
}
return firsterr
}
func (node Node) restoreExtendedAttributes(path string) error {
for _, attr := range node.ExtendedAttributes {
err := Setxattr(path, attr.Name, attr.Value)
if err != nil {
return err
}
}
2015-05-14 03:11:31 +00:00
return nil
}
func (node Node) RestoreTimestamps(path string) error {
var utimes = [...]syscall.Timespec{
2015-04-29 01:24:43 +00:00
syscall.NsecToTimespec(node.AccessTime.UnixNano()),
syscall.NsecToTimespec(node.ModTime.UnixNano()),
}
2015-05-14 03:11:31 +00:00
2016-09-01 19:20:03 +00:00
if node.Type == "symlink" {
2016-08-29 19:38:34 +00:00
return node.restoreSymlinkTimestamps(path, utimes)
}
if err := syscall.UtimesNano(path, utimes[:]); err != nil {
2016-08-29 19:38:34 +00:00
return errors.Wrap(err, "UtimesNano")
2015-02-03 20:07:55 +00:00
}
2015-04-29 01:24:43 +00:00
return nil
}
func (node Node) createDirAt(path string) error {
err := fs.Mkdir(path, node.Mode)
if err != nil && !os.IsExist(err) {
return errors.WithStack(err)
2015-02-03 20:07:55 +00:00
}
return nil
}
func (node Node) createFileAt(ctx context.Context, path string, repo BlobLoader) error {
f, err := fs.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
2015-04-29 01:24:43 +00:00
if err != nil {
return errors.WithStack(err)
2015-02-03 20:07:55 +00:00
}
err = node.writeNodeContent(ctx, repo, f)
closeErr := f.Close()
if err != nil {
return err
}
if closeErr != nil {
return errors.WithStack(closeErr)
}
return nil
}
func (node Node) writeNodeContent(ctx context.Context, repo BlobLoader, f *os.File) error {
var buf []byte
for _, id := range node.Content {
buf, err := repo.LoadBlob(ctx, DataBlob, id, buf)
2015-04-29 01:24:43 +00:00
if err != nil {
2016-08-21 15:24:13 +00:00
return err
2015-04-29 01:24:43 +00:00
}
_, err = f.Write(buf)
if err != nil {
return errors.WithStack(err)
2015-04-29 01:24:43 +00:00
}
2015-02-03 20:07:55 +00:00
}
2015-04-29 01:24:43 +00:00
return nil
}
func (node Node) createSymlinkAt(path string) error {
if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
return errors.Wrap(err, "Symlink")
}
if err := fs.Symlink(node.LinkTarget, path); err != nil {
return errors.WithStack(err)
2015-02-03 20:07:55 +00:00
}
2015-04-29 01:24:43 +00:00
return nil
2015-02-03 20:07:55 +00:00
}
func (node *Node) createDevAt(path string) error {
return mknod(path, syscall.S_IFBLK|0600, node.Device)
}
func (node *Node) createCharDevAt(path string) error {
return mknod(path, syscall.S_IFCHR|0600, node.Device)
}
func (node *Node) createFifoAt(path string) error {
return mkfifo(path, 0600)
}
// FixTime returns a time.Time which can safely be used to marshal as JSON. If
2019-12-22 02:00:28 +00:00
// the timestamp is earlier than year zero, the year is set to zero. In the same
// way, if the year is larger than 9999, the year is set to 9999. Other than
// the year nothing is changed.
func FixTime(t time.Time) time.Time {
switch {
case t.Year() < 0000:
return t.AddDate(-t.Year(), 0, 0)
case t.Year() > 9999:
return t.AddDate(-(t.Year() - 9999), 0, 0)
default:
return t
}
}
func (node Node) MarshalJSON() ([]byte, error) {
// make sure invalid timestamps for mtime and atime are converted to
// something we can actually save.
node.ModTime = FixTime(node.ModTime)
node.AccessTime = FixTime(node.AccessTime)
node.ChangeTime = FixTime(node.ChangeTime)
2015-02-03 20:07:55 +00:00
type nodeJSON Node
nj := nodeJSON(node)
name := strconv.Quote(node.Name)
nj.Name = name[1 : len(name)-1]
if nj.LinkTargetRaw != nil {
panic("LinkTargetRaw must not be set manually")
}
if !utf8.ValidString(node.LinkTarget) {
// store raw bytes if invalid utf8
nj.LinkTargetRaw = []byte(node.LinkTarget)
}
2015-02-03 20:07:55 +00:00
return json.Marshal(nj)
}
func (node *Node) UnmarshalJSON(data []byte) error {
type nodeJSON Node
2015-04-28 06:11:01 +00:00
nj := (*nodeJSON)(node)
2015-02-03 20:07:55 +00:00
err := json.Unmarshal(data, nj)
if err != nil {
2016-08-29 19:38:34 +00:00
return errors.Wrap(err, "Unmarshal")
2015-02-03 20:07:55 +00:00
}
nj.Name, err = strconv.Unquote(`"` + nj.Name + `"`)
if err != nil {
return errors.Wrap(err, "Unquote")
}
if nj.LinkTargetRaw != nil {
nj.LinkTarget = string(nj.LinkTargetRaw)
nj.LinkTargetRaw = nil
}
return nil
2015-02-03 20:07:55 +00:00
}
func (node Node) Equals(other Node) bool {
if node.Name != other.Name {
return false
}
2016-09-01 19:20:03 +00:00
if node.Type != other.Type {
2015-02-03 20:07:55 +00:00
return false
}
if node.Mode != other.Mode {
return false
}
2017-11-16 20:51:12 +00:00
if !node.ModTime.Equal(other.ModTime) {
2015-02-03 20:07:55 +00:00
return false
}
2017-11-16 20:51:12 +00:00
if !node.AccessTime.Equal(other.AccessTime) {
2015-02-03 20:07:55 +00:00
return false
}
2017-11-16 20:51:12 +00:00
if !node.ChangeTime.Equal(other.ChangeTime) {
2015-02-03 20:07:55 +00:00
return false
}
if node.UID != other.UID {
return false
}
if node.GID != other.GID {
return false
}
if node.User != other.User {
return false
}
if node.Group != other.Group {
return false
}
if node.Inode != other.Inode {
return false
}
2017-04-05 18:42:15 +00:00
if node.DeviceID != other.DeviceID {
return false
}
2015-02-03 20:07:55 +00:00
if node.Size != other.Size {
return false
}
if node.Links != other.Links {
return false
}
if node.LinkTarget != other.LinkTarget {
return false
}
if node.Device != other.Device {
return false
}
2015-04-29 01:24:43 +00:00
if !node.sameContent(other) {
2015-02-03 20:07:55 +00:00
return false
2015-04-29 01:24:43 +00:00
}
if !node.sameExtendedAttributes(other) {
return false
}
if !node.sameGenericAttributes(other) {
return false
}
if node.Subtree != nil {
if other.Subtree == nil {
return false
}
if !node.Subtree.Equal(*other.Subtree) {
return false
}
} else {
if other.Subtree != nil {
return false
}
2015-04-29 01:24:43 +00:00
}
if node.Error != other.Error {
return false
}
2015-02-03 20:07:55 +00:00
2015-04-29 01:24:43 +00:00
return true
}
func (node Node) sameContent(other Node) bool {
if node.Content == nil {
return other.Content == nil
2015-02-03 20:07:55 +00:00
}
2015-04-29 01:24:43 +00:00
if other.Content == nil {
2015-02-03 20:07:55 +00:00
return false
}
2015-04-29 01:24:43 +00:00
if len(node.Content) != len(other.Content) {
2015-02-03 20:07:55 +00:00
return false
}
2015-04-29 01:24:43 +00:00
for i := 0; i < len(node.Content); i++ {
if !node.Content[i].Equal(other.Content[i]) {
return false
}
}
return true
}
func (node Node) sameExtendedAttributes(other Node) bool {
ln := len(node.ExtendedAttributes)
lo := len(other.ExtendedAttributes)
if ln != lo {
return false
} else if ln == 0 {
// This means lo is also of length 0
return true
}
// build a set of all attributes that node has
type mapvalue struct {
value []byte
present bool
}
attributes := make(map[string]mapvalue)
for _, attr := range node.ExtendedAttributes {
attributes[attr.Name] = mapvalue{value: attr.Value}
}
for _, attr := range other.ExtendedAttributes {
v, ok := attributes[attr.Name]
if !ok {
// extended attribute is not set for node
debug.Log("other node has attribute %v, which is not present in node", attr.Name)
return false
}
if !bytes.Equal(v.value, attr.Value) {
// attribute has different value
debug.Log("attribute %v has different value", attr.Name)
return false
}
// remember that this attribute is present in other.
v.present = true
attributes[attr.Name] = v
}
// check for attributes that are not present in other
for name, v := range attributes {
if !v.present {
debug.Log("attribute %v not present in other node", name)
return false
}
}
2015-04-29 01:24:43 +00:00
2015-02-03 20:07:55 +00:00
return true
}
2015-04-29 03:31:07 +00:00
func (node Node) sameGenericAttributes(other Node) bool {
return deepEqual(node.GenericAttributes, other.GenericAttributes)
}
func deepEqual(map1, map2 map[GenericAttributeType]json.RawMessage) bool {
// Check if the maps have the same number of keys
if len(map1) != len(map2) {
return false
}
// Iterate over each key-value pair in map1
for key, value1 := range map1 {
// Check if the key exists in map2
value2, ok := map2[key]
if !ok {
return false
}
// Check if the JSON.RawMessage values are equal byte by byte
if !bytes.Equal(value1, value2) {
return false
}
}
return true
}
func (node *Node) fillUser(stat *statT) {
uid, gid := stat.uid(), stat.gid()
node.UID, node.GID = uid, gid
node.User = lookupUsername(uid)
node.Group = lookupGroup(gid)
2015-04-29 03:45:24 +00:00
}
2015-05-01 21:26:40 +00:00
var (
uidLookupCache = make(map[uint32]string)
2015-05-01 21:26:40 +00:00
uidLookupCacheMutex = sync.RWMutex{}
)
// Cached user name lookup by uid. Returns "" when no name can be found.
func lookupUsername(uid uint32) string {
2015-05-01 21:26:40 +00:00
uidLookupCacheMutex.RLock()
username, ok := uidLookupCache[uid]
2015-05-01 21:26:40 +00:00
uidLookupCacheMutex.RUnlock()
if ok {
return username
2015-05-01 21:26:40 +00:00
}
u, err := user.LookupId(strconv.Itoa(int(uid)))
if err == nil {
username = u.Username
2015-05-01 21:26:40 +00:00
}
uidLookupCacheMutex.Lock()
uidLookupCache[uid] = username
2015-05-01 21:26:40 +00:00
uidLookupCacheMutex.Unlock()
return username
2015-05-01 21:26:40 +00:00
}
var (
gidLookupCache = make(map[uint32]string)
gidLookupCacheMutex = sync.RWMutex{}
)
// Cached group name lookup by gid. Returns "" when no name can be found.
func lookupGroup(gid uint32) string {
gidLookupCacheMutex.RLock()
group, ok := gidLookupCache[gid]
gidLookupCacheMutex.RUnlock()
if ok {
return group
}
g, err := user.LookupGroupId(strconv.Itoa(int(gid)))
if err == nil {
group = g.Name
}
gidLookupCacheMutex.Lock()
gidLookupCache[gid] = group
gidLookupCacheMutex.Unlock()
return group
}
2015-04-29 03:45:24 +00:00
func (node *Node) fillExtra(path string, fi os.FileInfo) error {
stat, ok := toStatT(fi.Sys())
2015-04-29 03:45:24 +00:00
if !ok {
2018-03-31 11:22:36 +00:00
// fill minimal info with current values for uid, gid
node.UID = uint32(os.Getuid())
node.GID = uint32(os.Getgid())
node.ChangeTime = node.ModTime
2015-04-29 03:45:24 +00:00
return nil
}
node.Inode = uint64(stat.ino())
2017-04-05 18:45:24 +00:00
node.DeviceID = uint64(stat.dev())
2015-04-29 03:45:24 +00:00
node.fillTimes(stat)
node.fillUser(stat)
2015-05-01 21:31:02 +00:00
2016-09-01 19:20:03 +00:00
switch node.Type {
2015-04-29 03:45:24 +00:00
case "file":
node.Size = uint64(stat.size())
node.Links = uint64(stat.nlink())
2015-04-29 03:45:24 +00:00
case "dir":
case "symlink":
var err error
node.LinkTarget, err = fs.Readlink(path)
2017-01-30 23:14:20 +00:00
node.Links = uint64(stat.nlink())
if err != nil {
return errors.WithStack(err)
}
2015-04-29 03:45:24 +00:00
case "dev":
node.Device = uint64(stat.rdev())
2017-01-30 23:14:20 +00:00
node.Links = uint64(stat.nlink())
2015-04-29 03:45:24 +00:00
case "chardev":
node.Device = uint64(stat.rdev())
2017-01-30 23:14:20 +00:00
node.Links = uint64(stat.nlink())
2015-04-29 03:45:24 +00:00
case "fifo":
case "socket":
default:
return errors.Errorf("unsupported file type %q", node.Type)
2015-04-29 03:45:24 +00:00
}
allowExtended, err := node.fillGenericAttributes(path, fi, stat)
if allowExtended {
// Skip processing ExtendedAttributes if allowExtended is false.
errEx := node.fillExtendedAttributes(path)
if err == nil {
err = errEx
} else {
debug.Log("Error filling extended attributes for %v at %v : %v", node.Name, path, errEx)
}
}
return err
}
func (node *Node) fillExtendedAttributes(path string) error {
xattrs, err := Listxattr(path)
debug.Log("fillExtendedAttributes(%v) %v %v", path, xattrs, err)
if err != nil {
return err
}
node.ExtendedAttributes = make([]ExtendedAttribute, 0, len(xattrs))
for _, attr := range xattrs {
attrVal, err := Getxattr(path, attr)
if err != nil {
fmt.Fprintf(os.Stderr, "can not obtain extended attribute %v for %v:\n", attr, path)
continue
}
attr := ExtendedAttribute{
Name: attr,
Value: attrVal,
}
node.ExtendedAttributes = append(node.ExtendedAttributes, attr)
}
return nil
2015-04-29 03:45:24 +00:00
}
func mkfifo(path string, mode uint32) (err error) {
return mknod(path, mode|syscall.S_IFIFO, 0)
}
func (node *Node) fillTimes(stat *statT) {
ctim := stat.ctim()
atim := stat.atim()
node.ChangeTime = time.Unix(ctim.Unix())
node.AccessTime = time.Unix(atim.Unix())
}
// HandleUnknownGenericAttributesFound is used for handling and distinguing between scenarios related to future versions and cross-OS repositories
func HandleUnknownGenericAttributesFound(unknownAttribs []GenericAttributeType, warn func(msg string)) {
for _, unknownAttrib := range unknownAttribs {
handleUnknownGenericAttributeFound(unknownAttrib, warn)
}
}
// handleUnknownGenericAttributeFound is used for handling and distinguing between scenarios related to future versions and cross-OS repositories
func handleUnknownGenericAttributeFound(genericAttributeType GenericAttributeType, warn func(msg string)) {
if checkGenericAttributeNameNotHandledAndPut(genericAttributeType) {
// Print the unique error only once for a given execution
os, exists := genericAttributesForOS[genericAttributeType]
if exists {
// If genericAttributesForOS contains an entry but we still got here, it means the specific node_xx.go for the current OS did not handle it and the repository may have been originally created on a different OS.
// The fact that node.go knows about the attribute, means it is not a new attribute. This may be a common situation if a repo is used across OSs.
debug.Log("Ignoring a generic attribute found in the repository: %s which may not be compatible with your OS. Compatible OS: %s", genericAttributeType, os)
} else {
// If genericAttributesForOS in node.go does not know about this attribute, then the repository may have been created by a newer version which has a newer GenericAttributeType.
warn(fmt.Sprintf("Found an unrecognized generic attribute in the repository: %s. You may need to upgrade to latest version of restic.", genericAttributeType))
}
}
}
// handleAllUnknownGenericAttributesFound performs validations for all generic attributes in the node.
// This is not used on windows currently because windows has handling for generic attributes.
// nolint:unused
func (node Node) handleAllUnknownGenericAttributesFound(warn func(msg string)) error {
for name := range node.GenericAttributes {
handleUnknownGenericAttributeFound(name, warn)
}
return nil
}
var unknownGenericAttributesHandlingHistory sync.Map
// checkGenericAttributeNameNotHandledAndPut checks if the GenericAttributeType name entry
// already exists and puts it in the map if not.
func checkGenericAttributeNameNotHandledAndPut(value GenericAttributeType) bool {
// If Key doesn't exist, put the value and return true because it is not already handled
_, exists := unknownGenericAttributesHandlingHistory.LoadOrStore(value, "")
// Key exists, then it is already handled so return false
return !exists
}
// The functions below are common helper functions which can be used for generic attributes support
// across different OS.
// genericAttributesToOSAttrs gets the os specific attribute from the generic attribute using reflection
// nolint:unused
func genericAttributesToOSAttrs(attrs map[GenericAttributeType]json.RawMessage, attributeType reflect.Type, attributeValuePtr *reflect.Value, keyPrefix string) (unknownAttribs []GenericAttributeType, err error) {
attributeValue := *attributeValuePtr
for key, rawMsg := range attrs {
found := false
for i := 0; i < attributeType.NumField(); i++ {
if getFQKeyByIndex(attributeType, i, keyPrefix) == key {
found = true
fieldValue := attributeValue.Field(i)
// For directly supported types, use json.Unmarshal directly
if err := json.Unmarshal(rawMsg, fieldValue.Addr().Interface()); err != nil {
return unknownAttribs, errors.Wrap(err, "Unmarshal")
}
break
}
}
if !found {
unknownAttribs = append(unknownAttribs, key)
}
}
return unknownAttribs, nil
}
// getFQKey gets the fully qualified key for the field
// nolint:unused
func getFQKey(field reflect.StructField, keyPrefix string) GenericAttributeType {
return GenericAttributeType(fmt.Sprintf("%s.%s", keyPrefix, field.Tag.Get("generic")))
}
// getFQKeyByIndex gets the fully qualified key for the field index
// nolint:unused
func getFQKeyByIndex(attributeType reflect.Type, index int, keyPrefix string) GenericAttributeType {
return getFQKey(attributeType.Field(index), keyPrefix)
}
// osAttrsToGenericAttributes gets the generic attribute from the os specific attribute using reflection
// nolint:unused
func osAttrsToGenericAttributes(attributeType reflect.Type, attributeValuePtr *reflect.Value, keyPrefix string) (attrs map[GenericAttributeType]json.RawMessage, err error) {
attributeValue := *attributeValuePtr
attrs = make(map[GenericAttributeType]json.RawMessage)
// Iterate over the fields of the struct
for i := 0; i < attributeType.NumField(); i++ {
field := attributeType.Field(i)
// Get the field value using reflection
fieldValue := attributeValue.FieldByName(field.Name)
// Check if the field is nil
if fieldValue.IsNil() {
// If it's nil, skip this field
continue
}
// Marshal the field value into a json.RawMessage
var fieldBytes []byte
if fieldBytes, err = json.Marshal(fieldValue.Interface()); err != nil {
return attrs, errors.Wrap(err, "Marshal")
}
// Insert the field into the map
attrs[getFQKey(field, keyPrefix)] = json.RawMessage(fieldBytes)
}
return attrs, nil
}