mirror of
https://github.com/octoleo/restic.git
synced 2024-11-26 14:56:29 +00:00
parent
b2d00b2a86
commit
366bf4eb0c
@ -15,6 +15,7 @@ import (
|
||||
type dirEntry struct {
|
||||
path string
|
||||
fi os.FileInfo
|
||||
link uint64
|
||||
}
|
||||
|
||||
func walkDir(dir string) <-chan *dirEntry {
|
||||
@ -36,6 +37,7 @@ func walkDir(dir string) <-chan *dirEntry {
|
||||
ch <- &dirEntry{
|
||||
path: name,
|
||||
fi: info,
|
||||
link: nlink(info),
|
||||
}
|
||||
|
||||
return nil
|
||||
|
@ -4,7 +4,9 @@ package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
@ -37,5 +39,37 @@ func (e *dirEntry) equals(other *dirEntry) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
if stat.Nlink != stat2.Nlink {
|
||||
fmt.Fprintf(os.Stderr, "%v: Number of links do not match (%v != %v)\n", e.path, stat.Nlink, stat2.Nlink)
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func nlink(info os.FileInfo) uint64 {
|
||||
stat, _ := info.Sys().(*syscall.Stat_t)
|
||||
return uint64(stat.Nlink)
|
||||
}
|
||||
|
||||
func inode(info os.FileInfo) uint64 {
|
||||
stat, _ := info.Sys().(*syscall.Stat_t)
|
||||
return uint64(stat.Ino)
|
||||
}
|
||||
|
||||
func createFileSetPerHardlink(dir string) map[uint64][]string {
|
||||
var stat syscall.Stat_t
|
||||
linkTests := make(map[uint64][]string)
|
||||
files, err := ioutil.ReadDir(dir)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
for _, f := range files {
|
||||
|
||||
if err := syscall.Stat(filepath.Join(dir, f.Name()), &stat); err != nil {
|
||||
return nil
|
||||
}
|
||||
linkTests[uint64(stat.Ino)] = append(linkTests[uint64(stat.Ino)], f.Name())
|
||||
}
|
||||
return linkTests
|
||||
}
|
||||
|
@ -4,6 +4,7 @@ package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
)
|
||||
|
||||
@ -25,3 +26,24 @@ func (e *dirEntry) equals(other *dirEntry) bool {
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func nlink(info os.FileInfo) uint64 {
|
||||
return 1
|
||||
}
|
||||
|
||||
func inode(info os.FileInfo) uint64 {
|
||||
return uint64(0)
|
||||
}
|
||||
|
||||
func createFileSetPerHardlink(dir string) map[uint64][]string {
|
||||
linkTests := make(map[uint64][]string)
|
||||
files, err := ioutil.ReadDir(dir)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
for i, f := range files {
|
||||
linkTests[uint64(i)] = append(linkTests[uint64(i)], f.Name())
|
||||
i++
|
||||
}
|
||||
return linkTests
|
||||
}
|
||||
|
@ -1011,3 +1011,100 @@ func TestPrune(t *testing.T) {
|
||||
testRunCheck(t, gopts)
|
||||
})
|
||||
}
|
||||
|
||||
func TestHardLink(t *testing.T) {
|
||||
// this test assumes a test set with a single directory containing hard linked files
|
||||
withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
|
||||
datafile := filepath.Join("testdata", "test.hl.tar.gz")
|
||||
fd, err := os.Open(datafile)
|
||||
if os.IsNotExist(errors.Cause(err)) {
|
||||
t.Skipf("unable to find data file %q, skipping", datafile)
|
||||
return
|
||||
}
|
||||
OK(t, err)
|
||||
OK(t, fd.Close())
|
||||
|
||||
testRunInit(t, gopts)
|
||||
|
||||
SetupTarTestFixture(t, env.testdata, datafile)
|
||||
|
||||
linkTests := createFileSetPerHardlink(env.testdata)
|
||||
|
||||
opts := BackupOptions{}
|
||||
|
||||
// first backup
|
||||
testRunBackup(t, []string{env.testdata}, opts, gopts)
|
||||
snapshotIDs := testRunList(t, "snapshots", gopts)
|
||||
Assert(t, len(snapshotIDs) == 1,
|
||||
"expected one snapshot, got %v", snapshotIDs)
|
||||
|
||||
testRunCheck(t, gopts)
|
||||
|
||||
// restore all backups and compare
|
||||
for i, snapshotID := range snapshotIDs {
|
||||
restoredir := filepath.Join(env.base, fmt.Sprintf("restore%d", i))
|
||||
t.Logf("restoring snapshot %v to %v", snapshotID.Str(), restoredir)
|
||||
testRunRestore(t, gopts, restoredir, snapshotIDs[0])
|
||||
Assert(t, directoriesEqualContents(env.testdata, filepath.Join(restoredir, "testdata")),
|
||||
"directories are not equal")
|
||||
|
||||
linkResults := createFileSetPerHardlink(filepath.Join(restoredir, "testdata"))
|
||||
Assert(t, linksEqual(linkTests, linkResults),
|
||||
"links are not equal")
|
||||
}
|
||||
|
||||
testRunCheck(t, gopts)
|
||||
})
|
||||
}
|
||||
|
||||
func linksEqual(source, dest map[uint64][]string) bool {
|
||||
for _, vs := range source {
|
||||
found := false
|
||||
for kd, vd := range dest {
|
||||
if linkEqual(vs, vd) {
|
||||
delete(dest, kd)
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
if len(dest) != 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func linkEqual(source, dest []string) bool {
|
||||
// equal if sliced are equal without considering order
|
||||
if source == nil && dest == nil {
|
||||
return true
|
||||
}
|
||||
|
||||
if source == nil || dest == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
if len(source) != len(dest) {
|
||||
return false
|
||||
}
|
||||
|
||||
for i := range source {
|
||||
found := false
|
||||
for j := range dest {
|
||||
if source[i] == dest[j] {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
BIN
src/cmds/restic/testdata/test.hl.tar.gz
vendored
Normal file
BIN
src/cmds/restic/testdata/test.hl.tar.gz
vendored
Normal file
Binary file not shown.
@ -102,6 +102,12 @@ func Symlink(oldname, newname string) error {
|
||||
return os.Symlink(fixpath(oldname), fixpath(newname))
|
||||
}
|
||||
|
||||
// Link creates newname as a hard link to oldname.
|
||||
// If there is an error, it will be of type *LinkError.
|
||||
func Link(oldname, newname string) error {
|
||||
return os.Link(fixpath(oldname), fixpath(newname))
|
||||
}
|
||||
|
||||
// Stat returns a FileInfo structure describing the named file.
|
||||
// If there is an error, it will be of type *PathError.
|
||||
func Stat(name string) (os.FileInfo, error) {
|
||||
|
57
src/restic/hardlinks_index.go
Normal file
57
src/restic/hardlinks_index.go
Normal file
@ -0,0 +1,57 @@
|
||||
package restic
|
||||
|
||||
import (
|
||||
"sync"
|
||||
)
|
||||
|
||||
// HardlinkKey is a composed key for finding inodes on a specific device.
|
||||
type HardlinkKey struct {
|
||||
Inode, Device uint64
|
||||
}
|
||||
|
||||
// HardlinkIndex contains a list of inodes, devices these inodes are one, and associated file names.
|
||||
type HardlinkIndex struct {
|
||||
m sync.Mutex
|
||||
Index map[HardlinkKey]string
|
||||
}
|
||||
|
||||
// NewHardlinkIndex create a new index for hard links
|
||||
func NewHardlinkIndex() *HardlinkIndex {
|
||||
return &HardlinkIndex{
|
||||
Index: make(map[HardlinkKey]string),
|
||||
}
|
||||
}
|
||||
|
||||
// Has checks wether the link already exist in the index.
|
||||
func (idx *HardlinkIndex) Has(inode uint64, device uint64) bool {
|
||||
idx.m.Lock()
|
||||
defer idx.m.Unlock()
|
||||
_, ok := idx.Index[HardlinkKey{inode, device}]
|
||||
|
||||
return ok
|
||||
}
|
||||
|
||||
// Add adds a link to the index.
|
||||
func (idx *HardlinkIndex) Add(inode uint64, device uint64, name string) {
|
||||
idx.m.Lock()
|
||||
defer idx.m.Unlock()
|
||||
_, ok := idx.Index[HardlinkKey{inode, device}]
|
||||
|
||||
if !ok {
|
||||
idx.Index[HardlinkKey{inode, device}] = name
|
||||
}
|
||||
}
|
||||
|
||||
// GetFilename obtains the filename from the index.
|
||||
func (idx *HardlinkIndex) GetFilename(inode uint64, device uint64) string {
|
||||
idx.m.Lock()
|
||||
defer idx.m.Unlock()
|
||||
return idx.Index[HardlinkKey{inode, device}]
|
||||
}
|
||||
|
||||
// Remove removes a link from the index.
|
||||
func (idx *HardlinkIndex) Remove(inode uint64, device uint64) {
|
||||
idx.m.Lock()
|
||||
defer idx.m.Unlock()
|
||||
delete(idx.Index, HardlinkKey{inode, device})
|
||||
}
|
35
src/restic/hardlinks_index_test.go
Normal file
35
src/restic/hardlinks_index_test.go
Normal file
@ -0,0 +1,35 @@
|
||||
package restic_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"restic"
|
||||
. "restic/test"
|
||||
)
|
||||
|
||||
// TestHardLinks contains various tests for HardlinkIndex.
|
||||
func TestHardLinks(t *testing.T) {
|
||||
|
||||
idx := restic.NewHardlinkIndex()
|
||||
|
||||
idx.Add(1, 2, "inode1-file1-on-device2")
|
||||
idx.Add(2, 3, "inode2-file2-on-device3")
|
||||
|
||||
var sresult string
|
||||
sresult = idx.GetFilename(1, 2)
|
||||
Equals(t, sresult, "inode1-file1-on-device2")
|
||||
|
||||
sresult = idx.GetFilename(2, 3)
|
||||
Equals(t, sresult, "inode2-file2-on-device3")
|
||||
|
||||
var bresult bool
|
||||
bresult = idx.Has(1, 2)
|
||||
Equals(t, bresult, true)
|
||||
|
||||
bresult = idx.Has(1, 3)
|
||||
Equals(t, bresult, false)
|
||||
|
||||
idx.Remove(1, 2)
|
||||
bresult = idx.Has(1, 2)
|
||||
Equals(t, bresult, false)
|
||||
}
|
@ -97,7 +97,7 @@ func nodeTypeFromFileInfo(fi os.FileInfo) string {
|
||||
}
|
||||
|
||||
// CreateAt creates the node at the given path and restores all the meta data.
|
||||
func (node *Node) CreateAt(path string, repo Repository) error {
|
||||
func (node *Node) CreateAt(path string, repo Repository, idx *HardlinkIndex) error {
|
||||
debug.Log("create node %v at %v", node.Name, path)
|
||||
|
||||
switch node.Type {
|
||||
@ -106,7 +106,7 @@ func (node *Node) CreateAt(path string, repo Repository) error {
|
||||
return err
|
||||
}
|
||||
case "file":
|
||||
if err := node.createFileAt(path, repo); err != nil {
|
||||
if err := node.createFileAt(path, repo, idx); err != nil {
|
||||
return err
|
||||
}
|
||||
case "symlink":
|
||||
@ -191,7 +191,15 @@ func (node Node) createDirAt(path string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (node Node) createFileAt(path string, repo Repository) error {
|
||||
func (node Node) createFileAt(path string, repo Repository, idx *HardlinkIndex) error {
|
||||
if node.Links > 1 && idx.Has(node.Inode, node.Device) {
|
||||
err := fs.Link(idx.GetFilename(node.Inode, node.Device), path)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "CreateHardlink")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
f, err := fs.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0600)
|
||||
defer f.Close()
|
||||
|
||||
@ -223,6 +231,8 @@ func (node Node) createFileAt(path string, repo Repository) error {
|
||||
}
|
||||
}
|
||||
|
||||
idx.Add(node.Inode, node.Device, path)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -485,11 +495,14 @@ func (node *Node) fillExtra(path string, fi os.FileInfo) error {
|
||||
case "dir":
|
||||
case "symlink":
|
||||
node.LinkTarget, err = fs.Readlink(path)
|
||||
node.Links = uint64(stat.nlink())
|
||||
err = errors.Wrap(err, "Readlink")
|
||||
case "dev":
|
||||
node.Device = uint64(stat.rdev())
|
||||
node.Links = uint64(stat.nlink())
|
||||
case "chardev":
|
||||
node.Device = uint64(stat.rdev())
|
||||
node.Links = uint64(stat.nlink())
|
||||
case "fifo":
|
||||
case "socket":
|
||||
default:
|
||||
|
@ -176,9 +176,11 @@ func TestNodeRestoreAt(t *testing.T) {
|
||||
}
|
||||
}()
|
||||
|
||||
idx := restic.NewHardlinkIndex()
|
||||
|
||||
for _, test := range nodeTests {
|
||||
nodePath := filepath.Join(tempdir, test.Name)
|
||||
OK(t, test.CreateAt(nodePath, nil))
|
||||
OK(t, test.CreateAt(nodePath, nil, idx))
|
||||
|
||||
if test.Type == "symlink" && runtime.GOOS == "windows" {
|
||||
continue
|
||||
|
@ -24,6 +24,7 @@ func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespe
|
||||
|
||||
type statWin syscall.Win32FileAttributeData
|
||||
|
||||
//ToStatT call the Windows system call Win32FileAttributeData.
|
||||
func toStatT(i interface{}) (statT, bool) {
|
||||
if i == nil {
|
||||
return nil, false
|
||||
|
@ -38,7 +38,7 @@ func NewRestorer(repo Repository, id ID) (*Restorer, error) {
|
||||
return r, nil
|
||||
}
|
||||
|
||||
func (res *Restorer) restoreTo(dst string, dir string, treeID ID) error {
|
||||
func (res *Restorer) restoreTo(dst string, dir string, treeID ID, idx *HardlinkIndex) error {
|
||||
tree, err := res.repo.LoadTree(treeID)
|
||||
if err != nil {
|
||||
return res.Error(dir, nil, err)
|
||||
@ -50,7 +50,7 @@ func (res *Restorer) restoreTo(dst string, dir string, treeID ID) error {
|
||||
debug.Log("SelectForRestore returned %v", selectedForRestore)
|
||||
|
||||
if selectedForRestore {
|
||||
err := res.restoreNodeTo(node, dir, dst)
|
||||
err := res.restoreNodeTo(node, dir, dst, idx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -62,7 +62,7 @@ func (res *Restorer) restoreTo(dst string, dir string, treeID ID) error {
|
||||
}
|
||||
|
||||
subp := filepath.Join(dir, node.Name)
|
||||
err = res.restoreTo(dst, subp, *node.Subtree)
|
||||
err = res.restoreTo(dst, subp, *node.Subtree, idx)
|
||||
if err != nil {
|
||||
err = res.Error(subp, node, err)
|
||||
if err != nil {
|
||||
@ -83,11 +83,11 @@ func (res *Restorer) restoreTo(dst string, dir string, treeID ID) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (res *Restorer) restoreNodeTo(node *Node, dir string, dst string) error {
|
||||
func (res *Restorer) restoreNodeTo(node *Node, dir string, dst string, idx *HardlinkIndex) error {
|
||||
debug.Log("node %v, dir %v, dst %v", node.Name, dir, dst)
|
||||
dstPath := filepath.Join(dst, dir, node.Name)
|
||||
|
||||
err := node.CreateAt(dstPath, res.repo)
|
||||
err := node.CreateAt(dstPath, res.repo, idx)
|
||||
if err != nil {
|
||||
debug.Log("node.CreateAt(%s) error %v", dstPath, err)
|
||||
}
|
||||
@ -99,7 +99,7 @@ func (res *Restorer) restoreNodeTo(node *Node, dir string, dst string) error {
|
||||
// Create parent directories and retry
|
||||
err = fs.MkdirAll(filepath.Dir(dstPath), 0700)
|
||||
if err == nil || os.IsExist(errors.Cause(err)) {
|
||||
err = node.CreateAt(dstPath, res.repo)
|
||||
err = node.CreateAt(dstPath, res.repo, idx)
|
||||
}
|
||||
}
|
||||
|
||||
@ -119,7 +119,8 @@ func (res *Restorer) restoreNodeTo(node *Node, dir string, dst string) error {
|
||||
// RestoreTo creates the directories and files in the snapshot below dir.
|
||||
// Before an item is created, res.Filter is called.
|
||||
func (res *Restorer) RestoreTo(dir string) error {
|
||||
return res.restoreTo(dir, "", *res.sn.Tree)
|
||||
idx := NewHardlinkIndex()
|
||||
return res.restoreTo(dir, "", *res.sn.Tree, idx)
|
||||
}
|
||||
|
||||
// Snapshot returns the snapshot this restorer is configured to use.
|
||||
|
Loading…
Reference in New Issue
Block a user