2
2
mirror of https://github.com/octoleo/restic.git synced 2024-12-23 11:28:54 +00:00

Merge pull request #763 from jgfrm/issue25

Support hard links
This commit is contained in:
Alexander Neumann 2017-02-10 20:58:39 +01:00
commit 6300c8df56
12 changed files with 281 additions and 11 deletions

View File

@ -15,6 +15,7 @@ import (
type dirEntry struct {
path string
fi os.FileInfo
link uint64
}
func walkDir(dir string) <-chan *dirEntry {
@ -36,6 +37,7 @@ func walkDir(dir string) <-chan *dirEntry {
ch <- &dirEntry{
path: name,
fi: info,
link: nlink(info),
}
return nil

View File

@ -4,7 +4,9 @@ package main
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"syscall"
)
@ -37,5 +39,37 @@ func (e *dirEntry) equals(other *dirEntry) bool {
return false
}
if stat.Nlink != stat2.Nlink {
fmt.Fprintf(os.Stderr, "%v: Number of links do not match (%v != %v)\n", e.path, stat.Nlink, stat2.Nlink)
return false
}
return true
}
func nlink(info os.FileInfo) uint64 {
stat, _ := info.Sys().(*syscall.Stat_t)
return uint64(stat.Nlink)
}
func inode(info os.FileInfo) uint64 {
stat, _ := info.Sys().(*syscall.Stat_t)
return uint64(stat.Ino)
}
func createFileSetPerHardlink(dir string) map[uint64][]string {
var stat syscall.Stat_t
linkTests := make(map[uint64][]string)
files, err := ioutil.ReadDir(dir)
if err != nil {
return nil
}
for _, f := range files {
if err := syscall.Stat(filepath.Join(dir, f.Name()), &stat); err != nil {
return nil
}
linkTests[uint64(stat.Ino)] = append(linkTests[uint64(stat.Ino)], f.Name())
}
return linkTests
}

View File

@ -4,6 +4,7 @@ package main
import (
"fmt"
"io/ioutil"
"os"
)
@ -25,3 +26,24 @@ func (e *dirEntry) equals(other *dirEntry) bool {
return true
}
func nlink(info os.FileInfo) uint64 {
return 1
}
func inode(info os.FileInfo) uint64 {
return uint64(0)
}
func createFileSetPerHardlink(dir string) map[uint64][]string {
linkTests := make(map[uint64][]string)
files, err := ioutil.ReadDir(dir)
if err != nil {
return nil
}
for i, f := range files {
linkTests[uint64(i)] = append(linkTests[uint64(i)], f.Name())
i++
}
return linkTests
}

View File

@ -1011,3 +1011,100 @@ func TestPrune(t *testing.T) {
testRunCheck(t, gopts)
})
}
func TestHardLink(t *testing.T) {
// this test assumes a test set with a single directory containing hard linked files
withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
datafile := filepath.Join("testdata", "test.hl.tar.gz")
fd, err := os.Open(datafile)
if os.IsNotExist(errors.Cause(err)) {
t.Skipf("unable to find data file %q, skipping", datafile)
return
}
OK(t, err)
OK(t, fd.Close())
testRunInit(t, gopts)
SetupTarTestFixture(t, env.testdata, datafile)
linkTests := createFileSetPerHardlink(env.testdata)
opts := BackupOptions{}
// first backup
testRunBackup(t, []string{env.testdata}, opts, gopts)
snapshotIDs := testRunList(t, "snapshots", gopts)
Assert(t, len(snapshotIDs) == 1,
"expected one snapshot, got %v", snapshotIDs)
testRunCheck(t, gopts)
// restore all backups and compare
for i, snapshotID := range snapshotIDs {
restoredir := filepath.Join(env.base, fmt.Sprintf("restore%d", i))
t.Logf("restoring snapshot %v to %v", snapshotID.Str(), restoredir)
testRunRestore(t, gopts, restoredir, snapshotIDs[0])
Assert(t, directoriesEqualContents(env.testdata, filepath.Join(restoredir, "testdata")),
"directories are not equal")
linkResults := createFileSetPerHardlink(filepath.Join(restoredir, "testdata"))
Assert(t, linksEqual(linkTests, linkResults),
"links are not equal")
}
testRunCheck(t, gopts)
})
}
func linksEqual(source, dest map[uint64][]string) bool {
for _, vs := range source {
found := false
for kd, vd := range dest {
if linkEqual(vs, vd) {
delete(dest, kd)
found = true
break
}
}
if !found {
return false
}
}
if len(dest) != 0 {
return false
}
return true
}
func linkEqual(source, dest []string) bool {
// equal if sliced are equal without considering order
if source == nil && dest == nil {
return true
}
if source == nil || dest == nil {
return false
}
if len(source) != len(dest) {
return false
}
for i := range source {
found := false
for j := range dest {
if source[i] == dest[j] {
found = true
break
}
}
if !found {
return false
}
}
return true
}

BIN
src/cmds/restic/testdata/test.hl.tar.gz vendored Normal file

Binary file not shown.

View File

@ -102,6 +102,12 @@ func Symlink(oldname, newname string) error {
return os.Symlink(fixpath(oldname), fixpath(newname))
}
// Link creates newname as a hard link to oldname.
// If there is an error, it will be of type *LinkError.
func Link(oldname, newname string) error {
return os.Link(fixpath(oldname), fixpath(newname))
}
// Stat returns a FileInfo structure describing the named file.
// If there is an error, it will be of type *PathError.
func Stat(name string) (os.FileInfo, error) {

View File

@ -0,0 +1,57 @@
package restic
import (
"sync"
)
// HardlinkKey is a composed key for finding inodes on a specific device.
type HardlinkKey struct {
Inode, Device uint64
}
// HardlinkIndex contains a list of inodes, devices these inodes are one, and associated file names.
type HardlinkIndex struct {
m sync.Mutex
Index map[HardlinkKey]string
}
// NewHardlinkIndex create a new index for hard links
func NewHardlinkIndex() *HardlinkIndex {
return &HardlinkIndex{
Index: make(map[HardlinkKey]string),
}
}
// Has checks wether the link already exist in the index.
func (idx *HardlinkIndex) Has(inode uint64, device uint64) bool {
idx.m.Lock()
defer idx.m.Unlock()
_, ok := idx.Index[HardlinkKey{inode, device}]
return ok
}
// Add adds a link to the index.
func (idx *HardlinkIndex) Add(inode uint64, device uint64, name string) {
idx.m.Lock()
defer idx.m.Unlock()
_, ok := idx.Index[HardlinkKey{inode, device}]
if !ok {
idx.Index[HardlinkKey{inode, device}] = name
}
}
// GetFilename obtains the filename from the index.
func (idx *HardlinkIndex) GetFilename(inode uint64, device uint64) string {
idx.m.Lock()
defer idx.m.Unlock()
return idx.Index[HardlinkKey{inode, device}]
}
// Remove removes a link from the index.
func (idx *HardlinkIndex) Remove(inode uint64, device uint64) {
idx.m.Lock()
defer idx.m.Unlock()
delete(idx.Index, HardlinkKey{inode, device})
}

View File

@ -0,0 +1,35 @@
package restic_test
import (
"testing"
"restic"
. "restic/test"
)
// TestHardLinks contains various tests for HardlinkIndex.
func TestHardLinks(t *testing.T) {
idx := restic.NewHardlinkIndex()
idx.Add(1, 2, "inode1-file1-on-device2")
idx.Add(2, 3, "inode2-file2-on-device3")
var sresult string
sresult = idx.GetFilename(1, 2)
Equals(t, sresult, "inode1-file1-on-device2")
sresult = idx.GetFilename(2, 3)
Equals(t, sresult, "inode2-file2-on-device3")
var bresult bool
bresult = idx.Has(1, 2)
Equals(t, bresult, true)
bresult = idx.Has(1, 3)
Equals(t, bresult, false)
idx.Remove(1, 2)
bresult = idx.Has(1, 2)
Equals(t, bresult, false)
}

View File

@ -97,7 +97,7 @@ func nodeTypeFromFileInfo(fi os.FileInfo) string {
}
// CreateAt creates the node at the given path and restores all the meta data.
func (node *Node) CreateAt(path string, repo Repository) error {
func (node *Node) CreateAt(path string, repo Repository, idx *HardlinkIndex) error {
debug.Log("create node %v at %v", node.Name, path)
switch node.Type {
@ -106,7 +106,7 @@ func (node *Node) CreateAt(path string, repo Repository) error {
return err
}
case "file":
if err := node.createFileAt(path, repo); err != nil {
if err := node.createFileAt(path, repo, idx); err != nil {
return err
}
case "symlink":
@ -191,7 +191,15 @@ func (node Node) createDirAt(path string) error {
return nil
}
func (node Node) createFileAt(path string, repo Repository) error {
func (node Node) createFileAt(path string, repo Repository, idx *HardlinkIndex) error {
if node.Links > 1 && idx.Has(node.Inode, node.Device) {
err := fs.Link(idx.GetFilename(node.Inode, node.Device), path)
if err != nil {
return errors.Wrap(err, "CreateHardlink")
}
return nil
}
f, err := fs.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0600)
defer f.Close()
@ -223,6 +231,8 @@ func (node Node) createFileAt(path string, repo Repository) error {
}
}
idx.Add(node.Inode, node.Device, path)
return nil
}
@ -485,11 +495,14 @@ func (node *Node) fillExtra(path string, fi os.FileInfo) error {
case "dir":
case "symlink":
node.LinkTarget, err = fs.Readlink(path)
node.Links = uint64(stat.nlink())
err = errors.Wrap(err, "Readlink")
case "dev":
node.Device = uint64(stat.rdev())
node.Links = uint64(stat.nlink())
case "chardev":
node.Device = uint64(stat.rdev())
node.Links = uint64(stat.nlink())
case "fifo":
case "socket":
default:

View File

@ -176,9 +176,11 @@ func TestNodeRestoreAt(t *testing.T) {
}
}()
idx := restic.NewHardlinkIndex()
for _, test := range nodeTests {
nodePath := filepath.Join(tempdir, test.Name)
OK(t, test.CreateAt(nodePath, nil))
OK(t, test.CreateAt(nodePath, nil, idx))
if test.Type == "symlink" && runtime.GOOS == "windows" {
continue

View File

@ -24,6 +24,7 @@ func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespe
type statWin syscall.Win32FileAttributeData
//ToStatT call the Windows system call Win32FileAttributeData.
func toStatT(i interface{}) (statT, bool) {
if i == nil {
return nil, false

View File

@ -38,7 +38,7 @@ func NewRestorer(repo Repository, id ID) (*Restorer, error) {
return r, nil
}
func (res *Restorer) restoreTo(dst string, dir string, treeID ID) error {
func (res *Restorer) restoreTo(dst string, dir string, treeID ID, idx *HardlinkIndex) error {
tree, err := res.repo.LoadTree(treeID)
if err != nil {
return res.Error(dir, nil, err)
@ -50,7 +50,7 @@ func (res *Restorer) restoreTo(dst string, dir string, treeID ID) error {
debug.Log("SelectForRestore returned %v", selectedForRestore)
if selectedForRestore {
err := res.restoreNodeTo(node, dir, dst)
err := res.restoreNodeTo(node, dir, dst, idx)
if err != nil {
return err
}
@ -62,7 +62,7 @@ func (res *Restorer) restoreTo(dst string, dir string, treeID ID) error {
}
subp := filepath.Join(dir, node.Name)
err = res.restoreTo(dst, subp, *node.Subtree)
err = res.restoreTo(dst, subp, *node.Subtree, idx)
if err != nil {
err = res.Error(subp, node, err)
if err != nil {
@ -83,11 +83,11 @@ func (res *Restorer) restoreTo(dst string, dir string, treeID ID) error {
return nil
}
func (res *Restorer) restoreNodeTo(node *Node, dir string, dst string) error {
func (res *Restorer) restoreNodeTo(node *Node, dir string, dst string, idx *HardlinkIndex) error {
debug.Log("node %v, dir %v, dst %v", node.Name, dir, dst)
dstPath := filepath.Join(dst, dir, node.Name)
err := node.CreateAt(dstPath, res.repo)
err := node.CreateAt(dstPath, res.repo, idx)
if err != nil {
debug.Log("node.CreateAt(%s) error %v", dstPath, err)
}
@ -99,7 +99,7 @@ func (res *Restorer) restoreNodeTo(node *Node, dir string, dst string) error {
// Create parent directories and retry
err = fs.MkdirAll(filepath.Dir(dstPath), 0700)
if err == nil || os.IsExist(errors.Cause(err)) {
err = node.CreateAt(dstPath, res.repo)
err = node.CreateAt(dstPath, res.repo, idx)
}
}
@ -119,7 +119,8 @@ func (res *Restorer) restoreNodeTo(node *Node, dir string, dst string) error {
// RestoreTo creates the directories and files in the snapshot below dir.
// Before an item is created, res.Filter is called.
func (res *Restorer) RestoreTo(dir string) error {
return res.restoreTo(dir, "", *res.sn.Tree)
idx := NewHardlinkIndex()
return res.restoreTo(dir, "", *res.sn.Tree, idx)
}
// Snapshot returns the snapshot this restorer is configured to use.