2
2
mirror of https://github.com/octoleo/restic.git synced 2024-11-22 04:45:15 +00:00

Optimize repository layout

Use subdirectoriers for Data and Tree objects, e.g. store blob
`aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f`
at
```
ac/c0/aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f
```

Issue #58
This commit is contained in:
Alexander Neumann 2014-11-30 15:50:30 +01:00
parent bbaba349b2
commit 6d56d7d4c6
2 changed files with 127 additions and 22 deletions

View File

@ -149,20 +149,33 @@ func (b *Local) tempFile() (*os.File, error) {
// Rename temp file to final name according to type and ID.
func (b *Local) renameFile(file *os.File, t Type, id ID) error {
filename := filepath.Join(b.dir(t), id.String())
return os.Rename(file.Name(), filename)
newname := b.filename(t, id)
oldname := file.Name()
if t == Data || t == Tree {
// create directories if necessary, ignore errors
os.MkdirAll(filepath.Dir(newname), dirMode)
}
return os.Rename(oldname, newname)
}
// Construct directory for given Type.
func (b *Local) dir(t Type) string {
func (b *Local) dirname(t Type, id ID) string {
var n string
switch t {
case Data:
n = dataPath
if id != nil {
n = filepath.Join(dataPath, fmt.Sprintf("%02x", id[0]), fmt.Sprintf("%02x", id[1]))
}
case Snapshot:
n = snapshotPath
case Tree:
n = treePath
if id != nil {
n = filepath.Join(treePath, fmt.Sprintf("%02x", id[0]), fmt.Sprintf("%02x", id[1]))
}
case Map:
n = mapPath
case Lock:
@ -217,7 +230,7 @@ func (b *Local) Create(t Type, data []byte) (ID, error) {
// Construct path for given Type and ID.
func (b *Local) filename(t Type, id ID) string {
return filepath.Join(b.dir(t), id.String())
return filepath.Join(b.dirname(t, id), id.String())
}
// Get returns the content stored under the given ID. If the data doesn't match
@ -270,7 +283,12 @@ func (b *Local) Remove(t Type, id ID) error {
// List lists all objects of a given type.
func (b *Local) List(t Type) (IDs, error) {
// TODO: use os.Open() and d.Readdirnames() instead of Glob()
pattern := filepath.Join(b.dir(t), "*")
var pattern string
if t == Data || t == Tree {
pattern = filepath.Join(b.dirname(t, nil), "*", "*", "*")
} else {
pattern = filepath.Join(b.dirname(t, nil), "*")
}
matches, err := filepath.Glob(pattern)
if err != nil {

View File

@ -155,15 +155,11 @@ func CreateSFTP(dir string, program string, args ...string) (*SFTP, error) {
// create paths for data, refs and temp blobs
for _, d := range dirs {
// TODO: implement client.MkdirAll() and set mode to dirMode
_, err = sftp.c.Lstat(d)
if err != nil {
err = sftp.c.Mkdir(d)
err = sftp.mkdirAll(d, dirMode)
if err != nil {
return nil, err
}
}
}
// create version file
f, err := sftp.c.Create(versionFile)
@ -225,21 +221,69 @@ func (r *SFTP) tempFile() (string, *sftp.File, error) {
return name, f, nil
}
func (r *SFTP) mkdirAll(dir string, mode os.FileMode) error {
// check if directory already exists
fi, err := r.c.Lstat(dir)
if err == nil {
if fi.IsDir() {
return nil
}
return fmt.Errorf("mkdirAll(%s): entry exists but is not a directory", dir)
}
// create parent directories
errMkdirAll := r.mkdirAll(filepath.Dir(dir), dirMode)
// create directory
errMkdir := r.c.Mkdir(dir)
// test if directory was created successfully
fi, err = r.c.Lstat(dir)
if err != nil {
// return previous errors
return fmt.Errorf("mkdirAll(%s): unable to create directories: %v, %v", errMkdirAll, errMkdir)
}
if !fi.IsDir() {
return fmt.Errorf("mkdirAll(%s): entry exists but is not a directory", dir)
}
// set mode
return r.c.Chmod(dir, mode)
}
// Rename temp file to final name according to type and ID.
func (r *SFTP) renameFile(filename string, t Type, id ID) error {
return r.c.Rename(filename, filepath.Join(r.dir(t), id.String()))
func (r *SFTP) renameFile(oldname string, t Type, id ID) error {
newname := r.filename(t, id)
// create directories if necessary
if t == Data || t == Tree {
err := r.mkdirAll(filepath.Dir(newname), dirMode)
if err != nil {
return err
}
}
return r.c.Rename(oldname, newname)
}
// Construct directory for given Type.
func (r *SFTP) dir(t Type) string {
func (r *SFTP) dirname(t Type, id ID) string {
var n string
switch t {
case Data:
n = dataPath
if id != nil {
n = filepath.Join(dataPath, fmt.Sprintf("%02x", id[0]), fmt.Sprintf("%02x", id[1]))
}
case Snapshot:
n = snapshotPath
case Tree:
n = treePath
if id != nil {
n = filepath.Join(treePath, fmt.Sprintf("%02x", id[0]), fmt.Sprintf("%02x", id[1]))
}
case Map:
n = mapPath
case Lock:
@ -257,7 +301,12 @@ func (r *SFTP) Create(t Type, data []byte) (ID, error) {
// check if blob is already present in backend
id := IDFromData(data)
if ok, _ := r.Test(t, id); ok {
res, err := r.Test(t, id)
if err != nil {
return nil, arrar.Annotate(err, "test for presence")
}
if res {
return id, ErrAlreadyPresent
}
@ -289,7 +338,7 @@ func (r *SFTP) Create(t Type, data []byte) (ID, error) {
// Construct path for given Type and ID.
func (r *SFTP) filename(t Type, id ID) string {
return filepath.Join(r.dir(t), id.String())
return filepath.Join(r.dirname(t, id), id.String())
}
// Get returns the content stored under the given ID. If the data doesn't match
@ -325,12 +374,17 @@ func (r *SFTP) Test(t Type, id ID) (bool, error) {
}
}()
if err == nil {
return true, nil
if err != nil {
if _, ok := err.(*sftp.StatusError); ok {
return false, nil
}
return false, err
}
return true, nil
}
// Remove removes the content stored at ID.
func (r *SFTP) Remove(t Type, id ID) error {
return r.c.Remove(r.filename(t, id))
@ -338,11 +392,44 @@ func (r *SFTP) Remove(t Type, id ID) error {
// List lists all objects of a given type.
func (r *SFTP) List(t Type) (IDs, error) {
list, err := r.c.ReadDir(r.dir(t))
list := []os.FileInfo{}
var err error
if t == Data || t == Tree {
// read first level
basedir := r.dirname(t, nil)
list1, err := r.c.ReadDir(basedir)
if err != nil {
return nil, err
}
for _, dir1 := range list1 {
// read second level
list2, err := r.c.ReadDir(filepath.Join(basedir, dir1.Name()))
if err != nil {
return nil, err
}
// read files
for _, dir2 := range list2 {
entries, err := r.c.ReadDir(filepath.Join(basedir, dir1.Name(), dir2.Name()))
if err != nil {
return nil, err
}
for _, entry := range entries {
list = append(list, entry)
}
}
}
} else {
list, err = r.c.ReadDir(r.dirname(t, nil))
if err != nil {
return nil, err
}
}
ids := make(IDs, 0, len(list))
for _, item := range list {
id, err := ParseID(item.Name())