diff --git a/backend/local.go b/backend/local.go index f55f88587..dc7752cc1 100644 --- a/backend/local.go +++ b/backend/local.go @@ -149,20 +149,33 @@ func (b *Local) tempFile() (*os.File, error) { // Rename temp file to final name according to type and ID. func (b *Local) renameFile(file *os.File, t Type, id ID) error { - filename := filepath.Join(b.dir(t), id.String()) - return os.Rename(file.Name(), filename) + newname := b.filename(t, id) + oldname := file.Name() + + if t == Data || t == Tree { + // create directories if necessary, ignore errors + os.MkdirAll(filepath.Dir(newname), dirMode) + } + + return os.Rename(oldname, newname) } // Construct directory for given Type. -func (b *Local) dir(t Type) string { +func (b *Local) dirname(t Type, id ID) string { var n string switch t { case Data: n = dataPath + if id != nil { + n = filepath.Join(dataPath, fmt.Sprintf("%02x", id[0]), fmt.Sprintf("%02x", id[1])) + } case Snapshot: n = snapshotPath case Tree: n = treePath + if id != nil { + n = filepath.Join(treePath, fmt.Sprintf("%02x", id[0]), fmt.Sprintf("%02x", id[1])) + } case Map: n = mapPath case Lock: @@ -217,7 +230,7 @@ func (b *Local) Create(t Type, data []byte) (ID, error) { // Construct path for given Type and ID. func (b *Local) filename(t Type, id ID) string { - return filepath.Join(b.dir(t), id.String()) + return filepath.Join(b.dirname(t, id), id.String()) } // Get returns the content stored under the given ID. If the data doesn't match @@ -270,7 +283,12 @@ func (b *Local) Remove(t Type, id ID) error { // List lists all objects of a given type. func (b *Local) List(t Type) (IDs, error) { // TODO: use os.Open() and d.Readdirnames() instead of Glob() - pattern := filepath.Join(b.dir(t), "*") + var pattern string + if t == Data || t == Tree { + pattern = filepath.Join(b.dirname(t, nil), "*", "*", "*") + } else { + pattern = filepath.Join(b.dirname(t, nil), "*") + } matches, err := filepath.Glob(pattern) if err != nil { diff --git a/backend/sftp.go b/backend/sftp.go index 42db4fe62..704f7c450 100644 --- a/backend/sftp.go +++ b/backend/sftp.go @@ -155,13 +155,9 @@ func CreateSFTP(dir string, program string, args ...string) (*SFTP, error) { // create paths for data, refs and temp blobs for _, d := range dirs { - // TODO: implement client.MkdirAll() and set mode to dirMode - _, err = sftp.c.Lstat(d) + err = sftp.mkdirAll(d, dirMode) if err != nil { - err = sftp.c.Mkdir(d) - if err != nil { - return nil, err - } + return nil, err } } @@ -225,21 +221,69 @@ func (r *SFTP) tempFile() (string, *sftp.File, error) { return name, f, nil } +func (r *SFTP) mkdirAll(dir string, mode os.FileMode) error { + // check if directory already exists + fi, err := r.c.Lstat(dir) + if err == nil { + if fi.IsDir() { + return nil + } + + return fmt.Errorf("mkdirAll(%s): entry exists but is not a directory", dir) + } + + // create parent directories + errMkdirAll := r.mkdirAll(filepath.Dir(dir), dirMode) + + // create directory + errMkdir := r.c.Mkdir(dir) + + // test if directory was created successfully + fi, err = r.c.Lstat(dir) + if err != nil { + // return previous errors + return fmt.Errorf("mkdirAll(%s): unable to create directories: %v, %v", errMkdirAll, errMkdir) + } + + if !fi.IsDir() { + return fmt.Errorf("mkdirAll(%s): entry exists but is not a directory", dir) + } + + // set mode + return r.c.Chmod(dir, mode) +} + // Rename temp file to final name according to type and ID. -func (r *SFTP) renameFile(filename string, t Type, id ID) error { - return r.c.Rename(filename, filepath.Join(r.dir(t), id.String())) +func (r *SFTP) renameFile(oldname string, t Type, id ID) error { + newname := r.filename(t, id) + + // create directories if necessary + if t == Data || t == Tree { + err := r.mkdirAll(filepath.Dir(newname), dirMode) + if err != nil { + return err + } + } + + return r.c.Rename(oldname, newname) } // Construct directory for given Type. -func (r *SFTP) dir(t Type) string { +func (r *SFTP) dirname(t Type, id ID) string { var n string switch t { case Data: n = dataPath + if id != nil { + n = filepath.Join(dataPath, fmt.Sprintf("%02x", id[0]), fmt.Sprintf("%02x", id[1])) + } case Snapshot: n = snapshotPath case Tree: n = treePath + if id != nil { + n = filepath.Join(treePath, fmt.Sprintf("%02x", id[0]), fmt.Sprintf("%02x", id[1])) + } case Map: n = mapPath case Lock: @@ -257,7 +301,12 @@ func (r *SFTP) Create(t Type, data []byte) (ID, error) { // check if blob is already present in backend id := IDFromData(data) - if ok, _ := r.Test(t, id); ok { + res, err := r.Test(t, id) + if err != nil { + return nil, arrar.Annotate(err, "test for presence") + } + + if res { return id, ErrAlreadyPresent } @@ -289,7 +338,7 @@ func (r *SFTP) Create(t Type, data []byte) (ID, error) { // Construct path for given Type and ID. func (r *SFTP) filename(t Type, id ID) string { - return filepath.Join(r.dir(t), id.String()) + return filepath.Join(r.dirname(t, id), id.String()) } // Get returns the content stored under the given ID. If the data doesn't match @@ -325,10 +374,15 @@ func (r *SFTP) Test(t Type, id ID) (bool, error) { } }() - if err == nil { - return true, nil + if err != nil { + if _, ok := err.(*sftp.StatusError); ok { + return false, nil + } + + return false, err } - return false, err + + return true, nil } // Remove removes the content stored at ID. @@ -338,9 +392,42 @@ func (r *SFTP) Remove(t Type, id ID) error { // List lists all objects of a given type. func (r *SFTP) List(t Type) (IDs, error) { - list, err := r.c.ReadDir(r.dir(t)) - if err != nil { - return nil, err + list := []os.FileInfo{} + var err error + + if t == Data || t == Tree { + // read first level + basedir := r.dirname(t, nil) + + list1, err := r.c.ReadDir(basedir) + if err != nil { + return nil, err + } + + for _, dir1 := range list1 { + // read second level + list2, err := r.c.ReadDir(filepath.Join(basedir, dir1.Name())) + if err != nil { + return nil, err + } + + // read files + for _, dir2 := range list2 { + entries, err := r.c.ReadDir(filepath.Join(basedir, dir1.Name(), dir2.Name())) + if err != nil { + return nil, err + } + + for _, entry := range entries { + list = append(list, entry) + } + } + } + } else { + list, err = r.c.ReadDir(r.dirname(t, nil)) + if err != nil { + return nil, err + } } ids := make(IDs, 0, len(list))