package backend import ( "errors" "fmt" "hash" "io" "io/ioutil" "os" "path/filepath" "strconv" "strings" "github.com/juju/arrar" ) const ( dirMode = 0700 dataPath = "data" snapshotPath = "snapshots" treePath = "trees" lockPath = "locks" keyPath = "keys" tempPath = "tmp" versionFileName = "version" ) var ErrWrongData = errors.New("wrong data returned by backend, checksum does not match") type Local struct { p string ver uint } // OpenLocal opens the local backend at dir. func OpenLocal(dir string) (*Local, error) { items := []string{ dir, filepath.Join(dir, dataPath), filepath.Join(dir, snapshotPath), filepath.Join(dir, treePath), filepath.Join(dir, lockPath), filepath.Join(dir, keyPath), filepath.Join(dir, tempPath), } // test if all necessary dirs and files are there for _, d := range items { if _, err := os.Stat(d); err != nil { return nil, fmt.Errorf("%s does not exist", d) } } // read version file f, err := os.Open(filepath.Join(dir, versionFileName)) if err != nil { return nil, fmt.Errorf("unable to read version file: %v\n", err) } buf := make([]byte, 100) n, err := f.Read(buf) if err != nil { return nil, err } err = f.Close() if err != nil { return nil, err } version, err := strconv.Atoi(strings.TrimSpace(string(buf[:n]))) if err != nil { return nil, fmt.Errorf("unable to convert version to integer: %v\n", err) } // check version if version != BackendVersion { return nil, fmt.Errorf("wrong version %d", version) } return &Local{p: dir, ver: uint(version)}, nil } // CreateLocal creates all the necessary files and directories for a new local // backend at dir. func CreateLocal(dir string) (*Local, error) { versionFile := filepath.Join(dir, versionFileName) dirs := []string{ dir, filepath.Join(dir, dataPath), filepath.Join(dir, snapshotPath), filepath.Join(dir, treePath), filepath.Join(dir, lockPath), filepath.Join(dir, keyPath), filepath.Join(dir, tempPath), } // test if version file already exists _, err := os.Lstat(versionFile) if err == nil { return nil, errors.New("version file already exists") } // test if directories already exist for _, d := range dirs[1:] { if _, err := os.Stat(d); err == nil { return nil, fmt.Errorf("dir %s already exists", d) } } // create paths for data, refs and temp for _, d := range dirs { err := os.MkdirAll(d, dirMode) if err != nil { return nil, err } } // create version file f, err := os.Create(versionFile) if err != nil { return nil, err } _, err = f.Write([]byte(fmt.Sprintf("%d\n", BackendVersion))) if err != nil { return nil, err } err = f.Close() if err != nil { return nil, err } // open backend return OpenLocal(dir) } // Location returns this backend's location (the directory name). func (b *Local) Location() string { return b.p } // Return temp directory in correct directory for this backend. func (b *Local) tempFile() (*os.File, error) { return ioutil.TempFile(filepath.Join(b.p, tempPath), "temp-") } // Rename temp file to final name according to type and ID. func (b *Local) renameFile(file *os.File, t Type, id ID) error { newname := b.filename(t, id) oldname := file.Name() if t == Data || t == Tree { // create directories if necessary, ignore errors os.MkdirAll(filepath.Dir(newname), dirMode) } return os.Rename(oldname, newname) } // Construct directory for given Type. func (b *Local) dirname(t Type, id ID) string { var n string switch t { case Data: n = dataPath if id != nil { n = filepath.Join(dataPath, fmt.Sprintf("%02x", id[0])) } case Snapshot: n = snapshotPath case Tree: n = treePath if id != nil { n = filepath.Join(treePath, fmt.Sprintf("%02x", id[0])) } case Lock: n = lockPath case Key: n = keyPath } return filepath.Join(b.p, n) } // Create stores new content of type t and data and returns the ID. If the blob // is already present, returns ErrAlreadyPresent and the blob's ID. func (b *Local) Create(t Type, data []byte) (ID, error) { // TODO: make sure that tempfile is removed upon error // check if blob is already present in backend id := IDFromData(data) res, err := b.Test(t, id) if err != nil { return nil, arrar.Annotate(err, "test for presence") } if res { return id, ErrAlreadyPresent } // create tempfile in backend file, err := b.tempFile() if err != nil { return nil, err } // write data to tempfile _, err = file.Write(data) if err != nil { return nil, err } err = file.Close() if err != nil { return nil, err } // return id err = b.renameFile(file, t, id) if err != nil { return nil, err } return id, nil } // CreateFrom reads content from rd and stores it as type t. Returned is the // storage ID. If the blob is already present, returns ErrAlreadyPresent and // the blob's ID. func (b *Local) CreateFrom(t Type, rd io.Reader) (ID, error) { // TODO: make sure that tempfile is removed upon error // check hash while writing hr := NewHashingReader(rd, newHash()) // create tempfile in backend file, err := b.tempFile() if err != nil { return nil, err } // write data to tempfile _, err = io.Copy(file, hr) if err != nil { return nil, err } err = file.Close() if err != nil { return nil, err } // get ID id := ID(hr.Sum(nil)) // check for duplicate ID res, err := b.Test(t, id) if err != nil { return nil, arrar.Annotate(err, "test for presence") } if res { return id, ErrAlreadyPresent } // rename file err = b.renameFile(file, t, id) if err != nil { return nil, err } return id, nil } type localBlob struct { f *os.File h hash.Hash tw io.Writer backend *Local tpe Type id ID } func (lb *localBlob) Close() error { err := lb.f.Close() if err != nil { return err } // get ID lb.id = ID(lb.h.Sum(nil)) // check for duplicate ID res, err := lb.backend.Test(lb.tpe, lb.id) if err != nil { return fmt.Errorf("testing presence of ID %v failed: %v", lb.id, err) } if res { return ErrAlreadyPresent } // rename file err = lb.backend.renameFile(lb.f, lb.tpe, lb.id) if err != nil { return err } return nil } func (lb *localBlob) Write(p []byte) (int, error) { return lb.tw.Write(p) } func (lb *localBlob) ID() (ID, error) { if lb.id == nil { return nil, errors.New("blob is not closed, ID unavailable") } return lb.id, nil } // Create creates a new blob of type t. Blob implements io.WriteCloser. Once // Close() has been called, ID() can be used to retrieve the ID. If the blob is // already present, Close() returns ErrAlreadyPresent. func (b *Local) CreateBlob(t Type) (Blob, error) { // TODO: make sure that tempfile is removed upon error // create tempfile in backend file, err := b.tempFile() if err != nil { return nil, err } h := newHash() blob := localBlob{ h: h, tw: io.MultiWriter(h, file), f: file, backend: b, tpe: t, } return &blob, nil } // Construct path for given Type and ID. func (b *Local) filename(t Type, id ID) string { return filepath.Join(b.dirname(t, id), id.String()) } // Get returns the content stored under the given ID. If the data doesn't match // the requested ID, ErrWrongData is returned. func (b *Local) Get(t Type, id ID) ([]byte, error) { if id == nil { return nil, errors.New("unable to load nil ID") } // try to open file file, err := os.Open(b.filename(t, id)) defer file.Close() if err != nil { return nil, err } // read all buf, err := ioutil.ReadAll(file) if err != nil { return nil, err } // check id if !Hash(buf).Equal(id) { return nil, ErrWrongData } return buf, nil } // GetReader returns a reader that yields the content stored under the given // ID. The content is not verified. The reader should be closed after draining // it. func (b *Local) GetReader(t Type, id ID) (io.ReadCloser, error) { if id == nil { return nil, errors.New("unable to load nil ID") } // try to open file file, err := os.Open(b.filename(t, id)) if err != nil { return nil, err } return file, nil } // Test returns true if a blob of the given type and ID exists in the backend. func (b *Local) Test(t Type, id ID) (bool, error) { // try to open file file, err := os.Open(b.filename(t, id)) defer func() { file.Close() }() if err != nil { if os.IsNotExist(err) { return false, nil } return false, err } return true, nil } // Remove removes the content stored at ID. func (b *Local) Remove(t Type, id ID) error { return os.Remove(b.filename(t, id)) } // List lists all objects of a given type. func (b *Local) List(t Type) (IDs, error) { // TODO: use os.Open() and d.Readdirnames() instead of Glob() var pattern string if t == Data || t == Tree { pattern = filepath.Join(b.dirname(t, nil), "*", "*") } else { pattern = filepath.Join(b.dirname(t, nil), "*") } matches, err := filepath.Glob(pattern) if err != nil { return nil, err } ids := make(IDs, 0, len(matches)) for _, m := range matches { base := filepath.Base(m) if base == "" { continue } id, err := ParseID(base) if err != nil { continue } ids = append(ids, id) } return ids, nil } // Version returns the version of this local backend. func (b *Local) Version() uint { return b.ver } // Close closes the backend func (b *Local) Close() error { return nil } // Delete removes the repository and all files. func (b *Local) Delete() error { return os.RemoveAll(b.p) }