2015-11-18 19:20:25 +00:00
|
|
|
package repository
|
|
|
|
|
|
|
|
import (
|
2016-03-06 11:26:25 +00:00
|
|
|
"fmt"
|
|
|
|
"io"
|
|
|
|
"io/ioutil"
|
|
|
|
"os"
|
2015-11-18 19:20:25 +00:00
|
|
|
"sync"
|
|
|
|
|
2016-02-14 14:29:28 +00:00
|
|
|
"restic/backend"
|
|
|
|
"restic/crypto"
|
|
|
|
"restic/debug"
|
|
|
|
"restic/pack"
|
2015-11-18 19:20:25 +00:00
|
|
|
)
|
|
|
|
|
2016-03-05 14:58:39 +00:00
|
|
|
// Saver implements saving data in a backend.
|
|
|
|
type Saver interface {
|
2016-03-06 11:26:25 +00:00
|
|
|
Save(h backend.Handle, jp []byte) error
|
2016-03-05 14:58:39 +00:00
|
|
|
}
|
|
|
|
|
2015-11-18 19:20:25 +00:00
|
|
|
// packerManager keeps a list of open packs and creates new on demand.
|
|
|
|
type packerManager struct {
|
2016-03-05 14:58:39 +00:00
|
|
|
be Saver
|
2015-11-18 19:20:25 +00:00
|
|
|
key *crypto.Key
|
|
|
|
pm sync.Mutex
|
|
|
|
packs []*pack.Packer
|
2016-03-06 11:26:25 +00:00
|
|
|
|
|
|
|
tempdir string
|
2015-11-18 19:20:25 +00:00
|
|
|
}
|
|
|
|
|
2016-02-22 20:09:21 +00:00
|
|
|
const minPackSize = 4 * 1024 * 1024
|
|
|
|
const maxPackSize = 16 * 1024 * 1024
|
2015-11-18 19:20:25 +00:00
|
|
|
const maxPackers = 200
|
|
|
|
|
2016-03-06 11:26:25 +00:00
|
|
|
// NewPackerManager returns an new packer manager which writes temporary files
|
|
|
|
// to a temporary directory
|
|
|
|
func NewPackerManager(be Saver, key *crypto.Key) (pm *packerManager, err error) {
|
|
|
|
pm = &packerManager{
|
|
|
|
be: be,
|
|
|
|
key: key,
|
|
|
|
}
|
|
|
|
|
|
|
|
pm.tempdir, err = ioutil.TempDir("", fmt.Sprintf("restic-packs-%d-", os.Getpid()))
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
return pm, nil
|
|
|
|
}
|
|
|
|
|
2015-11-18 19:20:25 +00:00
|
|
|
// findPacker returns a packer for a new blob of size bytes. Either a new one is
|
|
|
|
// created or one is returned that already has some blobs.
|
|
|
|
func (r *packerManager) findPacker(size uint) (*pack.Packer, error) {
|
|
|
|
r.pm.Lock()
|
|
|
|
defer r.pm.Unlock()
|
|
|
|
|
|
|
|
// search for a suitable packer
|
|
|
|
if len(r.packs) > 0 {
|
|
|
|
debug.Log("Repo.findPacker", "searching packer for %d bytes\n", size)
|
|
|
|
for i, p := range r.packs {
|
|
|
|
if p.Size()+size < maxPackSize {
|
|
|
|
debug.Log("Repo.findPacker", "found packer %v", p)
|
|
|
|
// remove from list
|
|
|
|
r.packs = append(r.packs[:i], r.packs[i+1:]...)
|
|
|
|
return p, nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// no suitable packer found, return new
|
2016-01-24 18:30:14 +00:00
|
|
|
debug.Log("Repo.findPacker", "create new pack for %d bytes", size)
|
2016-03-06 11:26:25 +00:00
|
|
|
tmpfile, err := ioutil.TempFile(r.tempdir, "restic-pack-")
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
return pack.NewPacker(r.key, tmpfile), nil
|
2015-11-18 19:20:25 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// insertPacker appends p to s.packs.
|
|
|
|
func (r *packerManager) insertPacker(p *pack.Packer) {
|
|
|
|
r.pm.Lock()
|
|
|
|
defer r.pm.Unlock()
|
|
|
|
|
|
|
|
r.packs = append(r.packs, p)
|
|
|
|
debug.Log("Repo.insertPacker", "%d packers\n", len(r.packs))
|
|
|
|
}
|
|
|
|
|
|
|
|
// savePacker stores p in the backend.
|
|
|
|
func (r *Repository) savePacker(p *pack.Packer) error {
|
|
|
|
debug.Log("Repo.savePacker", "save packer with %d blobs\n", p.Count())
|
2016-03-06 11:26:25 +00:00
|
|
|
n, err := p.Finalize()
|
2015-11-18 19:20:25 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2016-03-06 11:26:25 +00:00
|
|
|
tmpfile := p.Writer().(*os.File)
|
|
|
|
f, err := os.Open(tmpfile.Name())
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
data := make([]byte, n)
|
|
|
|
m, err := io.ReadFull(f, data)
|
|
|
|
|
|
|
|
if uint(m) != n {
|
|
|
|
return fmt.Errorf("read wrong number of bytes from %v: want %v, got %v", tmpfile.Name(), n, m)
|
|
|
|
}
|
|
|
|
|
|
|
|
if err = f.Close(); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2016-01-24 18:30:14 +00:00
|
|
|
id := backend.Hash(data)
|
|
|
|
h := backend.Handle{Type: backend.Data, Name: id.String()}
|
|
|
|
|
|
|
|
err = r.be.Save(h, data)
|
2015-11-18 19:20:25 +00:00
|
|
|
if err != nil {
|
2016-01-24 18:30:14 +00:00
|
|
|
debug.Log("Repo.savePacker", "Save(%v) error: %v", h, err)
|
2015-11-18 19:20:25 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2016-01-24 18:30:14 +00:00
|
|
|
debug.Log("Repo.savePacker", "saved as %v", h)
|
2015-11-18 19:20:25 +00:00
|
|
|
|
|
|
|
// update blobs in the index
|
|
|
|
for _, b := range p.Blobs() {
|
2016-01-24 18:30:14 +00:00
|
|
|
debug.Log("Repo.savePacker", " updating blob %v to pack %v", b.ID.Str(), id.Str())
|
2015-11-18 19:20:25 +00:00
|
|
|
r.idx.Current().Store(PackedBlob{
|
|
|
|
Type: b.Type,
|
|
|
|
ID: b.ID,
|
2016-01-24 18:30:14 +00:00
|
|
|
PackID: id,
|
2015-11-18 19:20:25 +00:00
|
|
|
Offset: b.Offset,
|
|
|
|
Length: uint(b.Length),
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// countPacker returns the number of open (unfinished) packers.
|
|
|
|
func (r *packerManager) countPacker() int {
|
|
|
|
r.pm.Lock()
|
|
|
|
defer r.pm.Unlock()
|
|
|
|
|
|
|
|
return len(r.packs)
|
|
|
|
}
|
2016-03-06 11:26:25 +00:00
|
|
|
|
|
|
|
// removeTempdir deletes the temporary directory.
|
|
|
|
func (r *packerManager) removeTempdir() error {
|
|
|
|
err := os.RemoveAll(r.tempdir)
|
|
|
|
r.tempdir = ""
|
|
|
|
return err
|
|
|
|
}
|