2
2
mirror of https://github.com/octoleo/restic.git synced 2024-12-27 12:42:49 +00:00

Remove last ocurrence of Create()

This commit is contained in:
Alexander Neumann 2016-01-24 19:30:14 +01:00
parent 1a95e48389
commit ea29ad6f96
3 changed files with 47 additions and 66 deletions

View File

@ -1,7 +1,7 @@
package pack
import (
"crypto/sha256"
"bytes"
"encoding/binary"
"errors"
"fmt"
@ -12,8 +12,10 @@ import (
"github.com/restic/restic/crypto"
)
// BlobType specifies what a blob stored in a pack is.
type BlobType uint8
// These are the blob types that can be stored in a pack.
const (
Data BlobType = 0
Tree = 1
@ -30,6 +32,7 @@ func (t BlobType) String() string {
return fmt.Sprintf("<BlobType %d>", t)
}
// MarshalJSON encodes the BlobType into JSON.
func (t BlobType) MarshalJSON() ([]byte, error) {
switch t {
case Data:
@ -41,6 +44,7 @@ func (t BlobType) MarshalJSON() ([]byte, error) {
return nil, errors.New("unknown blob type")
}
// UnmarshalJSON decodes the BlobType from JSON.
func (t *BlobType) UnmarshalJSON(buf []byte) error {
switch string(buf) {
case `"data"`:
@ -79,16 +83,15 @@ type Packer struct {
bytes uint
k *crypto.Key
wr io.Writer
hw *backend.HashingWriter
buf *bytes.Buffer
m sync.Mutex
}
// NewPacker returns a new Packer that can be used to pack blobs
// together.
func NewPacker(k *crypto.Key, w io.Writer) *Packer {
return &Packer{k: k, wr: w, hw: backend.NewHashingWriter(w, sha256.New())}
func NewPacker(k *crypto.Key, buf []byte) *Packer {
return &Packer{k: k, buf: bytes.NewBuffer(buf)}
}
// Add saves the data read from rd as a new blob to the packer. Returned is the
@ -99,7 +102,7 @@ func (p *Packer) Add(t BlobType, id backend.ID, rd io.Reader) (int64, error) {
c := Blob{Type: t, ID: id}
n, err := io.Copy(p.hw, rd)
n, err := io.Copy(p.buf, rd)
c.Length = uint(n)
c.Offset = p.bytes
p.bytes += uint(n)
@ -118,45 +121,47 @@ type headerEntry struct {
}
// Finalize writes the header for all added blobs and finalizes the pack.
// Returned are the complete number of bytes written, including the header.
// After Finalize() has finished, the ID of this pack can be obtained by
// calling ID().
func (p *Packer) Finalize() (bytesWritten uint, err error) {
// Returned are all bytes written, including the header.
func (p *Packer) Finalize() ([]byte, error) {
p.m.Lock()
defer p.m.Unlock()
bytesWritten = p.bytes
bytesWritten := p.bytes
// create writer to encrypt header
wr := crypto.EncryptTo(p.k, p.hw)
bytesHeader, err := p.writeHeader(wr)
hdrBuf := bytes.NewBuffer(nil)
bytesHeader, err := p.writeHeader(hdrBuf)
if err != nil {
wr.Close()
return bytesWritten + bytesHeader, err
return nil, err
}
bytesWritten += bytesHeader
// finalize encrypted header
err = wr.Close()
encryptedHeader, err := crypto.Encrypt(p.k, nil, hdrBuf.Bytes())
if err != nil {
return bytesWritten, err
return nil, err
}
// account for crypto overhead
bytesWritten += crypto.Extension
// append the header
n, err := p.buf.Write(encryptedHeader)
if err != nil {
return nil, err
}
hdrBytes := bytesHeader + crypto.Extension
if uint(n) != hdrBytes {
return nil, errors.New("wrong number of bytes written")
}
bytesWritten += hdrBytes
// write length
err = binary.Write(p.hw, binary.LittleEndian, uint32(uint(len(p.blobs))*entrySize+crypto.Extension))
err = binary.Write(p.buf, binary.LittleEndian, uint32(uint(len(p.blobs))*entrySize+crypto.Extension))
if err != nil {
return bytesWritten, err
return nil, err
}
bytesWritten += uint(binary.Size(uint32(0)))
p.bytes = uint(bytesWritten)
return bytesWritten, nil
return p.buf.Bytes(), nil
}
// writeHeader constructs and writes the header to wr.
@ -179,18 +184,6 @@ func (p *Packer) writeHeader(wr io.Writer) (bytesWritten uint, err error) {
return
}
// ID returns the ID of all data written so far.
func (p *Packer) ID() backend.ID {
p.m.Lock()
defer p.m.Unlock()
hash := p.hw.Sum(nil)
id := backend.ID{}
copy(id[:], hash)
return id
}
// Size returns the number of bytes written so far.
func (p *Packer) Size() uint {
p.m.Lock()
@ -215,11 +208,6 @@ func (p *Packer) Blobs() []Blob {
return p.blobs
}
// Writer returns the underlying writer.
func (p *Packer) Writer() io.Writer {
return p.wr
}
func (p *Packer) String() string {
return fmt.Sprintf("<Packer %d blobs, %d bytes>", len(p.blobs), p.bytes)
}

View File

@ -34,23 +34,19 @@ func TestCreatePack(t *testing.T) {
bufs = append(bufs, Buf{data: b, id: h})
}
file := bytes.NewBuffer(nil)
// create random keys
k := crypto.NewRandomKey()
// pack blobs
p := pack.NewPacker(k, file)
p := pack.NewPacker(k, nil)
for _, b := range bufs {
p.Add(pack.Tree, b.id, bytes.NewReader(b.data))
}
// write file
n, err := p.Finalize()
packData, err := p.Finalize()
OK(t, err)
written := 0
// data
for _, l := range lengths {
written += l
}
@ -62,11 +58,11 @@ func TestCreatePack(t *testing.T) {
written += crypto.Extension
// check length
Equals(t, uint(written), n)
Equals(t, written, len(packData))
Equals(t, uint(written), p.Size())
// read and parse it again
rd := bytes.NewReader(file.Bytes())
rd := bytes.NewReader(packData)
np, err := pack.NewUnpacker(k, rd)
OK(t, err)
Equals(t, len(np.Entries), len(bufs))

View File

@ -42,12 +42,8 @@ func (r *packerManager) findPacker(size uint) (*pack.Packer, error) {
}
// no suitable packer found, return new
blob, err := r.be.Create()
if err != nil {
return nil, err
}
debug.Log("Repo.findPacker", "create new pack %p for %d bytes", blob, size)
return pack.NewPacker(r.key, blob), nil
debug.Log("Repo.findPacker", "create new pack for %d bytes", size)
return pack.NewPacker(r.key, nil), nil
}
// insertPacker appends p to s.packs.
@ -62,28 +58,29 @@ func (r *packerManager) insertPacker(p *pack.Packer) {
// savePacker stores p in the backend.
func (r *Repository) savePacker(p *pack.Packer) error {
debug.Log("Repo.savePacker", "save packer with %d blobs\n", p.Count())
_, err := p.Finalize()
data, err := p.Finalize()
if err != nil {
return err
}
// move file to the final location
sid := p.ID()
err = p.Writer().(backend.Blob).Finalize(backend.Data, sid.String())
id := backend.Hash(data)
h := backend.Handle{Type: backend.Data, Name: id.String()}
err = r.be.Save(h, data)
if err != nil {
debug.Log("Repo.savePacker", "blob Finalize() error: %v", err)
debug.Log("Repo.savePacker", "Save(%v) error: %v", h, err)
return err
}
debug.Log("Repo.savePacker", "saved as %v", sid.Str())
debug.Log("Repo.savePacker", "saved as %v", h)
// update blobs in the index
for _, b := range p.Blobs() {
debug.Log("Repo.savePacker", " updating blob %v to pack %v", b.ID.Str(), sid.Str())
debug.Log("Repo.savePacker", " updating blob %v to pack %v", b.ID.Str(), id.Str())
r.idx.Current().Store(PackedBlob{
Type: b.Type,
ID: b.ID,
PackID: sid,
PackID: id,
Offset: b.Offset,
Length: uint(b.Length),
})