2015-10-12 20:34:12 +00:00
|
|
|
package repository
|
|
|
|
|
|
|
|
import (
|
2017-06-18 12:45:02 +00:00
|
|
|
"context"
|
2015-10-12 20:34:12 +00:00
|
|
|
"sync"
|
|
|
|
|
2017-07-24 15:42:25 +00:00
|
|
|
"github.com/restic/restic/internal/restic"
|
2017-07-23 12:21:03 +00:00
|
|
|
|
|
|
|
"github.com/restic/restic/internal/debug"
|
2015-10-12 20:34:12 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
// MasterIndex is a collection of indexes and IDs of chunks that are in the process of being saved.
|
|
|
|
type MasterIndex struct {
|
2020-06-06 20:20:44 +00:00
|
|
|
idx []*Index
|
|
|
|
pendingBlobs restic.BlobSet
|
|
|
|
idxMutex sync.RWMutex
|
2015-10-12 20:34:12 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// NewMasterIndex creates a new master index.
|
|
|
|
func NewMasterIndex() *MasterIndex {
|
2020-07-04 05:06:14 +00:00
|
|
|
// Always add an empty final index, such that MergeFinalIndexes can merge into this.
|
|
|
|
// Note that removing this index could lead to a race condition in the rare
|
|
|
|
// sitation that only two indexes exist which are saved and merged concurrently.
|
|
|
|
idx := []*Index{NewIndex()}
|
|
|
|
idx[0].Finalize()
|
|
|
|
return &MasterIndex{idx: idx, pendingBlobs: restic.NewBlobSet()}
|
2015-10-12 20:34:12 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Lookup queries all known Indexes for the ID and returns the first match.
|
2018-01-12 06:20:12 +00:00
|
|
|
func (mi *MasterIndex) Lookup(id restic.ID, tpe restic.BlobType) (blobs []restic.PackedBlob, found bool) {
|
2015-10-12 20:34:12 +00:00
|
|
|
mi.idxMutex.RLock()
|
|
|
|
defer mi.idxMutex.RUnlock()
|
|
|
|
|
|
|
|
for _, idx := range mi.idx {
|
2018-01-12 06:20:12 +00:00
|
|
|
blobs, found = idx.Lookup(id, tpe)
|
|
|
|
if found {
|
2015-10-12 20:34:12 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-01-12 06:20:12 +00:00
|
|
|
return nil, false
|
2015-10-12 20:34:12 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// LookupSize queries all known Indexes for the ID and returns the first match.
|
2018-01-12 06:20:12 +00:00
|
|
|
func (mi *MasterIndex) LookupSize(id restic.ID, tpe restic.BlobType) (uint, bool) {
|
2015-10-12 20:34:12 +00:00
|
|
|
mi.idxMutex.RLock()
|
|
|
|
defer mi.idxMutex.RUnlock()
|
|
|
|
|
|
|
|
for _, idx := range mi.idx {
|
2018-01-12 06:20:12 +00:00
|
|
|
if size, found := idx.LookupSize(id, tpe); found {
|
|
|
|
return size, found
|
2015-10-12 20:34:12 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-01-12 06:20:12 +00:00
|
|
|
return 0, false
|
2015-10-12 20:34:12 +00:00
|
|
|
}
|
|
|
|
|
2015-11-01 21:32:28 +00:00
|
|
|
// ListPack returns the list of blobs in a pack. The first matching index is
|
|
|
|
// returned, or nil if no index contains information about the pack id.
|
2016-08-31 20:39:36 +00:00
|
|
|
func (mi *MasterIndex) ListPack(id restic.ID) (list []restic.PackedBlob) {
|
2015-11-01 21:32:28 +00:00
|
|
|
mi.idxMutex.RLock()
|
|
|
|
defer mi.idxMutex.RUnlock()
|
|
|
|
|
|
|
|
for _, idx := range mi.idx {
|
|
|
|
list := idx.ListPack(id)
|
|
|
|
if len(list) > 0 {
|
|
|
|
return list
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2020-06-06 20:20:44 +00:00
|
|
|
// AddPending adds a given blob to list of pending Blobs
|
|
|
|
// Before doing so it checks if this blob is already known.
|
|
|
|
// Returns true if adding was successful and false if the blob
|
|
|
|
// was already known
|
|
|
|
func (mi *MasterIndex) addPending(id restic.ID, tpe restic.BlobType) bool {
|
|
|
|
|
|
|
|
mi.idxMutex.Lock()
|
|
|
|
defer mi.idxMutex.Unlock()
|
|
|
|
|
|
|
|
// Check if blob is pending or in index
|
|
|
|
if mi.pendingBlobs.Has(restic.BlobHandle{ID: id, Type: tpe}) {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, idx := range mi.idx {
|
|
|
|
if idx.Has(id, tpe) {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// really not known -> insert
|
|
|
|
mi.pendingBlobs.Insert(restic.BlobHandle{ID: id, Type: tpe})
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
2015-10-12 20:34:12 +00:00
|
|
|
// Has queries all known Indexes for the ID and returns the first match.
|
2020-06-06 20:20:44 +00:00
|
|
|
// Also returns true if the ID is pending.
|
2016-08-31 18:58:57 +00:00
|
|
|
func (mi *MasterIndex) Has(id restic.ID, tpe restic.BlobType) bool {
|
2015-10-12 20:34:12 +00:00
|
|
|
mi.idxMutex.RLock()
|
|
|
|
defer mi.idxMutex.RUnlock()
|
|
|
|
|
2020-06-06 20:20:44 +00:00
|
|
|
// also return true if blob is pending
|
|
|
|
if mi.pendingBlobs.Has(restic.BlobHandle{ID: id, Type: tpe}) {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
2015-10-12 20:34:12 +00:00
|
|
|
for _, idx := range mi.idx {
|
2016-08-03 20:38:05 +00:00
|
|
|
if idx.Has(id, tpe) {
|
2015-10-12 20:34:12 +00:00
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
|
|
|
// Count returns the number of blobs of type t in the index.
|
2016-08-31 18:58:57 +00:00
|
|
|
func (mi *MasterIndex) Count(t restic.BlobType) (n uint) {
|
2015-10-12 20:34:12 +00:00
|
|
|
mi.idxMutex.RLock()
|
|
|
|
defer mi.idxMutex.RUnlock()
|
|
|
|
|
|
|
|
var sum uint
|
|
|
|
for _, idx := range mi.idx {
|
|
|
|
sum += idx.Count(t)
|
|
|
|
}
|
|
|
|
|
|
|
|
return sum
|
|
|
|
}
|
|
|
|
|
|
|
|
// Insert adds a new index to the MasterIndex.
|
|
|
|
func (mi *MasterIndex) Insert(idx *Index) {
|
|
|
|
mi.idxMutex.Lock()
|
|
|
|
defer mi.idxMutex.Unlock()
|
|
|
|
|
|
|
|
mi.idx = append(mi.idx, idx)
|
|
|
|
}
|
|
|
|
|
2017-01-02 13:14:51 +00:00
|
|
|
// Store remembers the id and pack in the index.
|
2020-06-06 20:20:44 +00:00
|
|
|
func (mi *MasterIndex) StorePack(id restic.ID, blobs []restic.Blob) {
|
2017-10-07 12:11:42 +00:00
|
|
|
mi.idxMutex.Lock()
|
|
|
|
defer mi.idxMutex.Unlock()
|
|
|
|
|
2020-06-06 20:20:44 +00:00
|
|
|
// delete blobs from pending
|
|
|
|
for _, blob := range blobs {
|
|
|
|
mi.pendingBlobs.Delete(restic.BlobHandle{Type: blob.Type, ID: blob.ID})
|
|
|
|
}
|
|
|
|
|
2015-10-12 20:34:12 +00:00
|
|
|
for _, idx := range mi.idx {
|
|
|
|
if !idx.Final() {
|
2020-06-06 20:20:44 +00:00
|
|
|
idx.StorePack(id, blobs)
|
2017-01-02 13:14:51 +00:00
|
|
|
return
|
2015-10-12 20:34:12 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
newIdx := NewIndex()
|
2020-06-06 20:20:44 +00:00
|
|
|
newIdx.StorePack(id, blobs)
|
2015-10-12 20:34:12 +00:00
|
|
|
mi.idx = append(mi.idx, newIdx)
|
|
|
|
}
|
|
|
|
|
2020-06-06 20:20:44 +00:00
|
|
|
// FinalizeNotFinalIndexes finalizes all indexes that
|
|
|
|
// have not yet been saved and returns that list
|
|
|
|
func (mi *MasterIndex) FinalizeNotFinalIndexes() []*Index {
|
2015-10-12 20:34:12 +00:00
|
|
|
mi.idxMutex.Lock()
|
|
|
|
defer mi.idxMutex.Unlock()
|
|
|
|
|
|
|
|
var list []*Index
|
|
|
|
|
|
|
|
for _, idx := range mi.idx {
|
|
|
|
if !idx.Final() {
|
2020-06-06 20:20:44 +00:00
|
|
|
idx.Finalize()
|
2015-10-12 20:34:12 +00:00
|
|
|
list = append(list, idx)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-09-27 20:35:08 +00:00
|
|
|
debug.Log("return %d indexes", len(list))
|
2015-10-12 21:59:17 +00:00
|
|
|
return list
|
|
|
|
}
|
|
|
|
|
2020-06-06 20:20:44 +00:00
|
|
|
// FinalizeFullIndexes finalizes all indexes that are full and returns that list.
|
|
|
|
func (mi *MasterIndex) FinalizeFullIndexes() []*Index {
|
2015-10-12 21:59:17 +00:00
|
|
|
mi.idxMutex.Lock()
|
|
|
|
defer mi.idxMutex.Unlock()
|
|
|
|
|
|
|
|
var list []*Index
|
|
|
|
|
2016-09-27 20:35:08 +00:00
|
|
|
debug.Log("checking %d indexes", len(mi.idx))
|
2015-10-12 21:59:17 +00:00
|
|
|
for _, idx := range mi.idx {
|
|
|
|
if idx.Final() {
|
2016-09-27 20:35:08 +00:00
|
|
|
debug.Log("index %p is final", idx)
|
2015-10-12 21:59:17 +00:00
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2015-10-25 14:05:22 +00:00
|
|
|
if IndexFull(idx) {
|
2016-09-27 20:35:08 +00:00
|
|
|
debug.Log("index %p is full", idx)
|
2020-06-06 20:20:44 +00:00
|
|
|
idx.Finalize()
|
2015-10-12 21:59:17 +00:00
|
|
|
list = append(list, idx)
|
|
|
|
} else {
|
2016-09-27 20:35:08 +00:00
|
|
|
debug.Log("index %p not full", idx)
|
2015-10-12 21:59:17 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-09-27 20:35:08 +00:00
|
|
|
debug.Log("return %d indexes", len(list))
|
2015-10-12 20:34:12 +00:00
|
|
|
return list
|
|
|
|
}
|
|
|
|
|
|
|
|
// All returns all indexes.
|
|
|
|
func (mi *MasterIndex) All() []*Index {
|
|
|
|
mi.idxMutex.Lock()
|
|
|
|
defer mi.idxMutex.Unlock()
|
|
|
|
|
|
|
|
return mi.idx
|
|
|
|
}
|
2015-11-02 18:28:30 +00:00
|
|
|
|
2017-06-18 12:45:02 +00:00
|
|
|
// Each returns a channel that yields all blobs known to the index. When the
|
|
|
|
// context is cancelled, the background goroutine terminates. This blocks any
|
|
|
|
// modification of the index.
|
|
|
|
func (mi *MasterIndex) Each(ctx context.Context) <-chan restic.PackedBlob {
|
|
|
|
mi.idxMutex.RLock()
|
|
|
|
|
|
|
|
ch := make(chan restic.PackedBlob)
|
|
|
|
|
|
|
|
go func() {
|
|
|
|
defer mi.idxMutex.RUnlock()
|
|
|
|
defer func() {
|
|
|
|
close(ch)
|
|
|
|
}()
|
|
|
|
|
|
|
|
for _, idx := range mi.idx {
|
|
|
|
idxCh := idx.Each(ctx)
|
|
|
|
for pb := range idxCh {
|
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
|
|
|
return
|
|
|
|
case ch <- pb:
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
|
|
|
|
return ch
|
|
|
|
}
|
|
|
|
|
2020-07-04 05:06:14 +00:00
|
|
|
// MergeFinalIndexes merges all final indexes together.
|
|
|
|
// After calling, there will be only one big final index in MasterIndex
|
|
|
|
// containing all final index contents.
|
|
|
|
// Indexes that are not final are left untouched.
|
|
|
|
// This merging can only be called after all index files are loaded - as
|
|
|
|
// removing of superseded index contents is only possible for unmerged indexes.
|
|
|
|
func (mi *MasterIndex) MergeFinalIndexes() {
|
|
|
|
mi.idxMutex.Lock()
|
|
|
|
defer mi.idxMutex.Unlock()
|
|
|
|
|
|
|
|
// The first index is always final and the one to merge into
|
|
|
|
newIdx := mi.idx[:1]
|
|
|
|
for i := 1; i < len(mi.idx); i++ {
|
|
|
|
idx := mi.idx[i]
|
|
|
|
// clear reference in masterindex as it may become stale
|
|
|
|
mi.idx[i] = nil
|
|
|
|
if !idx.Final() {
|
|
|
|
newIdx = append(newIdx, idx)
|
|
|
|
} else {
|
|
|
|
mi.idx[0].merge(idx)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
mi.idx = newIdx
|
|
|
|
}
|
|
|
|
|
2015-11-02 18:28:30 +00:00
|
|
|
// RebuildIndex combines all known indexes to a new index, leaving out any
|
|
|
|
// packs whose ID is contained in packBlacklist. The new index contains the IDs
|
|
|
|
// of all known indexes in the "supersedes" field.
|
2016-08-31 18:29:54 +00:00
|
|
|
func (mi *MasterIndex) RebuildIndex(packBlacklist restic.IDSet) (*Index, error) {
|
2015-11-02 18:28:30 +00:00
|
|
|
mi.idxMutex.Lock()
|
|
|
|
defer mi.idxMutex.Unlock()
|
|
|
|
|
2016-09-27 20:35:08 +00:00
|
|
|
debug.Log("start rebuilding index of %d indexes, pack blacklist: %v", len(mi.idx), packBlacklist)
|
2015-11-02 18:28:30 +00:00
|
|
|
|
|
|
|
newIndex := NewIndex()
|
2017-06-18 12:45:02 +00:00
|
|
|
|
|
|
|
ctx, cancel := context.WithCancel(context.TODO())
|
|
|
|
defer cancel()
|
2015-11-02 18:28:30 +00:00
|
|
|
|
|
|
|
for i, idx := range mi.idx {
|
2016-09-27 20:35:08 +00:00
|
|
|
debug.Log("adding index %d", i)
|
2015-11-02 18:28:30 +00:00
|
|
|
|
2017-06-18 12:45:02 +00:00
|
|
|
for pb := range idx.Each(ctx) {
|
2015-11-02 18:28:30 +00:00
|
|
|
if packBlacklist.Has(pb.PackID) {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
newIndex.Store(pb)
|
|
|
|
}
|
|
|
|
|
|
|
|
if !idx.Final() {
|
2016-09-27 20:35:08 +00:00
|
|
|
debug.Log("index %d isn't final, don't add to supersedes field", i)
|
2015-11-02 18:28:30 +00:00
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2020-07-04 05:06:14 +00:00
|
|
|
ids, err := idx.IDs()
|
2015-11-02 18:28:30 +00:00
|
|
|
if err != nil {
|
2016-09-27 20:35:08 +00:00
|
|
|
debug.Log("index %d does not have an ID: %v", err)
|
2015-11-02 18:28:30 +00:00
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2020-07-04 05:06:14 +00:00
|
|
|
debug.Log("adding index ids %v to supersedes field", ids)
|
2015-11-02 18:28:30 +00:00
|
|
|
|
2020-07-04 05:06:14 +00:00
|
|
|
err = newIndex.AddToSupersedes(ids...)
|
2015-11-02 18:28:30 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return newIndex, nil
|
|
|
|
}
|