octoleo/restic
2
2
Fork 0
mirror of https://github.com/octoleo/restic.git synced 2024-11-11 15:51:02 +00:00
Code Issues Releases Wiki Activity

archiver: Improve memory usage, tune buffer pool

Browse Source
This commit is contained in:
Alexander Neumann 2018-04-29 13:20:12 +02:00
parent d80e108b03
commit 400730afca
2 changed files with 14 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
internal/archiver/archiver.go
View File

@ -720,7 +720,11 @@ func (arch *Archiver) loadParentTree(ctx context.Context, snapshotID restic.ID)
// runWorkers starts the worker pools, which are stopped when the context is cancelled. // runWorkers starts the worker pools, which are stopped when the context is cancelled.
func (arch *Archiver) runWorkers(ctx context.Context) { func (arch *Archiver) runWorkers(ctx context.Context) {
arch.blobSaver = NewBlobSaver(ctx, arch.Repo, arch.Options.SaveBlobConcurrency) arch.blobSaver = NewBlobSaver(ctx, arch.Repo, arch.Options.SaveBlobConcurrency)
arch.fileSaver = NewFileSaver(ctx, arch.FS, arch.blobSaver, arch.Repo.Config().ChunkerPolynomial, arch.Options.FileReadConcurrency) arch.fileSaver = NewFileSaver(ctx,
arch.FS,
arch.blobSaver,
arch.Repo.Config().ChunkerPolynomial,
arch.Options.FileReadConcurrency, arch.Options.SaveBlobConcurrency)
arch.fileSaver.CompleteBlob = arch.CompleteBlob arch.fileSaver.CompleteBlob = arch.CompleteBlob
arch.fileSaver.NodeFromFileInfo = arch.nodeFromFileInfo arch.fileSaver.NodeFromFileInfo = arch.nodeFromFileInfo

13
internal/archiver/file_saver.go
View File

@ -63,20 +63,25 @@ type FileSaver struct {
// NewFileSaver returns a new file saver. A worker pool with workers is // NewFileSaver returns a new file saver. A worker pool with workers is
// started, it is stopped when ctx is cancelled. // started, it is stopped when ctx is cancelled.
func NewFileSaver(ctx context.Context, fs fs.FS, blobSaver *BlobSaver, pol chunker.Pol, workers uint) *FileSaver { func NewFileSaver(ctx context.Context, fs fs.FS, blobSaver *BlobSaver, pol chunker.Pol, fileWorkers, blobWorkers uint) *FileSaver {
ch := make(chan saveFileJob, workers) ch := make(chan saveFileJob, fileWorkers)
poolSize := fileWorkers
if blobWorkers > fileWorkers {
poolSize = blobWorkers
}
s := &FileSaver{ s := &FileSaver{
fs: fs, fs: fs,
blobSaver: blobSaver, blobSaver: blobSaver,
saveFilePool: NewBufferPool(ctx, 3*int(workers), chunker.MaxSize/4), saveFilePool: NewBufferPool(ctx, int(poolSize)*3/2, chunker.MaxSize/2),
pol: pol, pol: pol,
ch: ch, ch: ch,
CompleteBlob: func(string, uint64) {}, CompleteBlob: func(string, uint64) {},
} }
for i := uint(0); i < workers; i++ { for i := uint(0); i < fileWorkers; i++ {
s.wg.Add(1) s.wg.Add(1)
go s.worker(ctx, &s.wg, ch) go s.worker(ctx, &s.wg, ch)
} }