mirror of
https://github.com/octoleo/restic.git
synced 2024-11-02 11:46:36 +00:00
Merge pull request #792 from restic/fix-791
s3: Increase MaxIdleConnsPerHost
This commit is contained in:
commit
83538c745a
@ -14,11 +14,11 @@
|
||||
# docker run --rm -v $PWD:/home/travis/restic restic/test gb test -v ./backend
|
||||
#
|
||||
# build the image for an older version of Go:
|
||||
# docker build --build-arg GOVERSION=1.3.3 -t restic/test:go1.3.3 .
|
||||
# docker build --build-arg GOVERSION=1.6.4 -t restic/test:go1.6.4 .
|
||||
|
||||
FROM ubuntu:14.04
|
||||
|
||||
ARG GOVERSION=1.7
|
||||
ARG GOVERSION=1.7.5
|
||||
ARG GOARCH=amd64
|
||||
|
||||
# install dependencies
|
||||
|
@ -17,7 +17,7 @@ import (
|
||||
"restic/backend"
|
||||
)
|
||||
|
||||
const connLimit = 10
|
||||
const connLimit = 40
|
||||
|
||||
// make sure the rest backend implements restic.Backend
|
||||
var _ restic.Backend = &restBackend{}
|
||||
|
@ -3,6 +3,7 @@ package s3
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"net/http"
|
||||
"path"
|
||||
"restic"
|
||||
"strings"
|
||||
@ -15,7 +16,7 @@ import (
|
||||
"restic/debug"
|
||||
)
|
||||
|
||||
const connLimit = 10
|
||||
const connLimit = 40
|
||||
|
||||
// s3 is a backend which stores the data on an S3 endpoint.
|
||||
type s3 struct {
|
||||
@ -36,6 +37,10 @@ func Open(cfg Config) (restic.Backend, error) {
|
||||
}
|
||||
|
||||
be := &s3{client: client, bucketname: cfg.Bucket, prefix: cfg.Prefix}
|
||||
|
||||
tr := &http.Transport{MaxIdleConnsPerHost: connLimit}
|
||||
client.SetCustomTransport(tr)
|
||||
|
||||
be.createConnections()
|
||||
|
||||
found, err := client.BucketExists(cfg.Bucket)
|
||||
@ -104,6 +109,18 @@ func (be *s3) Save(h restic.Handle, rd io.Reader) (err error) {
|
||||
return errors.Wrap(err, "client.PutObject")
|
||||
}
|
||||
|
||||
// wrapReader wraps an io.ReadCloser to run an additional function on Close.
|
||||
type wrapReader struct {
|
||||
io.ReadCloser
|
||||
f func()
|
||||
}
|
||||
|
||||
func (wr wrapReader) Close() error {
|
||||
err := wr.ReadCloser.Close()
|
||||
wr.f()
|
||||
return err
|
||||
}
|
||||
|
||||
// Load returns a reader that yields the contents of the file at h at the
|
||||
// given offset. If length is nonzero, only a portion of the file is
|
||||
// returned. rd must be closed after use.
|
||||
@ -125,29 +142,49 @@ func (be *s3) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, er
|
||||
|
||||
objName := be.s3path(h)
|
||||
|
||||
// get token for connection
|
||||
<-be.connChan
|
||||
defer func() {
|
||||
be.connChan <- struct{}{}
|
||||
}()
|
||||
|
||||
obj, err := be.client.GetObject(be.bucketname, objName)
|
||||
if err != nil {
|
||||
debug.Log(" err %v", err)
|
||||
|
||||
// return token
|
||||
be.connChan <- struct{}{}
|
||||
|
||||
return nil, errors.Wrap(err, "client.GetObject")
|
||||
}
|
||||
|
||||
// if we're going to read the whole object, just pass it on.
|
||||
if length == 0 {
|
||||
debug.Log("Load %v: pass on object", h)
|
||||
|
||||
_, err = obj.Seek(offset, 0)
|
||||
if err != nil {
|
||||
_ = obj.Close()
|
||||
|
||||
// return token
|
||||
be.connChan <- struct{}{}
|
||||
|
||||
return nil, errors.Wrap(err, "obj.Seek")
|
||||
}
|
||||
|
||||
return obj, nil
|
||||
rd := wrapReader{
|
||||
ReadCloser: obj,
|
||||
f: func() {
|
||||
debug.Log("Close()")
|
||||
// return token
|
||||
be.connChan <- struct{}{}
|
||||
},
|
||||
}
|
||||
return rd, nil
|
||||
}
|
||||
|
||||
defer func() {
|
||||
// return token
|
||||
be.connChan <- struct{}{}
|
||||
}()
|
||||
|
||||
// otherwise use a buffer with ReadAt
|
||||
info, err := obj.Stat()
|
||||
if err != nil {
|
||||
|
@ -245,21 +245,25 @@ func TestLoad(t testing.TB) {
|
||||
buf, err := ioutil.ReadAll(rd)
|
||||
if err != nil {
|
||||
t.Errorf("Load(%d, %d) ReadAll() returned unexpected error: %v", l, o, err)
|
||||
rd.Close()
|
||||
continue
|
||||
}
|
||||
|
||||
if l <= len(d) && len(buf) != l {
|
||||
t.Errorf("Load(%d, %d) wrong number of bytes read: want %d, got %d", l, o, l, len(buf))
|
||||
rd.Close()
|
||||
continue
|
||||
}
|
||||
|
||||
if l > len(d) && len(buf) != len(d) {
|
||||
t.Errorf("Load(%d, %d) wrong number of bytes read for overlong read: want %d, got %d", l, o, l, len(buf))
|
||||
rd.Close()
|
||||
continue
|
||||
}
|
||||
|
||||
if !bytes.Equal(buf, d) {
|
||||
t.Errorf("Load(%d, %d) returned wrong bytes", l, o)
|
||||
rd.Close()
|
||||
continue
|
||||
}
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user