From 98ae30b51303f84ed40d0a6efe19c788b5feb323 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Fri, 10 Feb 2017 17:25:48 +0100 Subject: [PATCH 1/6] Update Dockerfile --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3fea9ab6d..0f0a83700 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,11 +14,11 @@ # docker run --rm -v $PWD:/home/travis/restic restic/test gb test -v ./backend # # build the image for an older version of Go: -# docker build --build-arg GOVERSION=1.3.3 -t restic/test:go1.3.3 . +# docker build --build-arg GOVERSION=1.6.4 -t restic/test:go1.6.4 . FROM ubuntu:14.04 -ARG GOVERSION=1.7 +ARG GOVERSION=1.7.5 ARG GOARCH=amd64 # install dependencies From 36dee7d892209a7cd5fea1e3a8a1f2cc0f0f6390 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Fri, 10 Feb 2017 19:24:54 +0100 Subject: [PATCH 2/6] s3: Increase MaxIdleConnsPerHost --- src/restic/backend/s3/s3.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/restic/backend/s3/s3.go b/src/restic/backend/s3/s3.go index 18f84ae70..29aad24a9 100644 --- a/src/restic/backend/s3/s3.go +++ b/src/restic/backend/s3/s3.go @@ -3,9 +3,12 @@ package s3 import ( "bytes" "io" + "net" + "net/http" "path" "restic" "strings" + "time" "restic/backend" "restic/errors" @@ -36,6 +39,21 @@ func Open(cfg Config) (restic.Backend, error) { } be := &s3{client: client, bucketname: cfg.Bucket, prefix: cfg.Prefix} + + t := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + MaxIdleConns: 100, + MaxIdleConnsPerHost: 30, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + } + client.SetCustomTransport(t) + be.createConnections() found, err := client.BucketExists(cfg.Bucket) From 64165ea4c8bd375c4fe8f8a65e75edec8cf79c73 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Fri, 10 Feb 2017 19:25:50 +0100 Subject: [PATCH 3/6] s3: Hold connection semaphore until Close() --- src/restic/backend/s3/s3.go | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/src/restic/backend/s3/s3.go b/src/restic/backend/s3/s3.go index 29aad24a9..410997bc7 100644 --- a/src/restic/backend/s3/s3.go +++ b/src/restic/backend/s3/s3.go @@ -122,6 +122,18 @@ func (be *s3) Save(h restic.Handle, rd io.Reader) (err error) { return errors.Wrap(err, "client.PutObject") } +// wrapReader wraps an io.ReadCloser to run an additional function on Close. +type wrapReader struct { + io.ReadCloser + f func() +} + +func (wr wrapReader) Close() error { + err := wr.ReadCloser.Close() + wr.f() + return err +} + // Load returns a reader that yields the contents of the file at h at the // given offset. If length is nonzero, only a portion of the file is // returned. rd must be closed after use. @@ -144,9 +156,6 @@ func (be *s3) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, er objName := be.s3path(h) <-be.connChan - defer func() { - be.connChan <- struct{}{} - }() obj, err := be.client.GetObject(be.bucketname, objName) if err != nil { @@ -157,15 +166,26 @@ func (be *s3) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, er // if we're going to read the whole object, just pass it on. if length == 0 { debug.Log("Load %v: pass on object", h) + _, err = obj.Seek(offset, 0) if err != nil { _ = obj.Close() return nil, errors.Wrap(err, "obj.Seek") } - return obj, nil + rd := wrapReader{ + ReadCloser: obj, + f: func() { + be.connChan <- struct{}{} + }, + } + return rd, nil } + defer func() { + be.connChan <- struct{}{} + }() + // otherwise use a buffer with ReadAt info, err := obj.Stat() if err != nil { From 7a0303f7ae3010bb18d69502d63d16332756fc9a Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Fri, 10 Feb 2017 20:49:37 +0100 Subject: [PATCH 4/6] s3: Make sure to return connection token --- src/restic/backend/s3/s3.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/restic/backend/s3/s3.go b/src/restic/backend/s3/s3.go index 410997bc7..bd79dda13 100644 --- a/src/restic/backend/s3/s3.go +++ b/src/restic/backend/s3/s3.go @@ -155,11 +155,16 @@ func (be *s3) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, er objName := be.s3path(h) + // get token for connection <-be.connChan obj, err := be.client.GetObject(be.bucketname, objName) if err != nil { debug.Log(" err %v", err) + + // return token + be.connChan <- struct{}{} + return nil, errors.Wrap(err, "client.GetObject") } @@ -170,12 +175,18 @@ func (be *s3) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, er _, err = obj.Seek(offset, 0) if err != nil { _ = obj.Close() + + // return token + be.connChan <- struct{}{} + return nil, errors.Wrap(err, "obj.Seek") } rd := wrapReader{ ReadCloser: obj, f: func() { + debug.Log("Close()") + // return token be.connChan <- struct{}{} }, } @@ -183,6 +194,7 @@ func (be *s3) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, er } defer func() { + // return token be.connChan <- struct{}{} }() From 21b358c74211cafbcd2ec0cbef6d8a46e0378b00 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Fri, 10 Feb 2017 20:49:46 +0100 Subject: [PATCH 5/6] backend tests: Always close reader --- src/restic/backend/test/tests.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/restic/backend/test/tests.go b/src/restic/backend/test/tests.go index a6105f9e9..5c11d2e4b 100644 --- a/src/restic/backend/test/tests.go +++ b/src/restic/backend/test/tests.go @@ -245,21 +245,25 @@ func TestLoad(t testing.TB) { buf, err := ioutil.ReadAll(rd) if err != nil { t.Errorf("Load(%d, %d) ReadAll() returned unexpected error: %v", l, o, err) + rd.Close() continue } if l <= len(d) && len(buf) != l { t.Errorf("Load(%d, %d) wrong number of bytes read: want %d, got %d", l, o, l, len(buf)) + rd.Close() continue } if l > len(d) && len(buf) != len(d) { t.Errorf("Load(%d, %d) wrong number of bytes read for overlong read: want %d, got %d", l, o, l, len(buf)) + rd.Close() continue } if !bytes.Equal(buf, d) { t.Errorf("Load(%d, %d) returned wrong bytes", l, o) + rd.Close() continue } From f266741f406d14f11142878d6040cbe03015585d Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 11 Feb 2017 10:40:51 +0100 Subject: [PATCH 6/6] s3/rest: raise connection limit to 40 --- src/restic/backend/rest/rest.go | 2 +- src/restic/backend/s3/s3.go | 19 +++---------------- 2 files changed, 4 insertions(+), 17 deletions(-) diff --git a/src/restic/backend/rest/rest.go b/src/restic/backend/rest/rest.go index 7121de945..2c133872b 100644 --- a/src/restic/backend/rest/rest.go +++ b/src/restic/backend/rest/rest.go @@ -17,7 +17,7 @@ import ( "restic/backend" ) -const connLimit = 10 +const connLimit = 40 // make sure the rest backend implements restic.Backend var _ restic.Backend = &restBackend{} diff --git a/src/restic/backend/s3/s3.go b/src/restic/backend/s3/s3.go index bd79dda13..2c7c02c60 100644 --- a/src/restic/backend/s3/s3.go +++ b/src/restic/backend/s3/s3.go @@ -3,12 +3,10 @@ package s3 import ( "bytes" "io" - "net" "net/http" "path" "restic" "strings" - "time" "restic/backend" "restic/errors" @@ -18,7 +16,7 @@ import ( "restic/debug" ) -const connLimit = 10 +const connLimit = 40 // s3 is a backend which stores the data on an S3 endpoint. type s3 struct { @@ -40,19 +38,8 @@ func Open(cfg Config) (restic.Backend, error) { be := &s3{client: client, bucketname: cfg.Bucket, prefix: cfg.Prefix} - t := &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - }).DialContext, - MaxIdleConns: 100, - MaxIdleConnsPerHost: 30, - IdleConnTimeout: 90 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - } - client.SetCustomTransport(t) + tr := &http.Transport{MaxIdleConnsPerHost: connLimit} + client.SetCustomTransport(tr) be.createConnections()