mirror of
https://github.com/octoleo/restic.git
synced 2024-12-22 10:58:55 +00:00
azure: Switch to azblob sdk
This commit is contained in:
Endre Karlson
Michael Eischer
parent
4ba31df08f
commit
25648e2501
@ -733,7 +733,7 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio
|
||||
case "gs":
|
||||
be, err = gs.Open(cfg.(gs.Config), rt)
|
||||
case "azure":
|
||||
be, err = azure.Open(cfg.(azure.Config), rt)
|
||||
be, err = azure.Open(ctx, cfg.(azure.Config), rt)
|
||||
case "swift":
|
||||
be, err = swift.Open(ctx, cfg.(swift.Config), rt)
|
||||
case "b2":
|
||||
@ -805,7 +805,7 @@ func create(ctx context.Context, s string, opts options.Options) (restic.Backend
|
||||
case "gs":
|
||||
return gs.Create(cfg.(gs.Config), rt)
|
||||
case "azure":
|
||||
return azure.Create(cfg.(azure.Config), rt)
|
||||
return azure.Create(ctx, cfg.(azure.Config), rt)
|
||||
case "swift":
|
||||
return swift.Open(ctx, cfg.(swift.Config), rt)
|
||||
case "b2":
|
||||
|
||||
14
go.mod
14
go.mod
@ -2,7 +2,8 @@ module github.com/restic/restic
|
||||
|
||||
require (
|
||||
cloud.google.com/go/storage v1.28.0
|
||||
github.com/Azure/azure-sdk-for-go v66.0.0+incompatible
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1
|
||||
github.com/anacrolix/fuse v0.2.0
|
||||
github.com/cenkalti/backoff/v4 v4.2.0
|
||||
github.com/cespare/xxhash/v2 v2.1.2
|
||||
@ -38,19 +39,11 @@ require (
|
||||
cloud.google.com/go/compute v1.12.1 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.2.1 // indirect
|
||||
cloud.google.com/go/iam v0.6.0 // indirect
|
||||
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
|
||||
github.com/Azure/go-autorest/autorest v0.11.28 // indirect
|
||||
github.com/Azure/go-autorest/autorest/adal v0.9.21 // indirect
|
||||
github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
|
||||
github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
|
||||
github.com/Azure/go-autorest/logger v0.2.1 // indirect
|
||||
github.com/Azure/go-autorest/tracing v0.6.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
|
||||
github.com/dnaeon/go-vcr v1.2.0 // indirect
|
||||
github.com/dustin/go-humanize v1.0.0 // indirect
|
||||
github.com/felixge/fgprof v0.9.3 // indirect
|
||||
github.com/gofrs/uuid v4.2.0+incompatible // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.4.2 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||
github.com/golang/protobuf v1.5.2 // indirect
|
||||
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
|
||||
@ -74,7 +67,6 @@ require (
|
||||
google.golang.org/grpc v1.50.1 // indirect
|
||||
google.golang.org/protobuf v1.28.1 // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
||||
|
||||
41
go.sum
41
go.sum
@ -10,26 +10,14 @@ cloud.google.com/go/iam v0.6.0/go.mod h1:+1AH33ueBne5MzYccyMHtEKqLE4/kJOibtffMHD
|
||||
cloud.google.com/go/longrunning v0.1.1 h1:y50CXG4j0+qvEukslYFBCrzaXX0qpFbBzc3PchSu/LE=
|
||||
cloud.google.com/go/storage v1.28.0 h1:DLrIZ6xkeZX6K70fU/boWx5INJumt6f+nwwWSHXzzGY=
|
||||
cloud.google.com/go/storage v1.28.0/go.mod h1:qlgZML35PXA3zoEnIkiPLY4/TOkUleufRlu6qmcf7sI=
|
||||
github.com/Azure/azure-sdk-for-go v66.0.0+incompatible h1:bmmC38SlE8/E81nNADlgmVGurPWMHDX2YNXVQMrBpEE=
|
||||
github.com/Azure/azure-sdk-for-go v66.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
|
||||
github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=
|
||||
github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
|
||||
github.com/Azure/go-autorest/autorest v0.11.28 h1:ndAExarwr5Y+GaHE6VCaY1kyS/HwwGGyuimVhWsHOEM=
|
||||
github.com/Azure/go-autorest/autorest v0.11.28/go.mod h1:MrkzG3Y3AH668QyF9KRk5neJnGgmhQ6krbhR8Q5eMvA=
|
||||
github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ=
|
||||
github.com/Azure/go-autorest/autorest/adal v0.9.21 h1:jjQnVFXPfekaqb8vIsv2G1lxshoW+oGv4MDlhRtnYZk=
|
||||
github.com/Azure/go-autorest/autorest/adal v0.9.21/go.mod h1:zua7mBUaCc5YnSLKYgGJR/w5ePdMDA6H56upLsHzA9U=
|
||||
github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw=
|
||||
github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
|
||||
github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
|
||||
github.com/Azure/go-autorest/autorest/mocks v0.4.2 h1:PGN4EDXnuQbojHbU0UWoNvmu9AGVwYHG9/fkDYhtAfw=
|
||||
github.com/Azure/go-autorest/autorest/mocks v0.4.2/go.mod h1:Vy7OitM9Kei0i1Oj+LvyAWMXJHeKH1MVlzFugfVrmyU=
|
||||
github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk=
|
||||
github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE=
|
||||
github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg=
|
||||
github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
|
||||
github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo=
|
||||
github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 h1:pqrAR74b6EoR4kcxF7L7Wg2B8Jgil9UUZtMvxhEFqWo=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4SathZPhDhF4mVwpBMFlYjyAqy8=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQiecy1SI+Eog7xVkPiqIbg=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 h1:BMTdr+ib5ljLa9MxTJK8x/Ds0MbBb4MfuW5BL0zMJnI=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1/go.mod h1:c6WvOhtmjNUWbLfOG1qxM/q0SPvQNSVJvolm+C52dIU=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1 h1:BWe8a+f/t+7KY7zH2mqygeUD0t8hNFXe08p1Pb3/jKE=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74=
|
||||
github.com/anacrolix/fuse v0.2.0 h1:pc+To78kI2d/WUjIyrsdqeJQAesuwpGxlI3h1nAv3Do=
|
||||
@ -65,12 +53,7 @@ github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g=
|
||||
github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw=
|
||||
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
|
||||
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
||||
github.com/gofrs/uuid v4.2.0+incompatible h1:yyYWMnhkhrKwwr8gAOcOCYxOOscHgDS9yZgBrnJfGa0=
|
||||
github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
|
||||
github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
|
||||
github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
|
||||
github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs=
|
||||
github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
|
||||
github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
|
||||
@ -128,6 +111,7 @@ github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
|
||||
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
|
||||
github.com/kurin/blazer v0.5.4-0.20211030221322-ba894c124ac6 h1:nz7i1au+nDzgExfqW5Zl6q85XNTvYoGnM5DHiQC0yYs=
|
||||
github.com/kurin/blazer v0.5.4-0.20211030221322-ba894c124ac6/go.mod h1:4FCXMUWo9DllR2Do4TtBd377ezyAJ51vB5uTBjt0pGU=
|
||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v7 v7.0.45 h1:g4IeM9M9pW/Lo8AGGNOjBZYlvmtlE1N5TQEYWXRWzIs=
|
||||
@ -142,6 +126,7 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY
|
||||
github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8=
|
||||
github.com/ncw/swift/v2 v2.0.1 h1:q1IN8hNViXEv8Zvg3Xdis4a3c4IlIGezkYz09zQL5J0=
|
||||
github.com/ncw/swift/v2 v2.0.1/go.mod h1:z0A9RVdYPjNjXVo2pDOPxZ4eu3oarO1P91fTItcb+Kg=
|
||||
github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 h1:Qj1ukM4GlMWXNdMBuXcXfz/Kw9s1qm0CLY32QxuSImI=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/profile v1.7.0 h1:hnbDkaNWPCLMO9wGLdBFTIZvzDrDfBM2072E1S9gJkA=
|
||||
@ -184,9 +169,7 @@ go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.0.0-20220817201139-bc19a97f63c8 h1:GIAS/yBem/gq2MUqgNIzUHW7cJMmx3TGZOrnyYaNQ6c=
|
||||
golang.org/x/crypto v0.0.0-20220817201139-bc19a97f63c8/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
@ -203,7 +186,6 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20221014081412-f15817d10f9b h1:tvrvnPFcdzp294diPnrdZZZ8XUt2Tyj7svb7X52iDuU=
|
||||
golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
|
||||
@ -290,7 +272,6 @@ gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
|
||||
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
package azure
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/base64"
|
||||
@ -18,14 +19,20 @@ import (
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/storage"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
|
||||
azContainer "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
|
||||
"github.com/cenkalti/backoff/v4"
|
||||
)
|
||||
|
||||
// Backend stores data on an azure endpoint.
|
||||
type Backend struct {
|
||||
accountName string
|
||||
container *storage.Container
|
||||
cfg Config
|
||||
container *azContainer.Client
|
||||
connections uint
|
||||
sem sema.Semaphore
|
||||
prefix string
|
||||
@ -33,6 +40,7 @@ type Backend struct {
|
||||
layout.Layout
|
||||
}
|
||||
|
||||
const saveLargeSize = 256 * 1024 * 1024
|
||||
const defaultListMaxItems = 5000
|
||||
|
||||
// make sure that *Backend implements backend.Backend
|
||||
@ -40,29 +48,47 @@ var _ restic.Backend = &Backend{}
|
||||
|
||||
func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
|
||||
debug.Log("open, config %#v", cfg)
|
||||
var client storage.Client
|
||||
var client *azContainer.Client
|
||||
var err error
|
||||
|
||||
url := fmt.Sprintf("https://%s.blob.core.windows.net/%s", cfg.AccountName, cfg.Container)
|
||||
opts := &azContainer.ClientOptions{
|
||||
ClientOptions: azcore.ClientOptions{
|
||||
Transport: http.DefaultClient,
|
||||
},
|
||||
}
|
||||
|
||||
if cfg.AccountKey.String() != "" {
|
||||
// We have an account key value, find the BlobServiceClient
|
||||
// from with a BasicClient
|
||||
debug.Log(" - using account key")
|
||||
client, err = storage.NewBasicClient(cfg.AccountName, cfg.AccountKey.Unwrap())
|
||||
cred, err := azblob.NewSharedKeyCredential(cfg.AccountName, cfg.AccountKey.Unwrap())
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "NewBasicClient")
|
||||
return nil, errors.Wrap(err, "NewSharedKeyCredential")
|
||||
}
|
||||
|
||||
client, err = azContainer.NewClientWithSharedKeyCredential(url, cred, opts)
|
||||
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "NewClientWithSharedKeyCredential")
|
||||
}
|
||||
} else if cfg.AccountSAS.String() != "" {
|
||||
// Get the client using the SAS Token as authentication, this
|
||||
// is longer winded than above because the SDK wants a URL for the Account
|
||||
// if your using a SAS token, and not just the account name
|
||||
// we (as per the SDK ) assume the default Azure portal.
|
||||
url := fmt.Sprintf("https://%s.blob.core.windows.net/", cfg.AccountName)
|
||||
// https://github.com/Azure/azure-storage-blob-go/issues/130
|
||||
debug.Log(" - using sas token")
|
||||
sas := cfg.AccountSAS.Unwrap()
|
||||
|
||||
// strip query sign prefix
|
||||
if sas[0] == '?' {
|
||||
sas = sas[1:]
|
||||
}
|
||||
client, err = storage.NewAccountSASClientFromEndpointToken(url, sas)
|
||||
|
||||
urlWithSAS := fmt.Sprintf("%s?%s", url, sas)
|
||||
|
||||
client, err = azContainer.NewClientWithNoCredential(urlWithSAS, opts)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "NewAccountSASClientFromEndpointToken")
|
||||
}
|
||||
@ -70,21 +96,16 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
|
||||
return nil, errors.New("no azure authentication information found")
|
||||
}
|
||||
|
||||
client.HTTPClient = &http.Client{Transport: rt}
|
||||
|
||||
service := client.GetBlobService()
|
||||
|
||||
sem, err := sema.New(cfg.Connections)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
be := &Backend{
|
||||
container: service.GetContainerReference(cfg.Container),
|
||||
accountName: cfg.AccountName,
|
||||
container: client,
|
||||
cfg: cfg,
|
||||
connections: cfg.Connections,
|
||||
sem: sem,
|
||||
prefix: cfg.Prefix,
|
||||
Layout: &layout.DefaultLayout{
|
||||
Path: cfg.Prefix,
|
||||
Join: path.Join,
|
||||
@ -96,26 +117,27 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
|
||||
}
|
||||
|
||||
// Open opens the Azure backend at specified container.
|
||||
func Open(cfg Config, rt http.RoundTripper) (*Backend, error) {
|
||||
func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, error) {
|
||||
return open(cfg, rt)
|
||||
}
|
||||
|
||||
// Create opens the Azure backend at specified container and creates the container if
|
||||
// it does not exist yet.
|
||||
func Create(cfg Config, rt http.RoundTripper) (*Backend, error) {
|
||||
func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, error) {
|
||||
be, err := open(cfg, rt)
|
||||
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "open")
|
||||
}
|
||||
|
||||
options := storage.CreateContainerOptions{
|
||||
Access: storage.ContainerAccessTypePrivate,
|
||||
}
|
||||
if err != nil && bloberror.HasCode(err, bloberror.ContainerNotFound) {
|
||||
_, err = be.container.Create(ctx, &azContainer.CreateOptions{})
|
||||
|
||||
_, err = be.container.CreateIfNotExists(&options)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "container.CreateIfNotExists")
|
||||
return nil, errors.Wrap(err, "container.Create")
|
||||
}
|
||||
} else if err != nil {
|
||||
return be, err
|
||||
}
|
||||
|
||||
return be, nil
|
||||
@ -129,8 +151,7 @@ func (be *Backend) SetListMaxItems(i int) {
|
||||
// IsNotExist returns true if the error is caused by a not existing file.
|
||||
func (be *Backend) IsNotExist(err error) bool {
|
||||
debug.Log("IsNotExist(%T, %#v)", err, err)
|
||||
var aerr storage.AzureStorageServiceError
|
||||
return errors.As(err, &aerr) && aerr.StatusCode == http.StatusNotFound
|
||||
return bloberror.HasCode(err, bloberror.BlobNotFound)
|
||||
}
|
||||
|
||||
// Join combines path components with slashes.
|
||||
@ -144,7 +165,7 @@ func (be *Backend) Connections() uint {
|
||||
|
||||
// Location returns this backend's location (the container name).
|
||||
func (be *Backend) Location() string {
|
||||
return be.Join(be.container.Name, be.prefix)
|
||||
return be.Join(be.cfg.AccountName, be.cfg.Prefix)
|
||||
}
|
||||
|
||||
// Hasher may return a hash function for calculating a content hash for the backend
|
||||
@ -162,16 +183,6 @@ func (be *Backend) Path() string {
|
||||
return be.prefix
|
||||
}
|
||||
|
||||
type azureAdapter struct {
|
||||
restic.RewindReader
|
||||
}
|
||||
|
||||
func (azureAdapter) Close() error { return nil }
|
||||
|
||||
func (a azureAdapter) Len() int {
|
||||
return int(a.Length())
|
||||
}
|
||||
|
||||
// Save stores data in the backend at the handle.
|
||||
func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
|
||||
if err := h.Valid(); err != nil {
|
||||
@ -184,41 +195,53 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe
|
||||
|
||||
be.sem.GetToken()
|
||||
|
||||
debug.Log("InsertObject(%v, %v)", be.container.Name, objName)
|
||||
debug.Log("InsertObject(%v, %v)", be.cfg.AccountName, objName)
|
||||
|
||||
var err error
|
||||
if rd.Length() < 256*1024*1024 {
|
||||
// wrap the reader so that net/http client cannot close the reader
|
||||
// CreateBlockBlobFromReader reads length from `Len()``
|
||||
dataReader := azureAdapter{rd}
|
||||
|
||||
if rd.Length() < saveLargeSize {
|
||||
// if it's smaller than 256miB, then just create the file directly from the reader
|
||||
ref := be.container.GetBlobReference(objName)
|
||||
ref.Properties.ContentMD5 = base64.StdEncoding.EncodeToString(rd.Hash())
|
||||
err = ref.CreateBlockBlobFromReader(dataReader, nil)
|
||||
err = be.saveSmall(ctx, objName, rd)
|
||||
} else {
|
||||
// otherwise use the more complicated method
|
||||
err = be.saveLarge(ctx, objName, rd)
|
||||
|
||||
}
|
||||
|
||||
be.sem.ReleaseToken()
|
||||
debug.Log("%v, err %#v", objName, err)
|
||||
|
||||
return errors.Wrap(err, "CreateBlockBlobFromReader")
|
||||
return err
|
||||
}
|
||||
|
||||
func (be *Backend) saveSmall(ctx context.Context, objName string, rd restic.RewindReader) error {
|
||||
blockBlobClient := be.container.NewBlockBlobClient(objName)
|
||||
|
||||
// upload it as a new "block", use the base64 hash for the ID
|
||||
id := base64.StdEncoding.EncodeToString(rd.Hash())
|
||||
|
||||
buf := make([]byte, rd.Length())
|
||||
_, err := io.ReadFull(rd, buf)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "ReadFull")
|
||||
}
|
||||
|
||||
reader := bytes.NewReader(buf)
|
||||
_, err = blockBlobClient.StageBlock(ctx, id, streaming.NopCloser(reader), &blockblob.StageBlockOptions{
|
||||
TransactionalContentMD5: rd.Hash(),
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "StageBlock")
|
||||
}
|
||||
|
||||
blocks := []string{id}
|
||||
_, err = blockBlobClient.CommitBlockList(ctx, blocks, &blockblob.CommitBlockListOptions{})
|
||||
return errors.Wrap(err, "CommitBlockList")
|
||||
}
|
||||
|
||||
func (be *Backend) saveLarge(ctx context.Context, objName string, rd restic.RewindReader) error {
|
||||
// create the file on the server
|
||||
file := be.container.GetBlobReference(objName)
|
||||
err := file.CreateBlockBlob(nil)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "CreateBlockBlob")
|
||||
}
|
||||
blockBlobClient := be.container.NewBlockBlobClient(objName)
|
||||
|
||||
// read the data, in 100 MiB chunks
|
||||
buf := make([]byte, 100*1024*1024)
|
||||
var blocks []storage.Block
|
||||
blocks := []string{}
|
||||
uploadedBytes := 0
|
||||
|
||||
for {
|
||||
@ -226,6 +249,7 @@ func (be *Backend) saveLarge(ctx context.Context, objName string, rd restic.Rewi
|
||||
if err == io.ErrUnexpectedEOF {
|
||||
err = nil
|
||||
}
|
||||
|
||||
if err == io.EOF {
|
||||
// end of file reached, no bytes have been read at all
|
||||
break
|
||||
@ -241,16 +265,18 @@ func (be *Backend) saveLarge(ctx context.Context, objName string, rd restic.Rewi
|
||||
// upload it as a new "block", use the base64 hash for the ID
|
||||
h := md5.Sum(buf)
|
||||
id := base64.StdEncoding.EncodeToString(h[:])
|
||||
debug.Log("PutBlock %v with %d bytes", id, len(buf))
|
||||
err = file.PutBlock(id, buf, &storage.PutBlockOptions{ContentMD5: id})
|
||||
|
||||
reader := bytes.NewReader(buf)
|
||||
debug.Log("StageBlock %v with %d bytes", id, len(buf))
|
||||
_, err = blockBlobClient.StageBlock(ctx, id, streaming.NopCloser(reader), &blockblob.StageBlockOptions{
|
||||
TransactionalContentMD5: h[:],
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "PutBlock")
|
||||
return errors.Wrap(err, "StageBlock")
|
||||
}
|
||||
|
||||
blocks = append(blocks, storage.Block{
|
||||
ID: id,
|
||||
Status: "Uncommitted",
|
||||
})
|
||||
blocks = append(blocks, id)
|
||||
}
|
||||
|
||||
// sanity check
|
||||
@ -258,10 +284,10 @@ func (be *Backend) saveLarge(ctx context.Context, objName string, rd restic.Rewi
|
||||
return errors.Errorf("wrote %d bytes instead of the expected %d bytes", uploadedBytes, rd.Length())
|
||||
}
|
||||
|
||||
_, err := blockBlobClient.CommitBlockList(ctx, blocks, &blockblob.CommitBlockListOptions{})
|
||||
|
||||
debug.Log("uploaded %d parts: %v", len(blocks), blocks)
|
||||
err = file.PutBlockList(blocks, nil)
|
||||
debug.Log("PutBlockList returned %v", err)
|
||||
return errors.Wrap(err, "PutBlockList")
|
||||
return errors.Wrap(err, "CommitBlockList")
|
||||
}
|
||||
|
||||
// Load runs fn with a reader that yields the contents of the file at h at the
|
||||
@ -285,26 +311,22 @@ func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int,
|
||||
}
|
||||
|
||||
objName := be.Filename(h)
|
||||
blob := be.container.GetBlobReference(objName)
|
||||
|
||||
start := uint64(offset)
|
||||
var end uint64
|
||||
|
||||
if length > 0 {
|
||||
end = uint64(offset + int64(length) - 1)
|
||||
} else {
|
||||
end = 0
|
||||
}
|
||||
blockBlobClient := be.container.NewBlobClient(objName)
|
||||
|
||||
be.sem.GetToken()
|
||||
resp, err := blockBlobClient.DownloadStream(ctx, &blob.DownloadStreamOptions{
|
||||
Range: azblob.HTTPRange{
|
||||
Offset: offset,
|
||||
Count: int64(length),
|
||||
},
|
||||
})
|
||||
|
||||
rd, err := blob.GetRange(&storage.GetBlobRangeOptions{Range: &storage.BlobRange{Start: start, End: end}})
|
||||
if err != nil {
|
||||
be.sem.ReleaseToken()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return be.sem.ReleaseTokenOnClose(rd, nil), err
|
||||
return be.sem.ReleaseTokenOnClose(resp.Body, nil), err
|
||||
}
|
||||
|
||||
// Stat returns information about a blob.
|
||||
@ -312,10 +334,10 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo,
|
||||
debug.Log("%v", h)
|
||||
|
||||
objName := be.Filename(h)
|
||||
blob := be.container.GetBlobReference(objName)
|
||||
blobClient := be.container.NewBlobClient(objName)
|
||||
|
||||
be.sem.GetToken()
|
||||
err := blob.GetProperties(nil)
|
||||
props, err := blobClient.GetProperties(ctx, nil)
|
||||
be.sem.ReleaseToken()
|
||||
|
||||
if err != nil {
|
||||
@ -324,7 +346,7 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo,
|
||||
}
|
||||
|
||||
fi := restic.FileInfo{
|
||||
Size: int64(blob.Properties.ContentLength),
|
||||
Size: *props.ContentLength,
|
||||
Name: h.Name,
|
||||
}
|
||||
return fi, nil
|
||||
@ -333,12 +355,18 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo,
|
||||
// Remove removes the blob with the given name and type.
|
||||
func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
|
||||
objName := be.Filename(h)
|
||||
blob := be.container.NewBlobClient(objName)
|
||||
|
||||
be.sem.GetToken()
|
||||
_, err := be.container.GetBlobReference(objName).DeleteIfExists(nil)
|
||||
_, err := blob.Delete(ctx, &azblob.DeleteBlobOptions{})
|
||||
be.sem.ReleaseToken()
|
||||
|
||||
debug.Log("Remove(%v) at %v -> err %v", h, objName, err)
|
||||
|
||||
if bloberror.HasCode(err, bloberror.BlobNotFound) {
|
||||
return nil
|
||||
}
|
||||
|
||||
return errors.Wrap(err, "client.RemoveObject")
|
||||
}
|
||||
|
||||
@ -354,31 +382,34 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F
|
||||
prefix += "/"
|
||||
}
|
||||
|
||||
params := storage.ListBlobsParameters{
|
||||
MaxResults: uint(be.listMaxItems),
|
||||
Prefix: prefix,
|
||||
}
|
||||
max := int32(be.listMaxItems)
|
||||
|
||||
for {
|
||||
opts := &azContainer.ListBlobsFlatOptions{
|
||||
MaxResults: &max,
|
||||
Prefix: &prefix,
|
||||
}
|
||||
lister := be.container.NewListBlobsFlatPager(opts)
|
||||
|
||||
for lister.More() {
|
||||
be.sem.GetToken()
|
||||
obj, err := be.container.ListBlobs(params)
|
||||
resp, err := lister.NextPage(ctx)
|
||||
be.sem.ReleaseToken()
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
debug.Log("got %v objects", len(obj.Blobs))
|
||||
debug.Log("got %v objects", len(resp.Segment.BlobItems))
|
||||
|
||||
for _, item := range obj.Blobs {
|
||||
m := strings.TrimPrefix(item.Name, prefix)
|
||||
for _, item := range resp.Segment.BlobItems {
|
||||
m := strings.TrimPrefix(*item.Name, prefix)
|
||||
if m == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
fi := restic.FileInfo{
|
||||
Name: path.Base(m),
|
||||
Size: item.Properties.ContentLength,
|
||||
Size: *item.Properties.ContentLength,
|
||||
}
|
||||
|
||||
if ctx.Err() != nil {
|
||||
@ -395,11 +426,6 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if obj.NextMarker == "" {
|
||||
break
|
||||
}
|
||||
params.Marker = obj.NextMarker
|
||||
}
|
||||
|
||||
return ctx.Err()
|
||||
|
||||
@ -46,7 +46,8 @@ func newAzureTestSuite(t testing.TB) *test.Suite {
|
||||
Create: func(config interface{}) (restic.Backend, error) {
|
||||
cfg := config.(azure.Config)
|
||||
|
||||
be, err := azure.Create(cfg, tr)
|
||||
ctx := context.TODO()
|
||||
be, err := azure.Create(ctx, cfg, tr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -66,15 +67,15 @@ func newAzureTestSuite(t testing.TB) *test.Suite {
|
||||
// OpenFn is a function that opens a previously created temporary repository.
|
||||
Open: func(config interface{}) (restic.Backend, error) {
|
||||
cfg := config.(azure.Config)
|
||||
|
||||
return azure.Open(cfg, tr)
|
||||
ctx := context.TODO()
|
||||
return azure.Open(ctx, cfg, tr)
|
||||
},
|
||||
|
||||
// CleanupFn removes data created during the tests.
|
||||
Cleanup: func(config interface{}) error {
|
||||
cfg := config.(azure.Config)
|
||||
|
||||
be, err := azure.Open(cfg, tr)
|
||||
ctx := context.TODO()
|
||||
be, err := azure.Open(ctx, cfg, tr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -155,7 +156,7 @@ func TestUploadLargeFile(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
be, err := azure.Create(cfg, tr)
|
||||
be, err := azure.Create(ctx, cfg, tr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user