diff --git a/cmd/restic/global.go b/cmd/restic/global.go index b4da9dfbe..faf74dc42 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -733,7 +733,7 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio case "gs": be, err = gs.Open(cfg.(gs.Config), rt) case "azure": - be, err = azure.Open(cfg.(azure.Config), rt) + be, err = azure.Open(ctx, cfg.(azure.Config), rt) case "swift": be, err = swift.Open(ctx, cfg.(swift.Config), rt) case "b2": @@ -805,7 +805,7 @@ func create(ctx context.Context, s string, opts options.Options) (restic.Backend case "gs": return gs.Create(cfg.(gs.Config), rt) case "azure": - return azure.Create(cfg.(azure.Config), rt) + return azure.Create(ctx, cfg.(azure.Config), rt) case "swift": return swift.Open(ctx, cfg.(swift.Config), rt) case "b2": diff --git a/go.mod b/go.mod index 41f1adf3c..88d9a7d6e 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,8 @@ module github.com/restic/restic require ( cloud.google.com/go/storage v1.28.0 - github.com/Azure/azure-sdk-for-go v66.0.0+incompatible + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 github.com/anacrolix/fuse v0.2.0 github.com/cenkalti/backoff/v4 v4.2.0 github.com/cespare/xxhash/v2 v2.1.2 @@ -38,19 +39,11 @@ require ( cloud.google.com/go/compute v1.12.1 // indirect cloud.google.com/go/compute/metadata v0.2.1 // indirect cloud.google.com/go/iam v0.6.0 // indirect - github.com/Azure/go-autorest v14.2.0+incompatible // indirect - github.com/Azure/go-autorest/autorest v0.11.28 // indirect - github.com/Azure/go-autorest/autorest/adal v0.9.21 // indirect - github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect - github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect - github.com/Azure/go-autorest/logger v0.2.1 // indirect - github.com/Azure/go-autorest/tracing v0.6.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/dnaeon/go-vcr v1.2.0 // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/felixge/fgprof v0.9.3 // indirect - github.com/gofrs/uuid v4.2.0+incompatible // indirect - github.com/golang-jwt/jwt/v4 v4.4.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect @@ -74,7 +67,6 @@ require ( google.golang.org/grpc v1.50.1 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index b334777cc..88c2a9421 100644 --- a/go.sum +++ b/go.sum @@ -10,26 +10,14 @@ cloud.google.com/go/iam v0.6.0/go.mod h1:+1AH33ueBne5MzYccyMHtEKqLE4/kJOibtffMHD cloud.google.com/go/longrunning v0.1.1 h1:y50CXG4j0+qvEukslYFBCrzaXX0qpFbBzc3PchSu/LE= cloud.google.com/go/storage v1.28.0 h1:DLrIZ6xkeZX6K70fU/boWx5INJumt6f+nwwWSHXzzGY= cloud.google.com/go/storage v1.28.0/go.mod h1:qlgZML35PXA3zoEnIkiPLY4/TOkUleufRlu6qmcf7sI= -github.com/Azure/azure-sdk-for-go v66.0.0+incompatible h1:bmmC38SlE8/E81nNADlgmVGurPWMHDX2YNXVQMrBpEE= -github.com/Azure/azure-sdk-for-go v66.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= -github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.11.28 h1:ndAExarwr5Y+GaHE6VCaY1kyS/HwwGGyuimVhWsHOEM= -github.com/Azure/go-autorest/autorest v0.11.28/go.mod h1:MrkzG3Y3AH668QyF9KRk5neJnGgmhQ6krbhR8Q5eMvA= -github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= -github.com/Azure/go-autorest/autorest/adal v0.9.21 h1:jjQnVFXPfekaqb8vIsv2G1lxshoW+oGv4MDlhRtnYZk= -github.com/Azure/go-autorest/autorest/adal v0.9.21/go.mod h1:zua7mBUaCc5YnSLKYgGJR/w5ePdMDA6H56upLsHzA9U= -github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= -github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= -github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/autorest/mocks v0.4.2 h1:PGN4EDXnuQbojHbU0UWoNvmu9AGVwYHG9/fkDYhtAfw= -github.com/Azure/go-autorest/autorest/mocks v0.4.2/go.mod h1:Vy7OitM9Kei0i1Oj+LvyAWMXJHeKH1MVlzFugfVrmyU= -github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= -github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= -github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= -github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= -github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= -github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 h1:pqrAR74b6EoR4kcxF7L7Wg2B8Jgil9UUZtMvxhEFqWo= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4SathZPhDhF4mVwpBMFlYjyAqy8= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQiecy1SI+Eog7xVkPiqIbg= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 h1:BMTdr+ib5ljLa9MxTJK8x/Ds0MbBb4MfuW5BL0zMJnI= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1/go.mod h1:c6WvOhtmjNUWbLfOG1qxM/q0SPvQNSVJvolm+C52dIU= +github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1 h1:BWe8a+f/t+7KY7zH2mqygeUD0t8hNFXe08p1Pb3/jKE= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= github.com/anacrolix/fuse v0.2.0 h1:pc+To78kI2d/WUjIyrsdqeJQAesuwpGxlI3h1nAv3Do= @@ -65,12 +53,7 @@ github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g= github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/gofrs/uuid v4.2.0+incompatible h1:yyYWMnhkhrKwwr8gAOcOCYxOOscHgDS9yZgBrnJfGa0= -github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= -github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= -github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= -github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= -github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= @@ -128,6 +111,7 @@ github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kurin/blazer v0.5.4-0.20211030221322-ba894c124ac6 h1:nz7i1au+nDzgExfqW5Zl6q85XNTvYoGnM5DHiQC0yYs= github.com/kurin/blazer v0.5.4-0.20211030221322-ba894c124ac6/go.mod h1:4FCXMUWo9DllR2Do4TtBd377ezyAJ51vB5uTBjt0pGU= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/minio/minio-go/v7 v7.0.45 h1:g4IeM9M9pW/Lo8AGGNOjBZYlvmtlE1N5TQEYWXRWzIs= @@ -142,6 +126,7 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/ncw/swift/v2 v2.0.1 h1:q1IN8hNViXEv8Zvg3Xdis4a3c4IlIGezkYz09zQL5J0= github.com/ncw/swift/v2 v2.0.1/go.mod h1:z0A9RVdYPjNjXVo2pDOPxZ4eu3oarO1P91fTItcb+Kg= +github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 h1:Qj1ukM4GlMWXNdMBuXcXfz/Kw9s1qm0CLY32QxuSImI= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.7.0 h1:hnbDkaNWPCLMO9wGLdBFTIZvzDrDfBM2072E1S9gJkA= @@ -184,9 +169,7 @@ go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220817201139-bc19a97f63c8 h1:GIAS/yBem/gq2MUqgNIzUHW7cJMmx3TGZOrnyYaNQ6c= golang.org/x/crypto v0.0.0-20220817201139-bc19a97f63c8/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -203,7 +186,6 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20221014081412-f15817d10f9b h1:tvrvnPFcdzp294diPnrdZZZ8XUt2Tyj7svb7X52iDuU= golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= @@ -290,7 +272,6 @@ gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/backend/azure/azure.go b/internal/backend/azure/azure.go index 9e1c64c47..b5a919b22 100644 --- a/internal/backend/azure/azure.go +++ b/internal/backend/azure/azure.go @@ -1,6 +1,7 @@ package azure import ( + "bytes" "context" "crypto/md5" "encoding/base64" @@ -18,14 +19,20 @@ import ( "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" - "github.com/Azure/azure-sdk-for-go/storage" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob" + azContainer "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" "github.com/cenkalti/backoff/v4" ) // Backend stores data on an azure endpoint. type Backend struct { - accountName string - container *storage.Container + cfg Config + container *azContainer.Client connections uint sem sema.Semaphore prefix string @@ -33,6 +40,7 @@ type Backend struct { layout.Layout } +const saveLargeSize = 256 * 1024 * 1024 const defaultListMaxItems = 5000 // make sure that *Backend implements backend.Backend @@ -40,29 +48,47 @@ var _ restic.Backend = &Backend{} func open(cfg Config, rt http.RoundTripper) (*Backend, error) { debug.Log("open, config %#v", cfg) - var client storage.Client + var client *azContainer.Client var err error + + url := fmt.Sprintf("https://%s.blob.core.windows.net/%s", cfg.AccountName, cfg.Container) + opts := &azContainer.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Transport: http.DefaultClient, + }, + } + if cfg.AccountKey.String() != "" { // We have an account key value, find the BlobServiceClient // from with a BasicClient debug.Log(" - using account key") - client, err = storage.NewBasicClient(cfg.AccountName, cfg.AccountKey.Unwrap()) + cred, err := azblob.NewSharedKeyCredential(cfg.AccountName, cfg.AccountKey.Unwrap()) if err != nil { - return nil, errors.Wrap(err, "NewBasicClient") + return nil, errors.Wrap(err, "NewSharedKeyCredential") + } + + client, err = azContainer.NewClientWithSharedKeyCredential(url, cred, opts) + + if err != nil { + return nil, errors.Wrap(err, "NewClientWithSharedKeyCredential") } } else if cfg.AccountSAS.String() != "" { // Get the client using the SAS Token as authentication, this // is longer winded than above because the SDK wants a URL for the Account // if your using a SAS token, and not just the account name // we (as per the SDK ) assume the default Azure portal. - url := fmt.Sprintf("https://%s.blob.core.windows.net/", cfg.AccountName) + // https://github.com/Azure/azure-storage-blob-go/issues/130 debug.Log(" - using sas token") sas := cfg.AccountSAS.Unwrap() + // strip query sign prefix if sas[0] == '?' { sas = sas[1:] } - client, err = storage.NewAccountSASClientFromEndpointToken(url, sas) + + urlWithSAS := fmt.Sprintf("%s?%s", url, sas) + + client, err = azContainer.NewClientWithNoCredential(urlWithSAS, opts) if err != nil { return nil, errors.Wrap(err, "NewAccountSASClientFromEndpointToken") } @@ -70,21 +96,16 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) { return nil, errors.New("no azure authentication information found") } - client.HTTPClient = &http.Client{Transport: rt} - - service := client.GetBlobService() - sem, err := sema.New(cfg.Connections) if err != nil { return nil, err } be := &Backend{ - container: service.GetContainerReference(cfg.Container), - accountName: cfg.AccountName, + container: client, + cfg: cfg, connections: cfg.Connections, sem: sem, - prefix: cfg.Prefix, Layout: &layout.DefaultLayout{ Path: cfg.Prefix, Join: path.Join, @@ -96,26 +117,27 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) { } // Open opens the Azure backend at specified container. -func Open(cfg Config, rt http.RoundTripper) (*Backend, error) { +func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, error) { return open(cfg, rt) } // Create opens the Azure backend at specified container and creates the container if // it does not exist yet. -func Create(cfg Config, rt http.RoundTripper) (*Backend, error) { +func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, error) { be, err := open(cfg, rt) if err != nil { return nil, errors.Wrap(err, "open") } - options := storage.CreateContainerOptions{ - Access: storage.ContainerAccessTypePrivate, - } + if err != nil && bloberror.HasCode(err, bloberror.ContainerNotFound) { + _, err = be.container.Create(ctx, &azContainer.CreateOptions{}) - _, err = be.container.CreateIfNotExists(&options) - if err != nil { - return nil, errors.Wrap(err, "container.CreateIfNotExists") + if err != nil { + return nil, errors.Wrap(err, "container.Create") + } + } else if err != nil { + return be, err } return be, nil @@ -129,8 +151,7 @@ func (be *Backend) SetListMaxItems(i int) { // IsNotExist returns true if the error is caused by a not existing file. func (be *Backend) IsNotExist(err error) bool { debug.Log("IsNotExist(%T, %#v)", err, err) - var aerr storage.AzureStorageServiceError - return errors.As(err, &aerr) && aerr.StatusCode == http.StatusNotFound + return bloberror.HasCode(err, bloberror.BlobNotFound) } // Join combines path components with slashes. @@ -144,7 +165,7 @@ func (be *Backend) Connections() uint { // Location returns this backend's location (the container name). func (be *Backend) Location() string { - return be.Join(be.container.Name, be.prefix) + return be.Join(be.cfg.AccountName, be.cfg.Prefix) } // Hasher may return a hash function for calculating a content hash for the backend @@ -162,16 +183,6 @@ func (be *Backend) Path() string { return be.prefix } -type azureAdapter struct { - restic.RewindReader -} - -func (azureAdapter) Close() error { return nil } - -func (a azureAdapter) Len() int { - return int(a.Length()) -} - // Save stores data in the backend at the handle. func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { if err := h.Valid(); err != nil { @@ -184,41 +195,53 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe be.sem.GetToken() - debug.Log("InsertObject(%v, %v)", be.container.Name, objName) + debug.Log("InsertObject(%v, %v)", be.cfg.AccountName, objName) var err error - if rd.Length() < 256*1024*1024 { - // wrap the reader so that net/http client cannot close the reader - // CreateBlockBlobFromReader reads length from `Len()`` - dataReader := azureAdapter{rd} - + if rd.Length() < saveLargeSize { // if it's smaller than 256miB, then just create the file directly from the reader - ref := be.container.GetBlobReference(objName) - ref.Properties.ContentMD5 = base64.StdEncoding.EncodeToString(rd.Hash()) - err = ref.CreateBlockBlobFromReader(dataReader, nil) + err = be.saveSmall(ctx, objName, rd) } else { // otherwise use the more complicated method err = be.saveLarge(ctx, objName, rd) - } be.sem.ReleaseToken() debug.Log("%v, err %#v", objName, err) - return errors.Wrap(err, "CreateBlockBlobFromReader") + return err +} + +func (be *Backend) saveSmall(ctx context.Context, objName string, rd restic.RewindReader) error { + blockBlobClient := be.container.NewBlockBlobClient(objName) + + // upload it as a new "block", use the base64 hash for the ID + id := base64.StdEncoding.EncodeToString(rd.Hash()) + + buf := make([]byte, rd.Length()) + _, err := io.ReadFull(rd, buf) + if err != nil { + return errors.Wrap(err, "ReadFull") + } + + reader := bytes.NewReader(buf) + _, err = blockBlobClient.StageBlock(ctx, id, streaming.NopCloser(reader), &blockblob.StageBlockOptions{ + TransactionalContentMD5: rd.Hash(), + }) + if err != nil { + return errors.Wrap(err, "StageBlock") + } + + blocks := []string{id} + _, err = blockBlobClient.CommitBlockList(ctx, blocks, &blockblob.CommitBlockListOptions{}) + return errors.Wrap(err, "CommitBlockList") } func (be *Backend) saveLarge(ctx context.Context, objName string, rd restic.RewindReader) error { - // create the file on the server - file := be.container.GetBlobReference(objName) - err := file.CreateBlockBlob(nil) - if err != nil { - return errors.Wrap(err, "CreateBlockBlob") - } + blockBlobClient := be.container.NewBlockBlobClient(objName) - // read the data, in 100 MiB chunks buf := make([]byte, 100*1024*1024) - var blocks []storage.Block + blocks := []string{} uploadedBytes := 0 for { @@ -226,6 +249,7 @@ func (be *Backend) saveLarge(ctx context.Context, objName string, rd restic.Rewi if err == io.ErrUnexpectedEOF { err = nil } + if err == io.EOF { // end of file reached, no bytes have been read at all break @@ -241,16 +265,18 @@ func (be *Backend) saveLarge(ctx context.Context, objName string, rd restic.Rewi // upload it as a new "block", use the base64 hash for the ID h := md5.Sum(buf) id := base64.StdEncoding.EncodeToString(h[:]) - debug.Log("PutBlock %v with %d bytes", id, len(buf)) - err = file.PutBlock(id, buf, &storage.PutBlockOptions{ContentMD5: id}) + + reader := bytes.NewReader(buf) + debug.Log("StageBlock %v with %d bytes", id, len(buf)) + _, err = blockBlobClient.StageBlock(ctx, id, streaming.NopCloser(reader), &blockblob.StageBlockOptions{ + TransactionalContentMD5: h[:], + }) + if err != nil { - return errors.Wrap(err, "PutBlock") + return errors.Wrap(err, "StageBlock") } - blocks = append(blocks, storage.Block{ - ID: id, - Status: "Uncommitted", - }) + blocks = append(blocks, id) } // sanity check @@ -258,10 +284,10 @@ func (be *Backend) saveLarge(ctx context.Context, objName string, rd restic.Rewi return errors.Errorf("wrote %d bytes instead of the expected %d bytes", uploadedBytes, rd.Length()) } + _, err := blockBlobClient.CommitBlockList(ctx, blocks, &blockblob.CommitBlockListOptions{}) + debug.Log("uploaded %d parts: %v", len(blocks), blocks) - err = file.PutBlockList(blocks, nil) - debug.Log("PutBlockList returned %v", err) - return errors.Wrap(err, "PutBlockList") + return errors.Wrap(err, "CommitBlockList") } // Load runs fn with a reader that yields the contents of the file at h at the @@ -285,26 +311,22 @@ func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, } objName := be.Filename(h) - blob := be.container.GetBlobReference(objName) - - start := uint64(offset) - var end uint64 - - if length > 0 { - end = uint64(offset + int64(length) - 1) - } else { - end = 0 - } + blockBlobClient := be.container.NewBlobClient(objName) be.sem.GetToken() + resp, err := blockBlobClient.DownloadStream(ctx, &blob.DownloadStreamOptions{ + Range: azblob.HTTPRange{ + Offset: offset, + Count: int64(length), + }, + }) - rd, err := blob.GetRange(&storage.GetBlobRangeOptions{Range: &storage.BlobRange{Start: start, End: end}}) if err != nil { be.sem.ReleaseToken() return nil, err } - return be.sem.ReleaseTokenOnClose(rd, nil), err + return be.sem.ReleaseTokenOnClose(resp.Body, nil), err } // Stat returns information about a blob. @@ -312,10 +334,10 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, debug.Log("%v", h) objName := be.Filename(h) - blob := be.container.GetBlobReference(objName) + blobClient := be.container.NewBlobClient(objName) be.sem.GetToken() - err := blob.GetProperties(nil) + props, err := blobClient.GetProperties(ctx, nil) be.sem.ReleaseToken() if err != nil { @@ -324,7 +346,7 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, } fi := restic.FileInfo{ - Size: int64(blob.Properties.ContentLength), + Size: *props.ContentLength, Name: h.Name, } return fi, nil @@ -333,12 +355,18 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, // Remove removes the blob with the given name and type. func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { objName := be.Filename(h) + blob := be.container.NewBlobClient(objName) be.sem.GetToken() - _, err := be.container.GetBlobReference(objName).DeleteIfExists(nil) + _, err := blob.Delete(ctx, &azblob.DeleteBlobOptions{}) be.sem.ReleaseToken() debug.Log("Remove(%v) at %v -> err %v", h, objName, err) + + if bloberror.HasCode(err, bloberror.BlobNotFound) { + return nil + } + return errors.Wrap(err, "client.RemoveObject") } @@ -354,31 +382,34 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F prefix += "/" } - params := storage.ListBlobsParameters{ - MaxResults: uint(be.listMaxItems), - Prefix: prefix, - } + max := int32(be.listMaxItems) - for { + opts := &azContainer.ListBlobsFlatOptions{ + MaxResults: &max, + Prefix: &prefix, + } + lister := be.container.NewListBlobsFlatPager(opts) + + for lister.More() { be.sem.GetToken() - obj, err := be.container.ListBlobs(params) + resp, err := lister.NextPage(ctx) be.sem.ReleaseToken() if err != nil { return err } - debug.Log("got %v objects", len(obj.Blobs)) + debug.Log("got %v objects", len(resp.Segment.BlobItems)) - for _, item := range obj.Blobs { - m := strings.TrimPrefix(item.Name, prefix) + for _, item := range resp.Segment.BlobItems { + m := strings.TrimPrefix(*item.Name, prefix) if m == "" { continue } fi := restic.FileInfo{ Name: path.Base(m), - Size: item.Properties.ContentLength, + Size: *item.Properties.ContentLength, } if ctx.Err() != nil { @@ -395,11 +426,6 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F } } - - if obj.NextMarker == "" { - break - } - params.Marker = obj.NextMarker } return ctx.Err() diff --git a/internal/backend/azure/azure_test.go b/internal/backend/azure/azure_test.go index 83d43145e..ada6ec2ca 100644 --- a/internal/backend/azure/azure_test.go +++ b/internal/backend/azure/azure_test.go @@ -46,7 +46,8 @@ func newAzureTestSuite(t testing.TB) *test.Suite { Create: func(config interface{}) (restic.Backend, error) { cfg := config.(azure.Config) - be, err := azure.Create(cfg, tr) + ctx := context.TODO() + be, err := azure.Create(ctx, cfg, tr) if err != nil { return nil, err } @@ -66,15 +67,15 @@ func newAzureTestSuite(t testing.TB) *test.Suite { // OpenFn is a function that opens a previously created temporary repository. Open: func(config interface{}) (restic.Backend, error) { cfg := config.(azure.Config) - - return azure.Open(cfg, tr) + ctx := context.TODO() + return azure.Open(ctx, cfg, tr) }, // CleanupFn removes data created during the tests. Cleanup: func(config interface{}) error { cfg := config.(azure.Config) - - be, err := azure.Open(cfg, tr) + ctx := context.TODO() + be, err := azure.Open(ctx, cfg, tr) if err != nil { return err } @@ -155,7 +156,7 @@ func TestUploadLargeFile(t *testing.T) { t.Fatal(err) } - be, err := azure.Create(cfg, tr) + be, err := azure.Create(ctx, cfg, tr) if err != nil { t.Fatal(err) }