Add Azure Blob Storage as backend

Environment variables:
AZURE_ACCOUNT_NAME=storage-account-name
AZURE_ACCOUNT_KEY=storage-account-key

Environment variables for test:
RESTIC_TEST_AZURE_ACCOUNT_NAME=storage-account-name
RESTIC_TEST_AZURE_ACCOUNT_KEY=storage-account-key
RESTIC_TEST_AZURE_REPOSITORY=azure:restic-test-container

Init repository:
$ restic -r azure:container-name:/prefix/dir init

cmd/restic/global.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"syscall"
 
+	"github.com/restic/restic/internal/backend/azure"
 	"github.com/restic/restic/internal/backend/b2"
 	"github.com/restic/restic/internal/backend/local"
 	"github.com/restic/restic/internal/backend/location"
@@ -363,6 +364,23 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro
 		debug.Log("opening s3 repository at %#v", cfg)
 		return cfg, nil
 
+	case "azure":
+		cfg := loc.Config.(azure.Config)
+		if cfg.AccountName == "" {
+			cfg.AccountName = os.Getenv("AZURE_ACCOUNT_NAME")
+		}
+
+		if cfg.AccountKey == "" {
+			cfg.AccountKey = os.Getenv("AZURE_ACCOUNT_KEY")
+		}
+
+		if err := opts.Apply(loc.Scheme, &cfg); err != nil {
+			return nil, err
+		}
+
+		debug.Log("opening gs repository at %#v", cfg)
+		return cfg, nil
+
 	case "swift":
 		cfg := loc.Config.(swift.Config)
 
@@ -429,6 +447,8 @@ func open(s string, opts options.Options) (restic.Backend, error) {
 		be, err = sftp.Open(cfg.(sftp.Config))
 	case "s3":
 		be, err = s3.Open(cfg.(s3.Config))
+	case "azure":
+		be, err = azure.Open(cfg.(azure.Config))
 	case "swift":
 		be, err = swift.Open(cfg.(swift.Config))
 	case "b2":
@@ -477,6 +497,8 @@ func create(s string, opts options.Options) (restic.Backend, error) {
 		return sftp.Create(cfg.(sftp.Config))
 	case "s3":
 		return s3.Create(cfg.(s3.Config))
+	case "azure":
+		return azure.Open(cfg.(azure.Config))
 	case "swift":
 		return swift.Open(cfg.(swift.Config))
 	case "b2":

internal/backend/azure/azure.go

@@ -0,0 +1,378 @@
+package azure
+
+import (
+	"context"
+	"io"
+	"os"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/Azure/azure-sdk-for-go/storage"
+	"github.com/pkg/errors"
+	"github.com/restic/restic/internal/backend"
+	"github.com/restic/restic/internal/debug"
+	"github.com/restic/restic/internal/restic"
+)
+
+// Backend stores data on an azure endpoint.
+type Backend struct {
+	accountName string
+	container   *storage.Container
+	sem         *backend.Semaphore
+	prefix      string
+	backend.Layout
+}
+
+// make sure that *Backend implements backend.Backend
+var _ restic.Backend = &Backend{}
+
+func open(cfg Config) (*Backend, error) {
+	debug.Log("open, config %#v", cfg)
+
+	client, err := storage.NewBasicClient(cfg.AccountName, cfg.AccountKey)
+	if err != nil {
+		return nil, errors.Wrap(err, "NewBasicClient")
+	}
+
+	service := client.GetBlobService()
+
+	sem, err := backend.NewSemaphore(cfg.Connections)
+	if err != nil {
+		return nil, err
+	}
+
+	be := &Backend{
+		container:   service.GetContainerReference(cfg.Container),
+		accountName: cfg.AccountName,
+		sem:         sem,
+		prefix:      cfg.Prefix,
+		Layout: &backend.DefaultLayout{
+			Path: cfg.Prefix,
+			Join: path.Join,
+		},
+	}
+
+	return be, nil
+}
+
+// Open opens the Azure backend at specified container.
+func Open(cfg Config) (restic.Backend, error) {
+	return open(cfg)
+}
+
+// Create opens the Azure backend at specified container and creates the container if
+// it does not exist yet.
+func Create(cfg Config) (restic.Backend, error) {
+	be, err := open(cfg)
+
+	if err != nil {
+		return nil, errors.Wrap(err, "open")
+	}
+
+	options := storage.CreateContainerOptions{
+		Access: storage.ContainerAccessTypePrivate,
+	}
+
+	_, err = be.container.CreateIfNotExists(&options)
+	if err != nil {
+		return nil, errors.Wrap(err, "container.CreateIfNotExists")
+	}
+
+	return be, nil
+}
+
+// IsNotExist returns true if the error is caused by a not existing file.
+func (be *Backend) IsNotExist(err error) bool {
+	debug.Log("IsNotExist(%T, %#v)", err, err)
+	return os.IsNotExist(err)
+}
+
+// Join combines path components with slashes.
+func (be *Backend) Join(p ...string) string {
+	return path.Join(p...)
+}
+
+type fileInfo struct {
+	name    string
+	size    int64
+	mode    os.FileMode
+	modTime time.Time
+	isDir   bool
+}
+
+func (fi fileInfo) Name() string       { return fi.name }    // base name of the file
+func (fi fileInfo) Size() int64        { return fi.size }    // length in bytes for regular files; system-dependent for others
+func (fi fileInfo) Mode() os.FileMode  { return fi.mode }    // file mode bits
+func (fi fileInfo) ModTime() time.Time { return fi.modTime } // modification time
+func (fi fileInfo) IsDir() bool        { return fi.isDir }   // abbreviation for Mode().IsDir()
+func (fi fileInfo) Sys() interface{}   { return nil }        // underlying data source (can return nil)
+
+// ReadDir returns the entries for a directory.
+func (be *Backend) ReadDir(dir string) (list []os.FileInfo, err error) {
+	debug.Log("ReadDir(%v)", dir)
+
+	// make sure dir ends with a slash
+	if dir[len(dir)-1] != '/' {
+		dir += "/"
+	}
+
+	obj, err := be.container.ListBlobs(storage.ListBlobsParameters{Prefix: dir, Delimiter: "/"})
+	if err != nil {
+		return nil, err
+	}
+
+	for _, item := range obj.BlobPrefixes {
+		entry := fileInfo{
+			name:  strings.TrimPrefix(item, dir),
+			isDir: true,
+			mode:  os.ModeDir | 0755,
+		}
+		if entry.name != "" {
+			list = append(list, entry)
+		}
+	}
+
+	for _, item := range obj.Blobs {
+		entry := fileInfo{
+			name:    strings.TrimPrefix(item.Name, dir),
+			isDir:   false,
+			mode:    0644,
+			size:    item.Properties.ContentLength,
+			modTime: time.Time(item.Properties.LastModified),
+		}
+		if entry.name != "" {
+			list = append(list, entry)
+		}
+	}
+
+	return list, nil
+}
+
+// Location returns this backend's location (the container name).
+func (be *Backend) Location() string {
+	return be.Join(be.container.Name, be.prefix)
+}
+
+// Path returns the path in the bucket that is used for this backend.
+func (be *Backend) Path() string {
+	return be.prefix
+}
+
+// preventCloser wraps an io.Reader to run a function instead of the original Close() function.
+type preventCloser struct {
+	io.Reader
+	f func()
+}
+
+func (wr preventCloser) Close() error {
+	wr.f()
+	return nil
+}
+
+// Save stores data in the backend at the handle.
+func (be *Backend) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err error) {
+	if err := h.Valid(); err != nil {
+		return err
+	}
+
+	objName := be.Filename(h)
+
+	debug.Log("Save %v at %v", h, objName)
+
+	// Check key does not already exist
+	found, err := be.container.GetBlobReference(objName).Exists()
+	if err != nil {
+		return errors.Wrap(err, "GetBlobReference().Exists()")
+	}
+	if found {
+		debug.Log("%v already exists", h)
+		return errors.New("key already exists")
+	}
+
+	be.sem.GetToken()
+
+	// wrap the reader so that net/http client cannot close the reader, return
+	// the token instead.
+	rd = preventCloser{
+		Reader: rd,
+		f: func() {
+			debug.Log("Close()")
+		},
+	}
+
+	debug.Log("InsertObject(%v, %v)", be.container.Name, objName)
+
+	err = be.container.GetBlobReference(objName).CreateBlockBlobFromReader(rd, nil)
+
+	be.sem.ReleaseToken()
+	debug.Log("%v, err %#v", objName, err)
+
+	return errors.Wrap(err, "CreateBlockBlobFromReader")
+}
+
+// wrapReader wraps an io.ReadCloser to run an additional function on Close.
+type wrapReader struct {
+	io.ReadCloser
+	f func()
+}
+
+func (wr wrapReader) Close() error {
+	err := wr.ReadCloser.Close()
+	wr.f()
+	return err
+}
+
+// Load returns a reader that yields the contents of the file at h at the
+// given offset. If length is nonzero, only a portion of the file is
+// returned. rd must be closed after use.
+func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	debug.Log("Load %v, length %v, offset %v from %v", h, length, offset, be.Filename(h))
+	if err := h.Valid(); err != nil {
+		return nil, err
+	}
+
+	if offset < 0 {
+		return nil, errors.New("offset is negative")
+	}
+
+	if length < 0 {
+		return nil, errors.Errorf("invalid length %d", length)
+	}
+
+	objName := be.Filename(h)
+	blob := be.container.GetBlobReference(objName)
+
+	start := uint64(offset)
+	var end uint64
+
+	if length > 0 {
+		end = uint64(offset + int64(length) - 1)
+	} else {
+		end = 0
+	}
+
+	be.sem.GetToken()
+
+	rd, err := blob.GetRange(&storage.GetBlobRangeOptions{Range: &storage.BlobRange{Start: start, End: end}})
+	if err != nil {
+		be.sem.ReleaseToken()
+		return nil, err
+	}
+
+	closeRd := wrapReader{
+		ReadCloser: rd,
+		f: func() {
+			debug.Log("Close()")
+			be.sem.ReleaseToken()
+		},
+	}
+
+	return closeRd, err
+}
+
+// Stat returns information about a blob.
+func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) {
+	debug.Log("%v", h)
+
+	objName := be.Filename(h)
+	blob := be.container.GetBlobReference(objName)
+
+	if err := blob.GetProperties(nil); err != nil {
+		debug.Log("blob.GetProperties err %v", err)
+		return restic.FileInfo{}, errors.Wrap(err, "blob.GetProperties")
+	}
+
+	return restic.FileInfo{Size: int64(blob.Properties.ContentLength)}, nil
+}
+
+// Test returns true if a blob of the given type and name exists in the backend.
+func (be *Backend) Test(ctx context.Context, h restic.Handle) (bool, error) {
+	objName := be.Filename(h)
+	found, err := be.container.GetBlobReference(objName).Exists()
+	if err != nil {
+		return false, err
+	}
+	return found, nil
+}
+
+// Remove removes the blob with the given name and type.
+func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
+	objName := be.Filename(h)
+	_, err := be.container.GetBlobReference(objName).DeleteIfExists(nil)
+	debug.Log("Remove(%v) at %v -> err %v", h, objName, err)
+	return errors.Wrap(err, "client.RemoveObject")
+}
+
+// List returns a channel that yields all names of blobs of type t. A
+// goroutine is started for this. If the channel done is closed, sending
+// stops.
+func (be *Backend) List(ctx context.Context, t restic.FileType) <-chan string {
+	debug.Log("listing %v", t)
+	ch := make(chan string)
+
+	prefix := be.Dirname(restic.Handle{Type: t})
+
+	// make sure prefix ends with a slash
+	if prefix[len(prefix)-1] != '/' {
+		prefix += "/"
+	}
+
+	go func() {
+		defer close(ch)
+
+		obj, err := be.container.ListBlobs(storage.ListBlobsParameters{Prefix: prefix})
+		if err != nil {
+			return
+		}
+
+		for _, item := range obj.Blobs {
+			m := strings.TrimPrefix(item.Name, prefix)
+			if m == "" {
+				continue
+			}
+
+			select {
+			case ch <- path.Base(m):
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+
+	return ch
+}
+
+// Remove keys for a specified backend type.
+func (be *Backend) removeKeys(ctx context.Context, t restic.FileType) error {
+	for key := range be.List(ctx, restic.DataFile) {
+		err := be.Remove(ctx, restic.Handle{Type: restic.DataFile, Name: key})
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Delete removes all restic keys in the bucket. It will not remove the bucket itself.
+func (be *Backend) Delete(ctx context.Context) error {
+	alltypes := []restic.FileType{
+		restic.DataFile,
+		restic.KeyFile,
+		restic.LockFile,
+		restic.SnapshotFile,
+		restic.IndexFile}
+
+	for _, t := range alltypes {
+		err := be.removeKeys(ctx, t)
+		if err != nil {
+			return nil
+		}
+	}
+
+	return be.Remove(ctx, restic.Handle{Type: restic.ConfigFile})
+}
+
+// Close does nothing
+func (be *Backend) Close() error { return nil }

internal/backend/azure/azure_test.go

@@ -0,0 +1,121 @@
+package azure_test
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/restic/restic/internal/backend/azure"
+	"github.com/restic/restic/internal/backend/test"
+	"github.com/restic/restic/internal/restic"
+	. "github.com/restic/restic/internal/test"
+)
+
+func newAzureTestSuite(t testing.TB) *test.Suite {
+	return &test.Suite{
+		// do not use excessive data
+		MinimalData: true,
+
+		// NewConfig returns a config for a new temporary backend that will be used in tests.
+		NewConfig: func() (interface{}, error) {
+			azcfg, err := azure.ParseConfig(os.Getenv("RESTIC_TEST_AZURE_REPOSITORY"))
+			if err != nil {
+				return nil, err
+			}
+
+			cfg := azcfg.(azure.Config)
+			cfg.AccountName = os.Getenv("RESTIC_TEST_AZURE_ACCOUNT_NAME")
+			cfg.AccountKey = os.Getenv("RESTIC_TEST_AZURE_ACCOUNT_KEY")
+			cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano())
+			return cfg, nil
+		},
+
+		// CreateFn is a function that creates a temporary repository for the tests.
+		Create: func(config interface{}) (restic.Backend, error) {
+			cfg := config.(azure.Config)
+
+			be, err := azure.Create(cfg)
+			if err != nil {
+				return nil, err
+			}
+
+			exists, err := be.Test(context.TODO(), restic.Handle{Type: restic.ConfigFile})
+			if err != nil {
+				return nil, err
+			}
+
+			if exists {
+				return nil, errors.New("config already exists")
+			}
+
+			return be, nil
+		},
+
+		// OpenFn is a function that opens a previously created temporary repository.
+		Open: func(config interface{}) (restic.Backend, error) {
+			cfg := config.(azure.Config)
+			return azure.Open(cfg)
+		},
+
+		// CleanupFn removes data created during the tests.
+		Cleanup: func(config interface{}) error {
+			cfg := config.(azure.Config)
+
+			be, err := azure.Open(cfg)
+			if err != nil {
+				return err
+			}
+
+			if err := be.(restic.Deleter).Delete(context.TODO()); err != nil {
+				return err
+			}
+
+			return nil
+		},
+	}
+}
+
+func TestBackendAzure(t *testing.T) {
+	defer func() {
+		if t.Skipped() {
+			SkipDisallowed(t, "restic/backend/azure.TestBackendAzure")
+		}
+	}()
+
+	vars := []string{
+		"RESTIC_TEST_AZURE_ACCOUNT_NAME",
+		"RESTIC_TEST_AZURE_ACCOUNT_KEY",
+		"RESTIC_TEST_AZURE_REPOSITORY",
+	}
+
+	for _, v := range vars {
+		if os.Getenv(v) == "" {
+			t.Skipf("environment variable %v not set", v)
+			return
+		}
+	}
+
+	t.Logf("run tests")
+	newAzureTestSuite(t).RunTests(t)
+}
+
+func BenchmarkBackendAzure(t *testing.B) {
+	vars := []string{
+		"RESTIC_TEST_AZURE_ACCOUNT_NAME",
+		"RESTIC_TEST_AZURE_ACCOUNT_KEY",
+		"RESTIC_TEST_AZURE_REPOSITORY",
+	}
+
+	for _, v := range vars {
+		if os.Getenv(v) == "" {
+			t.Skipf("environment variable %v not set", v)
+			return
+		}
+	}
+
+	t.Logf("run tests")
+	newAzureTestSuite(t).RunBenchmarks(t)
+}

internal/backend/azure/config.go

@@ -0,0 +1,57 @@
+package azure
+
+import (
+	"errors"
+	"path"
+	"strings"
+
+	"github.com/restic/restic/internal/options"
+)
+
+// Config contains all configuration necessary to connect to an azure compatible
+// server.
+type Config struct {
+	AccountName string
+	AccountKey  string
+	Container   string
+	Prefix      string
+
+	Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 20)"`
+}
+
+// NewConfig returns a new Config with the default values filled in.
+func NewConfig() Config {
+	return Config{
+		Connections: 5,
+	}
+}
+
+func init() {
+	options.Register("azure", Config{})
+}
+
+// ParseConfig parses the string s and extracts the azure config. The
+// configuration format is azure:containerName:/[prefix].
+func ParseConfig(s string) (interface{}, error) {
+	if strings.HasPrefix(s, "azure:") {
+		s = s[6:]
+	} else {
+		return nil, errors.New("azure: invalid format")
+	}
+	// use the first entry of the path as the container name and the
+	// remainder as prefix
+	path := strings.SplitN(s, ":/", 2)
+	return createConfig(path)
+}
+
+func createConfig(p []string) (interface{}, error) {
+	if len(p) < 2 {
+		return nil, errors.New("azure: invalid format, container name not found")
+	}
+	cfg := NewConfig()
+	cfg.Container = p[0]
+	if p[1] != "" {
+		cfg.Prefix = path.Clean(p[1])
+	}
+	return cfg, nil
+}

internal/backend/azure/config_test.go

@@ -0,0 +1,40 @@
+package azure
+
+import "testing"
+
+var configTests = []struct {
+	s   string
+	cfg Config
+}{
+	{"azure:container-name:/", Config{
+		Container:   "container-name",
+		Prefix:      "",
+		Connections: 5,
+	}},
+	{"azure:container-name:/prefix/directory", Config{
+		Container:   "container-name",
+		Prefix:      "prefix/directory",
+		Connections: 5,
+	}},
+	{"azure:container-name:/prefix/directory/", Config{
+		Container:   "container-name",
+		Prefix:      "prefix/directory",
+		Connections: 5,
+	}},
+}
+
+func TestParseConfig(t *testing.T) {
+	for i, test := range configTests {
+		cfg, err := ParseConfig(test.s)
+		if err != nil {
+			t.Errorf("test %d:%s failed: %v", i, test.s, err)
+			continue
+		}
+
+		if cfg != test.cfg {
+			t.Errorf("test %d:\ninput:\n  %s\n wrong config, want:\n  %v\ngot:\n  %v",
+				i, test.s, test.cfg, cfg)
+			continue
+		}
+	}
+}

internal/backend/location/location.go

@@ -4,6 +4,7 @@ package location
 import (
 	"strings"
 
+	"github.com/restic/restic/internal/backend/azure"
 	"github.com/restic/restic/internal/backend/b2"
 	"github.com/restic/restic/internal/backend/local"
 	"github.com/restic/restic/internal/backend/rest"
@@ -32,6 +33,7 @@ var parsers = []parser{
 	{"local", local.ParseConfig},
 	{"sftp", sftp.ParseConfig},
 	{"s3", s3.ParseConfig},
+	{"azure", azure.ParseConfig},
 	{"swift", swift.ParseConfig},
 	{"rest", rest.ParseConfig},
 }