Implement OpenStack swift backend

This commit implements support for OpenStack swift
storage server, tested on OVH public cloud storage.

Special thanks to jayme-github <tuxnet@gmail.com>
who helped with the implementation.

doc/manual.rst

@@ -282,6 +282,67 @@ this command.
     Please note that knowledge of your password is required to access
     the repository. Losing your password means that your data is irrecoverably lost.
 
+OpenStack Swift
+~~~~~~~~~~~~~~~
+
+Restic can backup data to an OpenStack Swift container. Because Swift supports
+various authentication methods, credentials are passed through environment
+variables. In order to help integration with existing OpenStack installations,
+the naming convention of those variables follows official python swift client:
+
+.. code-block:: console
+
+   # For keystone v1 authentication
+   $ export ST_AUTH=<MY_AUTH_URL>
+   $ export ST_USER=<MY_USER_NAME>
+   $ export ST_KEY=<MY_USER_PASSWORD>
+
+   # For keystone v2 authentication (some variables are optional)
+   $ export OS_AUTH_URL=<MY_AUTH_URL>
+   $ export OS_REGION_NAME=<MY_REGION_NAME>
+   $ export OS_USERNAME=<MY_USERNAME>
+   $ export OS_PASSWORD=<MY_PASSWORD>
+   $ export OS_TENANT_ID=<MY_TENANT_ID>
+   $ export OS_TENANT_NAME=<MY_TENANT_NAME>
+
+   # For keystone v3 authentication (some variables are optional)
+   $ export OS_AUTH_URL=<MY_AUTH_URL>
+   $ export OS_REGION_NAME=<MY_REGION_NAME>
+   $ export OS_USERNAME=<MY_USERNAME>
+   $ export OS_PASSWORD=<MY_PASSWORD>
+   $ export OS_USER_DOMAIN_NAME=<MY_DOMAIN_NAME>
+   $ export OS_PROJECT_NAME=<MY_PROJECT_NAME>
+   $ export OS_PROJECT_DOMAIN_NAME=<MY_PROJECT_DOMAIN_NAME>
+
+   # For authentication based on tokens
+   $ export OS_STORAGE_URL=<MY_STORAGE_URL>
+   $ export OS_AUTH_TOKEN=<MY_AUTH_TOKEN>
+
+
+Restic should be compatible with [OpenStack RC
+file](https://docs.openstack.org/user-guide/common/cli-set-environment-variables-using-openstack-rc.html)
+in most cases.
+
+Once environment variables are set up, a new repository can be created. The
+name of swift container and optional path can be specified. If
+the container does not exist, it will be created automatically:
+
+.. code-block:: console
+
+   $ restic -r swift:container_name:/path init   # path is optional
+   enter password for new backend:
+   enter password again:
+   created restic backend eefee03bbd at swift:container_name:/path
+   Please note that knowledge of your password is required to access the repository.
+   Losing your password means that your data is irrecoverably lost.
+
+The policy of new container created by restic can be changed using environment variable:
+
+.. code-block:: console
+
+   $ export SWIFT_DEFAULT_CONTAINER_POLICY=<MY_CONTAINER_POLICY>
+
+
 Password prompt on Windows
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 

src/cmds/restic/global.go

@@ -16,6 +16,7 @@ import (
 	"restic/backend/rest"
 	"restic/backend/s3"
 	"restic/backend/sftp"
+	"restic/backend/swift"
 	"restic/debug"
 	"restic/options"
 	"restic/repository"
@@ -356,6 +357,51 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro
 		debug.Log("opening s3 repository at %#v", cfg)
 		return cfg, nil
 
+	case "swift":
+		cfg := loc.Config.(swift.Config)
+
+		for _, val := range []struct {
+			s   *string
+			env string
+		}{
+			// v2/v3 specific
+			{&cfg.UserName, "OS_USERNAME"},
+			{&cfg.APIKey, "OS_PASSWORD"},
+			{&cfg.Region, "OS_REGION_NAME"},
+			{&cfg.AuthURL, "OS_AUTH_URL"},
+
+			// v3 specific
+			{&cfg.Domain, "OS_USER_DOMAIN_NAME"},
+			{&cfg.Tenant, "OS_PROJECT_NAME"},
+			{&cfg.TenantDomain, "OS_PROJECT_DOMAIN_NAME"},
+
+			// v2 specific
+			{&cfg.TenantID, "OS_TENANT_ID"},
+			{&cfg.Tenant, "OS_TENANT_NAME"},
+
+			// v1 specific
+			{&cfg.AuthURL, "ST_AUTH"},
+			{&cfg.UserName, "ST_USER"},
+			{&cfg.APIKey, "ST_KEY"},
+
+			// Manual authentication
+			{&cfg.StorageURL, "OS_STORAGE_URL"},
+			{&cfg.AuthToken, "OS_AUTH_TOKEN"},
+
+			{&cfg.DefaultContainerPolicy, "SWIFT_DEFAULT_CONTAINER_POLICY"},
+		} {
+			if *val.s == "" {
+				*val.s = os.Getenv(val.env)
+			}
+		}
+
+		if err := opts.Apply(loc.Scheme, &cfg); err != nil {
+			return nil, err
+		}
+
+		debug.Log("opening swift repository at %#v", cfg)
+		return cfg, nil
+
 	case "rest":
 		cfg := loc.Config.(rest.Config)
 		if err := opts.Apply(loc.Scheme, &cfg); err != nil {
@@ -391,6 +437,8 @@ func open(s string, opts options.Options) (restic.Backend, error) {
 		be, err = sftp.Open(cfg.(sftp.Config))
 	case "s3":
 		be, err = s3.Open(cfg.(s3.Config))
+	case "swift":
+		be, err = swift.Open(cfg.(swift.Config))
 	case "rest":
 		be, err = rest.Open(cfg.(rest.Config))
 
@@ -435,6 +483,8 @@ func create(s string, opts options.Options) (restic.Backend, error) {
 		return sftp.Create(cfg.(sftp.Config))
 	case "s3":
 		return s3.Open(cfg.(s3.Config))
+	case "swift":
+		return swift.Open(cfg.(swift.Config))
 	case "rest":
 		return rest.Create(cfg.(rest.Config))
 	}

src/restic/backend/location/location.go

@@ -8,6 +8,7 @@ import (
 	"restic/backend/rest"
 	"restic/backend/s3"
 	"restic/backend/sftp"
+	"restic/backend/swift"
 )
 
 // Location specifies the location of a repository, including the method of
@@ -28,6 +29,7 @@ var parsers = []parser{
 	{"local", local.ParseConfig},
 	{"sftp", sftp.ParseConfig},
 	{"s3", s3.ParseConfig},
+	{"swift", swift.ParseConfig},
 	{"rest", rest.ParseConfig},
 }
 

src/restic/backend/location/location_test.go

@@ -9,6 +9,7 @@ import (
 	"restic/backend/rest"
 	"restic/backend/s3"
 	"restic/backend/sftp"
+	"restic/backend/swift"
 )
 
 func parseURL(s string) *url.URL {
@@ -195,6 +196,24 @@ var parseTests = []struct {
 			},
 		},
 	},
+	{
+		"swift:container17:/",
+		Location{Scheme: "swift",
+			Config: swift.Config{
+				Container: "container17",
+				Prefix:    "",
+			},
+		},
+	},
+	{
+		"swift:container17:/prefix97",
+		Location{Scheme: "swift",
+			Config: swift.Config{
+				Container: "container17",
+				Prefix:    "prefix97",
+			},
+		},
+	},
 	{
 		"rest:http://hostname.foo:1234/",
 		Location{Scheme: "rest",

src/restic/backend/swift/backend_test.go

@@ -0,0 +1,87 @@
+// DO NOT EDIT, AUTOMATICALLY GENERATED
+package swift_test
+
+import (
+	"testing"
+
+	"restic/backend/test"
+)
+
+var SkipMessage string
+
+func TestSwiftBackendCreate(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestCreate(t)
+}
+
+func TestSwiftBackendOpen(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestOpen(t)
+}
+
+func TestSwiftBackendCreateWithConfig(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestCreateWithConfig(t)
+}
+
+func TestSwiftBackendLocation(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestLocation(t)
+}
+
+func TestSwiftBackendConfig(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestConfig(t)
+}
+
+func TestSwiftBackendLoad(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestLoad(t)
+}
+
+func TestSwiftBackendSave(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestSave(t)
+}
+
+func TestSwiftBackendSaveFilenames(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestSaveFilenames(t)
+}
+
+func TestSwiftBackendBackend(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestBackend(t)
+}
+
+func TestSwiftBackendDelete(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestDelete(t)
+}
+
+func TestSwiftBackendCleanup(t *testing.T) {
+	if SkipMessage != "" {
+		t.Skip(SkipMessage)
+	}
+	test.TestCleanup(t)
+}

src/restic/backend/swift/config.go

@@ -0,0 +1,52 @@
+package swift
+
+import (
+	"net/url"
+	"regexp"
+	"restic/errors"
+)
+
+var (
+	urlParser = regexp.MustCompile("^([^:]+):/(.*)$")
+)
+
+// Config contains basic configuration needed to specify swift location for a swift server
+type Config struct {
+	UserName     string
+	Domain       string
+	APIKey       string
+	AuthURL      string
+	Region       string
+	Tenant       string
+	TenantID     string
+	TenantDomain string
+	TrustID      string
+
+	StorageURL string
+	AuthToken  string
+
+	Container              string
+	Prefix                 string
+	DefaultContainerPolicy string
+}
+
+// ParseConfig parses the string s and extract swift's container name and prefix.
+func ParseConfig(s string) (interface{}, error) {
+
+	url, err := url.Parse(s)
+	if err != nil {
+		return nil, errors.Wrap(err, "url.Parse")
+	}
+
+	m := urlParser.FindStringSubmatch(url.Opaque)
+	if len(m) == 0 {
+		return nil, errors.New("swift: invalid URL, valid syntax is: 'swift:container-name:/[optional-prefix]'")
+	}
+
+	cfg := Config{
+		Container: m[1],
+		Prefix:    m[2],
+	}
+
+	return cfg, nil
+}

src/restic/backend/swift/config_test.go

@@ -0,0 +1,50 @@
+package swift
+
+import "testing"
+
+var configTests = []struct {
+	s   string
+	cfg Config
+}{
+	{"swift:cnt1:/", Config{Container: "cnt1", Prefix: ""}},
+	{"swift:cnt2:/prefix", Config{Container: "cnt2", Prefix: "prefix"}},
+	{"swift:cnt3:/prefix/longer", Config{Container: "cnt3", Prefix: "prefix/longer"}},
+	{"swift:cnt4:/prefix?params", Config{Container: "cnt4", Prefix: "prefix"}},
+	{"swift:cnt5:/prefix#params", Config{Container: "cnt5", Prefix: "prefix"}},
+}
+
+func TestParseConfigInternal(t *testing.T) {
+	for i, test := range configTests {
+		cfg, err := ParseConfig(test.s)
+		if err != nil {
+			t.Errorf("test %d:%s failed: %v", i, test.s, err)
+			continue
+		}
+
+		if cfg != test.cfg {
+			t.Errorf("test %d:\ninput:\n  %s\n wrong config, want:\n  %v\ngot:\n  %v",
+				i, test.s, test.cfg, cfg)
+			continue
+		}
+	}
+}
+
+var configTestsInvalid = []string{
+	"swift://hostname/container",
+	"swift:////",
+	"swift://",
+	"swift:////prefix",
+	"swift:container",
+	"swift:container:",
+	"swift:container/prefix",
+}
+
+func TestParseConfigInvalid(t *testing.T) {
+	for i, test := range configTestsInvalid {
+		_, err := ParseConfig(test)
+		if err == nil {
+			t.Errorf("test %d: invalid config %s did not return an error", i, test)
+			continue
+		}
+	}
+}

src/restic/backend/swift/swift.go

@@ -0,0 +1,335 @@
+package swift
+
+import (
+	"io"
+	"path"
+	"restic"
+	"restic/backend"
+	"restic/debug"
+	"restic/errors"
+	"strings"
+	"time"
+
+	"github.com/ncw/swift"
+)
+
+const connLimit = 10
+
+// beSwift is a backend which stores the data on a swift endpoint.
+type beSwift struct {
+	conn      *swift.Connection
+	connChan  chan struct{}
+	container string // Container name
+	prefix    string // Prefix of object names in the container
+}
+
+// Open opens the swift backend at a container in region. The container is
+// created if it does not exist yet.
+func Open(cfg Config) (restic.Backend, error) {
+
+	be := &beSwift{
+		conn: &swift.Connection{
+			UserName:       cfg.UserName,
+			Domain:         cfg.Domain,
+			ApiKey:         cfg.APIKey,
+			AuthUrl:        cfg.AuthURL,
+			Region:         cfg.Region,
+			Tenant:         cfg.Tenant,
+			TenantId:       cfg.TenantID,
+			TenantDomain:   cfg.TenantDomain,
+			TrustId:        cfg.TrustID,
+			StorageUrl:     cfg.StorageURL,
+			AuthToken:      cfg.AuthToken,
+			ConnectTimeout: time.Minute,
+			Timeout:        time.Minute,
+		},
+		container: cfg.Container,
+		prefix:    cfg.Prefix,
+	}
+	be.createConnections()
+
+	// Authenticate if needed
+	if !be.conn.Authenticated() {
+		if err := be.conn.Authenticate(); err != nil {
+			return nil, errors.Wrap(err, "conn.Authenticate")
+		}
+	}
+
+	// Ensure container exists
+	switch _, _, err := be.conn.Container(be.container); err {
+	case nil:
+		// Container exists
+
+	case swift.ContainerNotFound:
+		err = be.createContainer(cfg.DefaultContainerPolicy)
+		if err != nil {
+			return nil, errors.Wrap(err, "beSwift.createContainer")
+		}
+
+	default:
+		return nil, errors.Wrap(err, "conn.Container")
+	}
+
+	return be, nil
+}
+
+func (be *beSwift) swiftpath(h restic.Handle) string {
+
+	var dir string
+
+	switch h.Type {
+	case restic.ConfigFile:
+		dir = ""
+		h.Name = backend.Paths.Config
+	case restic.DataFile:
+		dir = backend.Paths.Data
+	case restic.SnapshotFile:
+		dir = backend.Paths.Snapshots
+	case restic.IndexFile:
+		dir = backend.Paths.Index
+	case restic.LockFile:
+		dir = backend.Paths.Locks
+	case restic.KeyFile:
+		dir = backend.Paths.Keys
+	default:
+		dir = string(h.Type)
+	}
+
+	return path.Join(be.prefix, dir, h.Name)
+}
+
+func (be *beSwift) createConnections() {
+	be.connChan = make(chan struct{}, connLimit)
+	for i := 0; i < connLimit; i++ {
+		be.connChan <- struct{}{}
+	}
+}
+
+func (be *beSwift) createContainer(policy string) error {
+	var h swift.Headers
+	if policy != "" {
+		h = swift.Headers{
+			"X-Storage-Policy": policy,
+		}
+	}
+
+	return be.conn.ContainerCreate(be.container, h)
+}
+
+// Location returns this backend's location (the container name).
+func (be *beSwift) Location() string {
+	return be.container
+}
+
+// Load returns a reader that yields the contents of the file at h at the
+// given offset. If length is nonzero, only a portion of the file is
+// returned. rd must be closed after use.
+func (be *beSwift) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	debug.Log("Load %v, length %v, offset %v", h, length, offset)
+	if err := h.Valid(); err != nil {
+		return nil, err
+	}
+
+	if offset < 0 {
+		return nil, errors.New("offset is negative")
+	}
+
+	if length < 0 {
+		return nil, errors.Errorf("invalid length %d", length)
+	}
+
+	objName := be.swiftpath(h)
+
+	<-be.connChan
+	defer func() {
+		be.connChan <- struct{}{}
+	}()
+
+	obj, _, err := be.conn.ObjectOpen(be.container, objName, false, nil)
+	if err != nil {
+		debug.Log("  err %v", err)
+		return nil, errors.Wrap(err, "conn.ObjectOpen")
+	}
+
+	// if we're going to read the whole object, just pass it on.
+	if length == 0 {
+		debug.Log("Load %v: pass on object", h)
+		_, err = obj.Seek(offset, 0)
+		if err != nil {
+			_ = obj.Close()
+			return nil, errors.Wrap(err, "obj.Seek")
+		}
+
+		return obj, nil
+	}
+
+	// otherwise pass a LimitReader
+	size, err := obj.Length()
+	if err != nil {
+		return nil, errors.Wrap(err, "obj.Length")
+	}
+
+	if offset > size {
+		_ = obj.Close()
+		return nil, errors.Errorf("offset larger than file size")
+	}
+
+	_, err = obj.Seek(offset, 0)
+	if err != nil {
+		_ = obj.Close()
+		return nil, errors.Wrap(err, "obj.Seek")
+	}
+
+	return backend.LimitReadCloser(obj, int64(length)), nil
+}
+
+// Save stores data in the backend at the handle.
+func (be *beSwift) Save(h restic.Handle, rd io.Reader) (err error) {
+	if err = h.Valid(); err != nil {
+		return err
+	}
+
+	debug.Log("Save %v", h)
+
+	objName := be.swiftpath(h)
+
+	// Check key does not already exist
+	switch _, _, err = be.conn.Object(be.container, objName); err {
+	case nil:
+		debug.Log("%v already exists", h)
+		return errors.New("key already exists")
+
+	case swift.ObjectNotFound:
+		// Ok, that's what we want
+
+	default:
+		return errors.Wrap(err, "conn.Object")
+	}
+
+	<-be.connChan
+	defer func() {
+		be.connChan <- struct{}{}
+	}()
+
+	encoding := "binary/octet-stream"
+
+	debug.Log("PutObject(%v, %v, %v)",
+		be.container, objName, encoding)
+	//err = be.conn.ObjectPutBytes(be.container, objName, p, encoding)
+	_, err = be.conn.ObjectPut(be.container, objName, rd, true, "", encoding, nil)
+	debug.Log("%v, err %#v", objName, err)
+
+	return errors.Wrap(err, "client.PutObject")
+}
+
+// Stat returns information about a blob.
+func (be *beSwift) Stat(h restic.Handle) (bi restic.FileInfo, err error) {
+	debug.Log("%v", h)
+
+	objName := be.swiftpath(h)
+
+	obj, _, err := be.conn.Object(be.container, objName)
+	if err != nil {
+		debug.Log("Object() err %v", err)
+		return restic.FileInfo{}, errors.Wrap(err, "conn.Object")
+	}
+
+	return restic.FileInfo{Size: obj.Bytes}, nil
+}
+
+// Test returns true if a blob of the given type and name exists in the backend.
+func (be *beSwift) Test(h restic.Handle) (bool, error) {
+	objName := be.swiftpath(h)
+	switch _, _, err := be.conn.Object(be.container, objName); err {
+	case nil:
+		return true, nil
+
+	case swift.ObjectNotFound:
+		return false, nil
+
+	default:
+		return false, errors.Wrap(err, "conn.Object")
+	}
+}
+
+// Remove removes the blob with the given name and type.
+func (be *beSwift) Remove(h restic.Handle) error {
+	objName := be.swiftpath(h)
+	err := be.conn.ObjectDelete(be.container, objName)
+	debug.Log("Remove(%v) -> err %v", h, err)
+	return errors.Wrap(err, "conn.ObjectDelete")
+}
+
+// List returns a channel that yields all names of blobs of type t. A
+// goroutine is started for this. If the channel done is closed, sending
+// stops.
+func (be *beSwift) List(t restic.FileType, done <-chan struct{}) <-chan string {
+	debug.Log("listing %v", t)
+	ch := make(chan string)
+
+	prefix := be.swiftpath(restic.Handle{Type: t}) + "/"
+
+	go func() {
+		defer close(ch)
+
+		be.conn.ObjectsWalk(be.container, &swift.ObjectsOpts{Prefix: prefix},
+			func(opts *swift.ObjectsOpts) (interface{}, error) {
+				newObjects, err := be.conn.ObjectNames(be.container, opts)
+				if err != nil {
+					return nil, errors.Wrap(err, "conn.ObjectNames")
+				}
+				for _, obj := range newObjects {
+					m := strings.TrimPrefix(obj, prefix)
+					if m == "" {
+						continue
+					}
+
+					select {
+					case ch <- m:
+					case <-done:
+						return nil, io.EOF
+					}
+				}
+				return newObjects, nil
+			})
+	}()
+
+	return ch
+}
+
+// Remove keys for a specified backend type.
+func (be *beSwift) removeKeys(t restic.FileType) error {
+	done := make(chan struct{})
+	defer close(done)
+	for key := range be.List(restic.DataFile, done) {
+		err := be.Remove(restic.Handle{Type: restic.DataFile, Name: key})
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Delete removes all restic objects in the container.
+// It will not remove the container itself.
+func (be *beSwift) Delete() error {
+	alltypes := []restic.FileType{
+		restic.DataFile,
+		restic.KeyFile,
+		restic.LockFile,
+		restic.SnapshotFile,
+		restic.IndexFile}
+
+	for _, t := range alltypes {
+		err := be.removeKeys(t)
+		if err != nil {
+			return nil
+		}
+	}
+
+	return be.Remove(restic.Handle{Type: restic.ConfigFile})
+}
+
+// Close does nothing
+func (be *beSwift) Close() error { return nil }

src/restic/backend/swift/swift_test.go

@@ -0,0 +1,76 @@
+package swift_test
+
+import (
+	"fmt"
+	"math/rand"
+	"restic"
+	"time"
+
+	"restic/errors"
+
+	"restic/backend/swift"
+	"restic/backend/test"
+	. "restic/test"
+
+	swiftclient "github.com/ncw/swift"
+)
+
+//go:generate go run ../test/generate_backend_tests.go
+
+func init() {
+	if TestSwiftServer == "" {
+		SkipMessage = "swift test server not available"
+		return
+	}
+
+	// Generate random container name to allow simultaneous test
+	// on the same swift backend
+	containerName := fmt.Sprintf(
+		"restictestcontainer_%d_%d",
+		time.Now().Unix(),
+		rand.Uint32(),
+	)
+
+	cfg := swift.Config{
+		Container:  containerName,
+		StorageURL: TestSwiftServer,
+		AuthToken:  TestSwiftToken,
+	}
+
+	test.CreateFn = func() (restic.Backend, error) {
+		be, err := swift.Open(cfg)
+		if err != nil {
+			return nil, err
+		}
+
+		exists, err := be.Test(restic.Handle{Type: restic.ConfigFile})
+		if err != nil {
+			return nil, err
+		}
+
+		if exists {
+			return nil, errors.New("config already exists")
+		}
+
+		return be, nil
+	}
+
+	test.OpenFn = func() (restic.Backend, error) {
+		return swift.Open(cfg)
+	}
+
+	test.CleanupFn = func() error {
+		client := swiftclient.Connection{
+			StorageUrl: TestSwiftServer,
+			AuthToken:  TestSwiftToken,
+		}
+		objects, err := client.ObjectsAll(containerName, nil)
+		if err != nil {
+			return err
+		}
+		for _, o := range objects {
+			client.ObjectDelete(containerName, o.Name)
+		}
+		return client.ContainerDelete(containerName)
+	}
+}

src/restic/backend/test/tests.go

@@ -434,6 +434,20 @@ func testLoad(b restic.Backend, h restic.Handle, length int, offset int64) error
 	return err
 }
 
+func delayedRemove(b restic.Backend, h restic.Handle) error {
+	// Some backend (swift, I'm looking at you) may implement delayed
+	// removal of data. Let's wait a bit if this happens.
+	err := b.Remove(h)
+	found, err := b.Test(h)
+	for i := 0; found && i < 10; i++ {
+		found, err = b.Test(h)
+		if found {
+			time.Sleep(100 * time.Millisecond)
+		}
+	}
+	return err
+}
+
 // TestBackend tests all functions of the backend.
 func (s *Suite) TestBackend(t *testing.T) {
 	b := s.open(t)
@@ -508,7 +522,7 @@ func (s *Suite) TestBackend(t *testing.T) {
 		test.Assert(t, err != nil, "expected error for %v, got %v", h, err)
 
 		// remove and recreate
-		err = b.Remove(h)
+		err = delayedRemove(b, h)
 		test.OK(t, err)
 
 		// test that the blob is gone
@@ -558,7 +572,7 @@ func (s *Suite) TestBackend(t *testing.T) {
 				test.OK(t, err)
 				test.Assert(t, found, fmt.Sprintf("id %q not found", id))
 
-				test.OK(t, b.Remove(h))
+				test.OK(t, delayedRemove(b, h))
 
 				found, err = b.Test(h)
 				test.OK(t, err)

src/restic/test/vars.go

@@ -18,6 +18,8 @@ var (
 	BenchArchiveDirectory       = getStringVar("RESTIC_BENCH_DIR", ".")
 	TestS3Server                = getStringVar("RESTIC_TEST_S3_SERVER", "")
 	TestRESTServer              = getStringVar("RESTIC_TEST_REST_SERVER", "")
+	TestSwiftServer             = getStringVar("RESTIC_TEST_SWIFT_SERVER", "")
+	TestSwiftToken              = getStringVar("RESTIC_TEST_SWIFT_TOKEN", "")
 	TestIntegrationDisallowSkip = getStringVar("RESTIC_TEST_DISALLOW_SKIP", "")
 )