mirror of
https://github.com/octoleo/restic.git
synced 2024-11-02 11:46:36 +00:00
604 lines
16 KiB
Go
604 lines
16 KiB
Go
// Copyright 2016, Google
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
// Package b2 provides a high-level interface to Backblaze's B2 cloud storage
|
|
// service.
|
|
//
|
|
// It is specifically designed to abstract away the Backblaze API details by
|
|
// providing familiar Go interfaces, specifically an io.Writer for object
|
|
// storage, and an io.Reader for object download. Handling of transient
|
|
// errors, including network and authentication timeouts, is transparent.
|
|
//
|
|
// Methods that perform network requests accept a context.Context argument.
|
|
// Callers should use the context's cancellation abilities to end requests
|
|
// early, or to provide timeout or deadline guarantees.
|
|
//
|
|
// This package is in development and may make API changes.
|
|
package b2
|
|
|
|
import (
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"regexp"
|
|
"strconv"
|
|
"sync"
|
|
"time"
|
|
|
|
"golang.org/x/net/context"
|
|
)
|
|
|
|
// Client is a Backblaze B2 client.
|
|
type Client struct {
|
|
backend beRootInterface
|
|
|
|
slock sync.Mutex
|
|
sWriters map[string]*Writer
|
|
sReaders map[string]*Reader
|
|
}
|
|
|
|
// NewClient creates and returns a new Client with valid B2 service account
|
|
// tokens.
|
|
func NewClient(ctx context.Context, account, key string, opts ...ClientOption) (*Client, error) {
|
|
c := &Client{
|
|
backend: &beRoot{
|
|
b2i: &b2Root{},
|
|
},
|
|
}
|
|
if err := c.backend.authorizeAccount(ctx, account, key, opts...); err != nil {
|
|
return nil, err
|
|
}
|
|
return c, nil
|
|
}
|
|
|
|
type clientOptions struct {
|
|
transport http.RoundTripper
|
|
failSomeUploads bool
|
|
expireTokens bool
|
|
capExceeded bool
|
|
}
|
|
|
|
// A ClientOption allows callers to adjust various per-client settings.
|
|
type ClientOption func(*clientOptions)
|
|
|
|
// Transport sets the underlying HTTP transport mechanism. If unset,
|
|
// http.DefaultTransport is used.
|
|
func Transport(rt http.RoundTripper) ClientOption {
|
|
return func(c *clientOptions) {
|
|
c.transport = rt
|
|
}
|
|
}
|
|
|
|
// FailSomeUploads requests intermittent upload failures from the B2 service.
|
|
// This is mostly useful for testing.
|
|
func FailSomeUploads() ClientOption {
|
|
return func(c *clientOptions) {
|
|
c.failSomeUploads = true
|
|
}
|
|
}
|
|
|
|
// ExpireSomeAuthTokens requests intermittent authentication failures from the
|
|
// B2 service.
|
|
func ExpireSomeAuthTokens() ClientOption {
|
|
return func(c *clientOptions) {
|
|
c.expireTokens = true
|
|
}
|
|
}
|
|
|
|
// ForceCapExceeded requests a cap limit from the B2 service. This causes all
|
|
// uploads to be treated as if they would exceed the configure B2 capacity.
|
|
func ForceCapExceeded() ClientOption {
|
|
return func(c *clientOptions) {
|
|
c.capExceeded = true
|
|
}
|
|
}
|
|
|
|
// Bucket is a reference to a B2 bucket.
|
|
type Bucket struct {
|
|
b beBucketInterface
|
|
r beRootInterface
|
|
|
|
c *Client
|
|
}
|
|
|
|
type BucketType string
|
|
|
|
const (
|
|
UnknownType BucketType = ""
|
|
Private = "allPrivate"
|
|
Public = "allPublic"
|
|
)
|
|
|
|
// BucketAttrs holds a bucket's metadata attributes.
|
|
type BucketAttrs struct {
|
|
// Type lists or sets the new bucket type. If Type is UnknownType during a
|
|
// bucket.Update, the type is not changed.
|
|
Type BucketType
|
|
|
|
// Info records user data, limited to ten keys. If nil during a
|
|
// bucket.Update, the existing bucket info is not modified. A bucket's
|
|
// metadata can be removed by updating with an empty map.
|
|
Info map[string]string
|
|
|
|
// Reports or sets bucket lifecycle rules. If nil during a bucket.Update,
|
|
// the rules are not modified. A bucket's rules can be removed by updating
|
|
// with an empty slice.
|
|
LifecycleRules []LifecycleRule
|
|
}
|
|
|
|
// A LifecycleRule describes an object's life cycle, namely how many days after
|
|
// uploading an object should be hidden, and after how many days hidden an
|
|
// object should be deleted. Multiple rules may not apply to the same file or
|
|
// set of files. Be careful when using this feature; it can (is designed to)
|
|
// delete your data.
|
|
type LifecycleRule struct {
|
|
// Prefix specifies all the files in the bucket to which this rule applies.
|
|
Prefix string
|
|
|
|
// DaysUploadedUntilHidden specifies the number of days after which a file
|
|
// will automatically be hidden. 0 means "do not automatically hide new
|
|
// files".
|
|
DaysNewUntilHidden int
|
|
|
|
// DaysHiddenUntilDeleted specifies the number of days after which a hidden
|
|
// file is deleted. 0 means "do not automatically delete hidden files".
|
|
DaysHiddenUntilDeleted int
|
|
}
|
|
|
|
type b2err struct {
|
|
err error
|
|
notFoundErr bool
|
|
isUpdateConflict bool
|
|
}
|
|
|
|
func (e b2err) Error() string {
|
|
return e.err.Error()
|
|
}
|
|
|
|
// IsNotExist reports whether a given error indicates that an object or bucket
|
|
// does not exist.
|
|
func IsNotExist(err error) bool {
|
|
berr, ok := err.(b2err)
|
|
if !ok {
|
|
return false
|
|
}
|
|
return berr.notFoundErr
|
|
}
|
|
|
|
// Bucket returns a bucket if it exists.
|
|
func (c *Client) Bucket(ctx context.Context, name string) (*Bucket, error) {
|
|
buckets, err := c.backend.listBuckets(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, bucket := range buckets {
|
|
if bucket.name() == name {
|
|
return &Bucket{
|
|
b: bucket,
|
|
r: c.backend,
|
|
c: c,
|
|
}, nil
|
|
}
|
|
}
|
|
return nil, b2err{
|
|
err: fmt.Errorf("%s: bucket not found", name),
|
|
notFoundErr: true,
|
|
}
|
|
}
|
|
|
|
// NewBucket returns a bucket. The bucket is created with the given attributes
|
|
// if it does not already exist. If attrs is nil, it is created as a private
|
|
// bucket with no info metadata and no lifecycle rules.
|
|
func (c *Client) NewBucket(ctx context.Context, name string, attrs *BucketAttrs) (*Bucket, error) {
|
|
buckets, err := c.backend.listBuckets(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, bucket := range buckets {
|
|
if bucket.name() == name {
|
|
return &Bucket{
|
|
b: bucket,
|
|
r: c.backend,
|
|
c: c,
|
|
}, nil
|
|
}
|
|
}
|
|
if attrs == nil {
|
|
attrs = &BucketAttrs{Type: Private}
|
|
}
|
|
b, err := c.backend.createBucket(ctx, name, string(attrs.Type), attrs.Info, attrs.LifecycleRules)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &Bucket{
|
|
b: b,
|
|
r: c.backend,
|
|
c: c,
|
|
}, err
|
|
}
|
|
|
|
// ListBucket returns all the available buckets.
|
|
func (c *Client) ListBuckets(ctx context.Context) ([]*Bucket, error) {
|
|
bs, err := c.backend.listBuckets(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var buckets []*Bucket
|
|
for _, b := range bs {
|
|
buckets = append(buckets, &Bucket{
|
|
b: b,
|
|
r: c.backend,
|
|
c: c,
|
|
})
|
|
}
|
|
return buckets, nil
|
|
}
|
|
|
|
// IsUpdateConflict reports whether a given error is the result of a bucket
|
|
// update conflict.
|
|
func IsUpdateConflict(err error) bool {
|
|
e, ok := err.(b2err)
|
|
if !ok {
|
|
return false
|
|
}
|
|
return e.isUpdateConflict
|
|
}
|
|
|
|
// Update modifies the given bucket with new attributes. It is possible that
|
|
// this method could fail with an update conflict, in which case you should
|
|
// retrieve the latest bucket attributes with Attrs and try again.
|
|
func (b *Bucket) Update(ctx context.Context, attrs *BucketAttrs) error {
|
|
return b.b.updateBucket(ctx, attrs)
|
|
}
|
|
|
|
// Attrs retrieves and returns the current bucket's attributes.
|
|
func (b *Bucket) Attrs(ctx context.Context) (*BucketAttrs, error) {
|
|
bucket, err := b.c.Bucket(ctx, b.Name())
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
b.b = bucket.b
|
|
return b.b.attrs(), nil
|
|
}
|
|
|
|
var bNotExist = regexp.MustCompile("Bucket.*does not exist")
|
|
|
|
// Delete removes a bucket. The bucket must be empty.
|
|
func (b *Bucket) Delete(ctx context.Context) error {
|
|
err := b.b.deleteBucket(ctx)
|
|
if err == nil {
|
|
return err
|
|
}
|
|
// So, the B2 documentation disagrees with the implementation here, and the
|
|
// error code is not really helpful. If the bucket doesn't exist, the error is
|
|
// 400, not 404, and the string is "Bucket <name> does not exist". However, the
|
|
// documentation says it will be "Bucket id <name> does not exist". In case
|
|
// they update the implementation to match the documentation, we're just going
|
|
// to regexp over the error message and hope it's okay.
|
|
if bNotExist.MatchString(err.Error()) {
|
|
return b2err{
|
|
err: err,
|
|
notFoundErr: true,
|
|
}
|
|
}
|
|
return err
|
|
}
|
|
|
|
// BaseURL returns the base URL to use for all files uploaded to this bucket.
|
|
func (b *Bucket) BaseURL() string {
|
|
return b.b.baseURL()
|
|
}
|
|
|
|
// Name returns the bucket's name.
|
|
func (b *Bucket) Name() string {
|
|
return b.b.name()
|
|
}
|
|
|
|
// Object represents a B2 object.
|
|
type Object struct {
|
|
attrs *Attrs
|
|
name string
|
|
f beFileInterface
|
|
b *Bucket
|
|
}
|
|
|
|
// Attrs holds an object's metadata.
|
|
type Attrs struct {
|
|
Name string // Not used on upload.
|
|
Size int64 // Not used on upload.
|
|
ContentType string // Used on upload, default is "application/octet-stream".
|
|
Status ObjectState // Not used on upload.
|
|
UploadTimestamp time.Time // Not used on upload.
|
|
SHA1 string // Not used on upload. Can be "none" for large files.
|
|
LastModified time.Time // If present, and there are fewer than 10 keys in the Info field, this is saved on upload.
|
|
Info map[string]string // Save arbitrary metadata on upload, but limited to 10 keys.
|
|
}
|
|
|
|
// Name returns an object's name
|
|
func (o *Object) Name() string {
|
|
return o.name
|
|
}
|
|
|
|
// Attrs returns an object's attributes.
|
|
func (o *Object) Attrs(ctx context.Context) (*Attrs, error) {
|
|
if err := o.ensure(ctx); err != nil {
|
|
return nil, err
|
|
}
|
|
fi, err := o.f.getFileInfo(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
name, sha, size, ct, info, st, stamp := fi.stats()
|
|
var state ObjectState
|
|
switch st {
|
|
case "upload":
|
|
state = Uploaded
|
|
case "start":
|
|
state = Started
|
|
case "hide":
|
|
state = Hider
|
|
case "folder":
|
|
state = Folder
|
|
}
|
|
var mtime time.Time
|
|
if v, ok := info["src_last_modified_millis"]; ok {
|
|
ms, err := strconv.ParseInt(v, 10, 64)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
mtime = time.Unix(ms/1e3, (ms%1e3)*1e6)
|
|
delete(info, "src_last_modified_millis")
|
|
}
|
|
return &Attrs{
|
|
Name: name,
|
|
Size: size,
|
|
ContentType: ct,
|
|
UploadTimestamp: stamp,
|
|
SHA1: sha,
|
|
Info: info,
|
|
Status: state,
|
|
LastModified: mtime,
|
|
}, nil
|
|
}
|
|
|
|
// ObjectState represents the various states an object can be in.
|
|
type ObjectState int
|
|
|
|
const (
|
|
Unknown ObjectState = iota
|
|
// Started represents a large upload that has been started but not finished
|
|
// or canceled.
|
|
Started
|
|
// Uploaded represents an object that has finished uploading and is complete.
|
|
Uploaded
|
|
// Hider represents an object that exists only to hide another object. It
|
|
// cannot in itself be downloaded and, in particular, is not a hidden object.
|
|
Hider
|
|
|
|
// Folder is a special state given to non-objects that are returned during a
|
|
// List*Objects call with a non-empty Delimiter.
|
|
Folder
|
|
)
|
|
|
|
// Object returns a reference to the named object in the bucket. Hidden
|
|
// objects cannot be referenced in this manner; they can only be found by
|
|
// finding the appropriate reference in ListObjects.
|
|
func (b *Bucket) Object(name string) *Object {
|
|
return &Object{
|
|
name: name,
|
|
b: b,
|
|
}
|
|
}
|
|
|
|
// URL returns the full URL to the given object.
|
|
func (o *Object) URL() string {
|
|
return fmt.Sprintf("%s/file/%s/%s", o.b.BaseURL(), o.b.Name(), o.name)
|
|
}
|
|
|
|
// NewWriter returns a new writer for the given object. Objects that are
|
|
// overwritten are not deleted, but are "hidden".
|
|
//
|
|
// Callers must close the writer when finished and check the error status.
|
|
func (o *Object) NewWriter(ctx context.Context) *Writer {
|
|
ctx, cancel := context.WithCancel(ctx)
|
|
return &Writer{
|
|
o: o,
|
|
name: o.name,
|
|
ctx: ctx,
|
|
cancel: cancel,
|
|
}
|
|
}
|
|
|
|
// NewRangeReader returns a reader for the given object, reading up to length
|
|
// bytes. If length is negative, the rest of the object is read.
|
|
func (o *Object) NewRangeReader(ctx context.Context, offset, length int64) *Reader {
|
|
ctx, cancel := context.WithCancel(ctx)
|
|
return &Reader{
|
|
ctx: ctx,
|
|
cancel: cancel,
|
|
o: o,
|
|
name: o.name,
|
|
chunks: make(map[int]*rchunk),
|
|
length: length,
|
|
offset: offset,
|
|
}
|
|
}
|
|
|
|
// NewReader returns a reader for the given object.
|
|
func (o *Object) NewReader(ctx context.Context) *Reader {
|
|
return o.NewRangeReader(ctx, 0, -1)
|
|
}
|
|
|
|
func (o *Object) ensure(ctx context.Context) error {
|
|
if o.f == nil {
|
|
f, err := o.b.getObject(ctx, o.name)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
o.f = f.f
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Delete removes the given object.
|
|
func (o *Object) Delete(ctx context.Context) error {
|
|
if err := o.ensure(ctx); err != nil {
|
|
return err
|
|
}
|
|
return o.f.deleteFileVersion(ctx)
|
|
}
|
|
|
|
// Cursor is passed to ListObjects to return subsequent pages.
|
|
type Cursor struct {
|
|
// Prefix limits the listed objects to those that begin with this string.
|
|
Prefix string
|
|
|
|
// Delimiter denotes the path separator. If set, object listings will be
|
|
// truncated at this character.
|
|
//
|
|
// For example, if the bucket contains objects foo/bar, foo/baz, and foo,
|
|
// then a delimiter of "/" will cause the listing to return "foo" and "foo/".
|
|
// Otherwise, the listing would have returned all object names.
|
|
//
|
|
// Note that objects returned that end in the delimiter may not be actual
|
|
// objects, e.g. you cannot read from (or write to, or delete) an object "foo/",
|
|
// both because no actual object exists and because B2 disallows object names
|
|
// that end with "/". If you want to ensure that all objects returned by
|
|
// ListObjects and ListCurrentObjects are actual objects, leave this unset.
|
|
Delimiter string
|
|
|
|
name string
|
|
id string
|
|
}
|
|
|
|
// ListObjects returns all objects in the bucket, including multiple versions
|
|
// of the same object. Cursor may be nil; when passed to a subsequent query,
|
|
// it will continue the listing.
|
|
//
|
|
// ListObjects will return io.EOF when there are no objects left in the bucket,
|
|
// however it may do so concurrently with the last objects.
|
|
func (b *Bucket) ListObjects(ctx context.Context, count int, c *Cursor) ([]*Object, *Cursor, error) {
|
|
if c == nil {
|
|
c = &Cursor{}
|
|
}
|
|
fs, name, id, err := b.b.listFileVersions(ctx, count, c.name, c.id, c.Prefix, c.Delimiter)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
var next *Cursor
|
|
if name != "" && id != "" {
|
|
next = &Cursor{
|
|
Prefix: c.Prefix,
|
|
Delimiter: c.Delimiter,
|
|
name: name,
|
|
id: id,
|
|
}
|
|
}
|
|
var objects []*Object
|
|
for _, f := range fs {
|
|
objects = append(objects, &Object{
|
|
name: f.name(),
|
|
f: f,
|
|
b: b,
|
|
})
|
|
}
|
|
var rtnErr error
|
|
if len(objects) == 0 || next == nil {
|
|
rtnErr = io.EOF
|
|
}
|
|
return objects, next, rtnErr
|
|
}
|
|
|
|
// ListCurrentObjects is similar to ListObjects, except that it returns only
|
|
// current, unhidden objects in the bucket.
|
|
func (b *Bucket) ListCurrentObjects(ctx context.Context, count int, c *Cursor) ([]*Object, *Cursor, error) {
|
|
if c == nil {
|
|
c = &Cursor{}
|
|
}
|
|
fs, name, err := b.b.listFileNames(ctx, count, c.name, c.Prefix, c.Delimiter)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
var next *Cursor
|
|
if name != "" {
|
|
next = &Cursor{
|
|
Prefix: c.Prefix,
|
|
Delimiter: c.Delimiter,
|
|
name: name,
|
|
}
|
|
}
|
|
var objects []*Object
|
|
for _, f := range fs {
|
|
objects = append(objects, &Object{
|
|
name: f.name(),
|
|
f: f,
|
|
b: b,
|
|
})
|
|
}
|
|
var rtnErr error
|
|
if len(objects) == 0 || next == nil {
|
|
rtnErr = io.EOF
|
|
}
|
|
return objects, next, rtnErr
|
|
}
|
|
|
|
// Hide hides the object from name-based listing.
|
|
func (o *Object) Hide(ctx context.Context) error {
|
|
if err := o.ensure(ctx); err != nil {
|
|
return err
|
|
}
|
|
_, err := o.b.b.hideFile(ctx, o.name)
|
|
return err
|
|
}
|
|
|
|
// Reveal unhides (if hidden) the named object. If there are multiple objects
|
|
// of a given name, it will reveal the most recent.
|
|
func (b *Bucket) Reveal(ctx context.Context, name string) error {
|
|
cur := &Cursor{
|
|
name: name,
|
|
}
|
|
objs, _, err := b.ListObjects(ctx, 1, cur)
|
|
if err != nil && err != io.EOF {
|
|
return err
|
|
}
|
|
if len(objs) < 1 || objs[0].name != name {
|
|
return b2err{err: fmt.Errorf("%s: not found", name), notFoundErr: true}
|
|
}
|
|
obj := objs[0]
|
|
if obj.f.status() != "hide" {
|
|
return nil
|
|
}
|
|
return obj.Delete(ctx)
|
|
}
|
|
|
|
func (b *Bucket) getObject(ctx context.Context, name string) (*Object, error) {
|
|
fr, err := b.b.downloadFileByName(ctx, name, 0, 1)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
fr.Close()
|
|
return &Object{
|
|
name: name,
|
|
f: b.b.file(fr.id(), name),
|
|
b: b,
|
|
}, nil
|
|
}
|
|
|
|
// AuthToken returns an authorization token that can be used to access objects
|
|
// in a private bucket. Only objects that begin with prefix can be accessed.
|
|
// The token expires after the given duration.
|
|
func (b *Bucket) AuthToken(ctx context.Context, prefix string, valid time.Duration) (string, error) {
|
|
return b.b.getDownloadAuthorization(ctx, prefix, valid)
|
|
}
|