From 8ca58b487cd7c6f4df04b2dd874b9eb6becf4e4e Mon Sep 17 00:00:00 2001 From: Vladislav Belous Date: Tue, 5 Dec 2023 00:20:27 +0200 Subject: [PATCH 1/2] S3: do not set storage class for metadata when using archive storage --- changelog/unreleased/issue-4583 | 9 +++++++++ internal/backend/s3/s3.go | 14 ++++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 changelog/unreleased/issue-4583 diff --git a/changelog/unreleased/issue-4583 b/changelog/unreleased/issue-4583 new file mode 100644 index 000000000..6af8ab475 --- /dev/null +++ b/changelog/unreleased/issue-4583 @@ -0,0 +1,9 @@ +Bugfix: Ignoring the s3.storage-class option for metadata when archive tier is specified + +Restic now will save snapshot metadata to non-archive storage tier whatsoever, +this will help avoid issues when data is being saved to archive storage class. +It is not providing any support for cold storages in restic, +only saving users from making backups unusable. + +https://github.com/restic/restic/issues/4583 +https://github.com/restic/restic/issues/3202 \ No newline at end of file diff --git a/internal/backend/s3/s3.go b/internal/backend/s3/s3.go index f0447224f..f9947b3a4 100644 --- a/internal/backend/s3/s3.go +++ b/internal/backend/s3/s3.go @@ -325,12 +325,22 @@ func (be *Backend) Path() string { return be.cfg.Prefix } +// useStorageClass returns whether file should be saved in the provided Storage Class +func (be *Backend) useStorageClass(h backend.Handle) bool { + var notArchiveClass bool = be.cfg.StorageClass != "GLACIER" && be.cfg.StorageClass != "DEEP_ARCHIVE" + isDataFile := h.Type == backend.PackFile && !h.IsMetadata + return isDataFile || notArchiveClass +} + // Save stores data in the backend at the handle. func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { objName := be.Filename(h) - opts := minio.PutObjectOptions{StorageClass: be.cfg.StorageClass} - opts.ContentType = "application/octet-stream" + opts := minio.PutObjectOptions{ContentType: "application/octet-stream"} + + if be.useStorageClass(h) { + opts.StorageClass = be.cfg.StorageClass + } // the only option with the high-level api is to let the library handle the checksum computation opts.SendContentMd5 = true // only use multipart uploads for very large files From a763a5c67de69b45eb1d56acaf4f639bae9fb891 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 20 Jan 2024 11:18:09 +0100 Subject: [PATCH 2/2] s3: minor cleanups for archive storage class handling --- changelog/unreleased/issue-4583 | 15 +++++++++------ internal/backend/s3/s3.go | 17 ++++++++++------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/changelog/unreleased/issue-4583 b/changelog/unreleased/issue-4583 index 6af8ab475..97b0e6ba7 100644 --- a/changelog/unreleased/issue-4583 +++ b/changelog/unreleased/issue-4583 @@ -1,9 +1,12 @@ -Bugfix: Ignoring the s3.storage-class option for metadata when archive tier is specified +Enhancement: Ignore s3.storage-class for metadata if archive tier is specified -Restic now will save snapshot metadata to non-archive storage tier whatsoever, -this will help avoid issues when data is being saved to archive storage class. -It is not providing any support for cold storages in restic, -only saving users from making backups unusable. +There is no official cold storage support in restic, use this option at your +own risk. + +Restic always stored all files on s3 using the specified `s3.storage-class`. +Now, restic will store metadata using a non-archive storage tier to avoid +problems when accessing a repository. To restore any data, it is still +necessary to manually warm up the required data beforehand. https://github.com/restic/restic/issues/4583 -https://github.com/restic/restic/issues/3202 \ No newline at end of file +https://github.com/restic/restic/pull/4584 diff --git a/internal/backend/s3/s3.go b/internal/backend/s3/s3.go index f9947b3a4..d41f4479d 100644 --- a/internal/backend/s3/s3.go +++ b/internal/backend/s3/s3.go @@ -326,8 +326,10 @@ func (be *Backend) Path() string { } // useStorageClass returns whether file should be saved in the provided Storage Class +// For archive storage classes, only data files are stored using that class; metadata +// must remain instantly accessible. func (be *Backend) useStorageClass(h backend.Handle) bool { - var notArchiveClass bool = be.cfg.StorageClass != "GLACIER" && be.cfg.StorageClass != "DEEP_ARCHIVE" + notArchiveClass := be.cfg.StorageClass != "GLACIER" && be.cfg.StorageClass != "DEEP_ARCHIVE" isDataFile := h.Type == backend.PackFile && !h.IsMetadata return isDataFile || notArchiveClass } @@ -336,15 +338,16 @@ func (be *Backend) useStorageClass(h backend.Handle) bool { func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { objName := be.Filename(h) - opts := minio.PutObjectOptions{ContentType: "application/octet-stream"} - + opts := minio.PutObjectOptions{ + ContentType: "application/octet-stream", + // the only option with the high-level api is to let the library handle the checksum computation + SendContentMd5: true, + // only use multipart uploads for very large files + PartSize: 200 * 1024 * 1024, + } if be.useStorageClass(h) { opts.StorageClass = be.cfg.StorageClass } - // the only option with the high-level api is to let the library handle the checksum computation - opts.SendContentMd5 = true - // only use multipart uploads for very large files - opts.PartSize = 200 * 1024 * 1024 info, err := be.client.PutObject(ctx, be.cfg.Bucket, objName, io.NopCloser(rd), int64(rd.Length()), opts)