From 2293835242df548251244e02fcbf921cc19db461 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 19 May 2023 17:16:29 +0200 Subject: [PATCH 1/9] Release multi-platform docker containers --- docker/Dockerfile.release | 20 ++++++++++++++++++++ helpers/prepare-release/main.go | 24 +++++++++++++++--------- 2 files changed, 35 insertions(+), 9 deletions(-) create mode 100644 docker/Dockerfile.release diff --git a/docker/Dockerfile.release b/docker/Dockerfile.release new file mode 100644 index 000000000..a2564b6d5 --- /dev/null +++ b/docker/Dockerfile.release @@ -0,0 +1,20 @@ +FROM --platform=$BUILDPLATFORM alpine:latest as helper + +ARG VERSION +ARG TARGETOS +ARG TARGETARCH + +# add release binary for the appropriate platform +COPY restic_${VERSION}_${TARGETOS}_${TARGETARCH}.bz2 / +RUN apk add --update --no-cache bzip2 +RUN set -e && \ + bzcat restic_${VERSION}_${TARGETOS}_${TARGETARCH}.bz2 > restic && \ + chmod +x restic + + +FROM alpine:latest + +COPY --from=helper /restic /usr/bin +RUN apk add --update --no-cache ca-certificates fuse openssh-client tzdata jq + +ENTRYPOINT ["/usr/bin/restic"] diff --git a/helpers/prepare-release/main.go b/helpers/prepare-release/main.go index 03924b0d9..2340a65d4 100644 --- a/helpers/prepare-release/main.go +++ b/helpers/prepare-release/main.go @@ -409,13 +409,19 @@ func signFiles(filenames ...string) { } } -func updateDocker(outputDir, version string) { - cmd := fmt.Sprintf("bzcat %s/restic_%s_linux_amd64.bz2 > restic", outputDir, version) - run("sh", "-c", cmd) - run("chmod", "+x", "restic") - run("docker", "pull", "alpine:latest") - run("docker", "build", "--rm", "--tag", "restic/restic:latest", "-f", "docker/Dockerfile", ".") - run("docker", "tag", "restic/restic:latest", "restic/restic:"+version) +func updateDocker(outputDir, version string) string { + run("docker", "buildx", "create", "--name", "restic-release-builder", "--driver", "docker-container", "--bootstrap") + + cmds := "" + + for _, tag := range []string{"restic/restic:latest", "restic/restic:" + version} { + cmd := fmt.Sprintf("docker buildx build --builder restic-release-builder --platform linux/386,linux/amd64,linux/arm,linux/arm64 --pull --tag %q -f docker/Dockerfile.release --build-arg VERSION=%q %q", tag, version, outputDir) + run("sh", "-c", cmd) + + cmds += cmd + " --push\n" + } + + return cmds + "\ndocker buildx rm restic-release-builder" } func tempdir(prefix string) string { @@ -470,9 +476,9 @@ func main() { signFiles(filepath.Join(opts.OutputDir, "SHA256SUMS"), tarFilename) - updateDocker(opts.OutputDir, opts.Version) + dockerCmds := updateDocker(opts.OutputDir, opts.Version) msg("done, output dir is %v", opts.OutputDir) - msg("now run:\n\ngit push --tags origin master\ndocker push restic/restic:latest\ndocker push restic/restic:%s\n", opts.Version) + msg("now run:\n\ngit push --tags origin master\n%s\n", dockerCmds) } From 43fa0515462f626084859acf6300ce8c3a0653f7 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 19 May 2023 17:43:11 +0200 Subject: [PATCH 2/9] Directly build restic binary in release Docker container --- .dockerignore | 1 - docker/Dockerfile.release | 16 +++++++--------- helpers/build-release-binaries/main.go | 10 +++++++++- helpers/prepare-release/main.go | 24 ++++++++++++------------ 4 files changed, 28 insertions(+), 23 deletions(-) diff --git a/.dockerignore b/.dockerignore index 2e1b785e0..b7f28c69f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,7 +4,6 @@ changelog/ doc/ docker/ -helpers/ # Files .gitignore diff --git a/docker/Dockerfile.release b/docker/Dockerfile.release index a2564b6d5..01f9df150 100644 --- a/docker/Dockerfile.release +++ b/docker/Dockerfile.release @@ -1,20 +1,18 @@ -FROM --platform=$BUILDPLATFORM alpine:latest as helper +# the official binaries are cross-built from Linux running on an AMD64 host +# other architectures also seem to generate identical binaries but stay on the safe side +FROM --platform=linux/amd64 restic/builder:latest as helper -ARG VERSION ARG TARGETOS ARG TARGETARCH -# add release binary for the appropriate platform -COPY restic_${VERSION}_${TARGETOS}_${TARGETARCH}.bz2 / -RUN apk add --update --no-cache bzip2 -RUN set -e && \ - bzcat restic_${VERSION}_${TARGETOS}_${TARGETARCH}.bz2 > restic && \ - chmod +x restic +COPY . /restic +RUN go run helpers/build-release-binaries/main.go --platform $TARGETOS/$TARGETARCH --skip-compress +RUN mv /output/restic_${TARGETOS}_${TARGETARCH} /output/restic FROM alpine:latest -COPY --from=helper /restic /usr/bin +COPY --from=helper /output/restic /usr/bin RUN apk add --update --no-cache ca-certificates fuse openssh-client tzdata jq ENTRYPOINT ["/usr/bin/restic"] diff --git a/helpers/build-release-binaries/main.go b/helpers/build-release-binaries/main.go index 6938aff84..caa90ff82 100644 --- a/helpers/build-release-binaries/main.go +++ b/helpers/build-release-binaries/main.go @@ -22,6 +22,8 @@ var opts = struct { OutputDir string Tags string PlatformSubset string + Platform string + SkipCompress bool Version string }{} @@ -31,6 +33,8 @@ func init() { pflag.StringVarP(&opts.OutputDir, "output", "o", "/output", "path to the output `directory`") pflag.StringVar(&opts.Tags, "tags", "", "additional build `tags`") pflag.StringVar(&opts.PlatformSubset, "platform-subset", "", "specify `n/t` to only build this subset") + pflag.StringVarP(&opts.Platform, "platform", "p", "", "specify `os/arch` to only build this specific platform") + pflag.BoolVar(&opts.SkipCompress, "skip-compress", false, "skip binary compression step") pflag.StringVar(&opts.Version, "version", "", "use `x.y.z` as the version for output files") pflag.Parse() } @@ -188,7 +192,9 @@ func buildForTarget(sourceDir, outputDir, goos, goarch string) (filename string) filename = build(sourceDir, outputDir, goos, goarch) touch(filepath.Join(outputDir, filename), mtime) chmod(filepath.Join(outputDir, filename), 0755) - filename = compress(goos, outputDir, filename) + if !opts.SkipCompress { + filename = compress(goos, outputDir, filename) + } return filename } @@ -311,6 +317,8 @@ func main() { if err != nil { die("%s", err) } + } else if opts.Platform != "" { + targets = buildPlatformList([]string{opts.Platform}) } sourceDir := abs(opts.SourceDir) diff --git a/helpers/prepare-release/main.go b/helpers/prepare-release/main.go index 2340a65d4..a6c7bd4f4 100644 --- a/helpers/prepare-release/main.go +++ b/helpers/prepare-release/main.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "fmt" + "math/rand" "os" "os/exec" "path/filepath" @@ -409,19 +410,19 @@ func signFiles(filenames ...string) { } } -func updateDocker(outputDir, version string) string { - run("docker", "buildx", "create", "--name", "restic-release-builder", "--driver", "docker-container", "--bootstrap") +func updateDocker(sourceDir, version string) string { + r := rand.New(rand.NewSource(time.Now().UnixNano())) + builderName := fmt.Sprintf("restic-release-builder-%d", r.Int()) + run("docker", "buildx", "create", "--name", builderName, "--driver", "docker-container", "--bootstrap") - cmds := "" + buildCmd := fmt.Sprintf("docker buildx build --builder %s --platform linux/386,linux/amd64,linux/arm,linux/arm64 --pull -f docker/Dockerfile.release %q", builderName, sourceDir) + run("sh", "-c", buildCmd+" --no-cache") + publishCmds := "" for _, tag := range []string{"restic/restic:latest", "restic/restic:" + version} { - cmd := fmt.Sprintf("docker buildx build --builder restic-release-builder --platform linux/386,linux/amd64,linux/arm,linux/arm64 --pull --tag %q -f docker/Dockerfile.release --build-arg VERSION=%q %q", tag, version, outputDir) - run("sh", "-c", cmd) - - cmds += cmd + " --push\n" + publishCmds += buildCmd + fmt.Sprintf(" --tag %q --push\n", tag) } - - return cmds + "\ndocker buildx rm restic-release-builder" + return publishCmds + "\ndocker buildx rm " + builderName } func tempdir(prefix string) string { @@ -470,15 +471,14 @@ func main() { extractTar(tarFilename, sourceDir) runBuild(sourceDir, opts.OutputDir, opts.Version) - rmdir(sourceDir) sha256sums(opts.OutputDir, filepath.Join(opts.OutputDir, "SHA256SUMS")) signFiles(filepath.Join(opts.OutputDir, "SHA256SUMS"), tarFilename) - dockerCmds := updateDocker(opts.OutputDir, opts.Version) + dockerCmds := updateDocker(sourceDir, opts.Version) msg("done, output dir is %v", opts.OutputDir) - msg("now run:\n\ngit push --tags origin master\n%s\n", dockerCmds) + msg("now run:\n\ngit push --tags origin master\n%s\n\nrm -rf %q", dockerCmds, sourceDir) } From eff3124f1599310bb41718208b761af06ee4209c Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 20 May 2023 11:10:00 +0200 Subject: [PATCH 3/9] CI: Setup automatic container builds for ghcr.io Containers are built for new tags and pushes to the master branch. --- .github/workflows/docker.yml | 58 ++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 .github/workflows/docker.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 000000000..9fe5b3a30 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,58 @@ + +name: Create and publish a Docker image + +on: + push: + tags: + - 'v*' + branches: + - 'master' + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + + - name: Set up QEMU + uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c + + - name: Build and push Docker image + uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 + with: + push: true + context: . + file: docker/Dockerfile.release + platforms: linux/386,linux/amd64,linux/arm,linux/arm64 + pull: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} From b0987ff570317cee25b1f2cc869fcdcf1d379490 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 20 May 2023 12:12:50 +0200 Subject: [PATCH 4/9] CI: only build containers on restic/restic --- .github/workflows/docker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 9fe5b3a30..43c427109 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -14,6 +14,7 @@ env: jobs: build-and-push-image: + if: github.repository == 'restic/restic' runs-on: ubuntu-latest permissions: contents: read From ccd19b7e88e9c3bf2bb5889e46db6135d393ee05 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 20 May 2023 12:23:45 +0200 Subject: [PATCH 5/9] CI: run cloud backend tests only on restic/restic --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9bca02f8d..8e8a5b099 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -187,7 +187,7 @@ jobs: # own repo, otherwise the secrets are not available # Skip for Dependabot pull requests as these are run without secrets # https://docs.github.com/en/code-security/dependabot/working-with-dependabot/automating-dependabot-with-github-actions#responding-to-events - if: (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) && (github.actor != 'dependabot[bot]') && matrix.test_cloud_backends + if: ((github.repository == 'restic/restic' && github.event_name == 'push') || github.event.pull_request.head.repo.full_name == github.repository) && (github.actor != 'dependabot[bot]') && matrix.test_cloud_backends - name: Check changelog files with calens run: | From c181b51360de7e0754663bb94cc3b9f54080a5e4 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 9 Jun 2023 12:40:25 +0200 Subject: [PATCH 6/9] doc: Update docker build process --- doc/020_installation.rst | 6 ++++++ doc/developer_information.rst | 2 ++ 2 files changed, 8 insertions(+) diff --git a/doc/020_installation.rst b/doc/020_installation.rst index b53c350b1..a39ae91e9 100644 --- a/doc/020_installation.rst +++ b/doc/020_installation.rst @@ -265,6 +265,12 @@ binary, you can get it with `docker pull` like this: $ docker pull restic/restic +The container is also available on the GitHub Container Registry: + +.. code-block:: console + + $ docker pull ghcr.io/restic/restic + Restic relies on the hostname for various operations. Make sure to set a static hostname using `--hostname` when creating a Docker container, otherwise Docker will assign a random hostname each time. diff --git a/doc/developer_information.rst b/doc/developer_information.rst index 307851757..9de517901 100644 --- a/doc/developer_information.rst +++ b/doc/developer_information.rst @@ -127,3 +127,5 @@ required argument is the new version number (in `Semantic Versioning go run helpers/prepare-release/main.go 0.14.0 Checks can be skipped on demand via flags, please see ``--help`` for details. + +The build process requires ``docker``, ``docker-buildx`` and ``qemu-user-static-binfmt``. From 31e07cecbb75370aa055bd5a81cda7945a49222c Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 9 Jun 2023 12:40:49 +0200 Subject: [PATCH 7/9] docker: update to Go 1.20 for custom container builds --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 72fc85093..ecc283f8a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.19-alpine AS builder +FROM golang:1.20-alpine AS builder WORKDIR /go/src/github.com/restic/restic From 6b82cce1bd2c5d2c20566749a7229a538552731f Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 9 Jun 2023 12:46:29 +0200 Subject: [PATCH 8/9] add changelog for multiplatform containers --- changelog/unreleased/issue-2359 | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 changelog/unreleased/issue-2359 diff --git a/changelog/unreleased/issue-2359 b/changelog/unreleased/issue-2359 new file mode 100644 index 000000000..0399a96f1 --- /dev/null +++ b/changelog/unreleased/issue-2359 @@ -0,0 +1,11 @@ +Enhancement: Provide multi-platform Docker containers + +The official Docker containers are now built for the architectures linux/386, +linux/amd64, linux/arm and linux/arm64. + +As an alternative to the Docker Hub, the Docker containers are now also +available on ghcr.io, the GitHub Container Registry. + +https://github.com/restic/restic/issues/2359 +https://github.com/restic/restic/issues/4269 +https://github.com/restic/restic/pull/4364 From 3a93e28605ae6b64f0e854fd4601ac814517cb94 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 17 Jun 2023 16:22:50 +0200 Subject: [PATCH 9/9] CI: Remove .dockerignore to ensure reproducible builds Since go 1.18, built binaries also include VCS information such as the built commit. This information is also included in the official binaries. To ensure that the Docker container recreates the same binaries, the .git folder must also be transferred into the container. Thus, remove the .dockerignore file. The copied files must also be owned by the current user within the container, as git refuses to work otherwise. --- .dockerignore | 11 ----------- docker/Dockerfile.release | 2 +- 2 files changed, 1 insertion(+), 12 deletions(-) delete mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index b7f28c69f..000000000 --- a/.dockerignore +++ /dev/null @@ -1,11 +0,0 @@ -# Folders -.git/ -.github/ -changelog/ -doc/ -docker/ - -# Files -.gitignore -.golangci.yml -*.md diff --git a/docker/Dockerfile.release b/docker/Dockerfile.release index 01f9df150..ccf80376a 100644 --- a/docker/Dockerfile.release +++ b/docker/Dockerfile.release @@ -5,7 +5,7 @@ FROM --platform=linux/amd64 restic/builder:latest as helper ARG TARGETOS ARG TARGETARCH -COPY . /restic +COPY --chown=build . /restic RUN go run helpers/build-release-binaries/main.go --platform $TARGETOS/$TARGETARCH --skip-compress RUN mv /output/restic_${TARGETOS}_${TARGETARCH} /output/restic