Merge pull request #4364 from MichaelEischer/multiplatform-docker

Setup multiplatform docker builds

.dockerignore

@@ -1,12 +0,0 @@
-# Folders
-.git/
-.github/
-changelog/
-doc/
-docker/
-helpers/
-
-# Files
-.gitignore
-.golangci.yml
-*.md

.github/workflows/docker.yml

@@ -0,0 +1,59 @@
+
+name: Create and publish a Docker image
+
+on:
+  push:
+    tags:
+      - 'v*'
+    branches:
+      - 'master'
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build-and-push-image:
+    if: github.repository == 'restic/restic'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=ref,event=branch
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+        with:
+          push: true
+          context: .
+          file: docker/Dockerfile.release
+          platforms: linux/386,linux/amd64,linux/arm,linux/arm64
+          pull: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/tests.yml

@@ -187,7 +187,7 @@ jobs:
         # own repo, otherwise the secrets are not available
         # Skip for Dependabot pull requests as these are run without secrets
         # https://docs.github.com/en/code-security/dependabot/working-with-dependabot/automating-dependabot-with-github-actions#responding-to-events
-        if: (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) && (github.actor != 'dependabot[bot]') && matrix.test_cloud_backends
+        if: ((github.repository == 'restic/restic' && github.event_name == 'push') || github.event.pull_request.head.repo.full_name == github.repository) && (github.actor != 'dependabot[bot]') && matrix.test_cloud_backends
 
       - name: Check changelog files with calens
         run: |

changelog/unreleased/issue-2359

@@ -0,0 +1,11 @@
+Enhancement: Provide multi-platform Docker containers
+
+The official Docker containers are now built for the architectures linux/386,
+linux/amd64, linux/arm and linux/arm64.
+
+As an alternative to the Docker Hub, the Docker containers are now also
+available on ghcr.io, the GitHub Container Registry.
+
+https://github.com/restic/restic/issues/2359
+https://github.com/restic/restic/issues/4269
+https://github.com/restic/restic/pull/4364

doc/020_installation.rst

@@ -265,6 +265,12 @@ binary, you can get it with `docker pull` like this:
 
     $ docker pull restic/restic
 
+The container is also available on the GitHub Container Registry:
+
+.. code-block:: console
+
+    $ docker pull ghcr.io/restic/restic
+
 Restic relies on the hostname for various operations. Make sure to set a static
 hostname using `--hostname` when creating a Docker container, otherwise Docker
 will assign a random hostname each time.

doc/developer_information.rst

@@ -127,3 +127,5 @@ required argument is the new version number (in `Semantic Versioning
     go run helpers/prepare-release/main.go 0.14.0
 
 Checks can be skipped on demand via flags, please see ``--help`` for details.
+
+The build process requires ``docker``, ``docker-buildx`` and ``qemu-user-static-binfmt``.

docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.19-alpine AS builder
+FROM golang:1.20-alpine AS builder
 
 WORKDIR /go/src/github.com/restic/restic
 

docker/Dockerfile.release

@@ -0,0 +1,18 @@
+# the official binaries are cross-built from Linux running on an AMD64 host
+# other architectures also seem to generate identical binaries but stay on the safe side
+FROM --platform=linux/amd64 restic/builder:latest as helper
+
+ARG TARGETOS
+ARG TARGETARCH
+
+COPY --chown=build . /restic
+RUN go run helpers/build-release-binaries/main.go --platform $TARGETOS/$TARGETARCH --skip-compress
+RUN mv /output/restic_${TARGETOS}_${TARGETARCH} /output/restic
+
+
+FROM alpine:latest
+
+COPY --from=helper /output/restic /usr/bin
+RUN apk add --update --no-cache ca-certificates fuse openssh-client tzdata jq
+
+ENTRYPOINT ["/usr/bin/restic"]

helpers/build-release-binaries/main.go

@@ -22,6 +22,8 @@ var opts = struct {
 	OutputDir      string
 	Tags           string
 	PlatformSubset string
+	Platform       string
+	SkipCompress   bool
 	Version        string
 }{}
 
@@ -31,6 +33,8 @@ func init() {
 	pflag.StringVarP(&opts.OutputDir, "output", "o", "/output", "path to the output `directory`")
 	pflag.StringVar(&opts.Tags, "tags", "", "additional build `tags`")
 	pflag.StringVar(&opts.PlatformSubset, "platform-subset", "", "specify `n/t` to only build this subset")
+	pflag.StringVarP(&opts.Platform, "platform", "p", "", "specify `os/arch` to only build this specific platform")
+	pflag.BoolVar(&opts.SkipCompress, "skip-compress", false, "skip binary compression step")
 	pflag.StringVar(&opts.Version, "version", "", "use `x.y.z` as the version for output files")
 	pflag.Parse()
 }
@@ -188,7 +192,9 @@ func buildForTarget(sourceDir, outputDir, goos, goarch string) (filename string)
 	filename = build(sourceDir, outputDir, goos, goarch)
 	touch(filepath.Join(outputDir, filename), mtime)
 	chmod(filepath.Join(outputDir, filename), 0755)
-	filename = compress(goos, outputDir, filename)
+	if !opts.SkipCompress {
+		filename = compress(goos, outputDir, filename)
+	}
 	return filename
 }
 
@@ -311,6 +317,8 @@ func main() {
 		if err != nil {
 			die("%s", err)
 		}
+	} else if opts.Platform != "" {
+		targets = buildPlatformList([]string{opts.Platform})
 	}
 
 	sourceDir := abs(opts.SourceDir)

helpers/prepare-release/main.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"bytes"
 	"fmt"
+	"math/rand"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -409,13 +410,19 @@ func signFiles(filenames ...string) {
 	}
 }
 
-func updateDocker(outputDir, version string) {
+func updateDocker(sourceDir, version string) string {
-	cmd := fmt.Sprintf("bzcat %s/restic_%s_linux_amd64.bz2 > restic", outputDir, version)
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
-	run("sh", "-c", cmd)
+	builderName := fmt.Sprintf("restic-release-builder-%d", r.Int())
-	run("chmod", "+x", "restic")
+	run("docker", "buildx", "create", "--name", builderName, "--driver", "docker-container", "--bootstrap")
-	run("docker", "pull", "alpine:latest")
+
-	run("docker", "build", "--rm", "--tag", "restic/restic:latest", "-f", "docker/Dockerfile", ".")
+	buildCmd := fmt.Sprintf("docker buildx build --builder %s --platform linux/386,linux/amd64,linux/arm,linux/arm64 --pull -f docker/Dockerfile.release %q", builderName, sourceDir)
-	run("docker", "tag", "restic/restic:latest", "restic/restic:"+version)
+	run("sh", "-c", buildCmd+" --no-cache")
+
+	publishCmds := ""
+	for _, tag := range []string{"restic/restic:latest", "restic/restic:" + version} {
+		publishCmds += buildCmd + fmt.Sprintf(" --tag %q --push\n", tag)
+	}
+	return publishCmds + "\ndocker buildx rm " + builderName
 }
 
 func tempdir(prefix string) string {
@@ -464,15 +471,14 @@ func main() {
 
 	extractTar(tarFilename, sourceDir)
 	runBuild(sourceDir, opts.OutputDir, opts.Version)
-	rmdir(sourceDir)
 
 	sha256sums(opts.OutputDir, filepath.Join(opts.OutputDir, "SHA256SUMS"))
 
 	signFiles(filepath.Join(opts.OutputDir, "SHA256SUMS"), tarFilename)
 
-	updateDocker(opts.OutputDir, opts.Version)
+	dockerCmds := updateDocker(sourceDir, opts.Version)
 
 	msg("done, output dir is %v", opts.OutputDir)
 
-	msg("now run:\n\ngit push --tags origin master\ndocker push restic/restic:latest\ndocker push restic/restic:%s\n", opts.Version)
+	msg("now run:\n\ngit push --tags origin master\n%s\n\nrm -rf %q", dockerCmds, sourceDir)
 }