feat!: assume BuildKit is available

2023-10-27 04:54:23 -04:00 · 2023-10-27 04:54:23 -04:00 · c273e7f105
parent 7f47c3c8af
commit c273e7f105
9 changed files with 68 additions and 78 deletions
--- a/changelog.d/20230818_112124_kyle_buildkit.md
+++ b/changelog.d/20230818_112124_kyle_buildkit.md
@ -0,0 +1,2 @@
+- [Deprecation] The template variable ``is_buildkit_enabled``, which now always returns True, is deprecated. Plugin authors should assume BuildKit is enabled and remove the variable from their templates (by @kdmccormick).
+- 💥[Deprecation] Tutor no longer supports the legacy Docker builder, which was previously available by setting ``DOCKER_BUILDKIT=0`` in the host environment. Going forward, Tutor will always use BuildKit (a.k.a. ``docker buildx`` in Docker v19-v22, or just ``docker build`` in Docker v23). This transition will improve build performance and should be seamless for Tutor users who are running a supported Docker version (by @kdmccormick).
--- a/docs/tutor.rst
+++ b/docs/tutor.rst
@ -93,6 +93,13 @@ An optional BRANCH suffix may be appended to the release name to indicate that e

 `Officially-supported plugins <https://overhang.io/tutor/plugins>`__ follow the same versioning pattern. As a third-party plugin developer, you are encouraged to use the same pattern to make it immediately clear to your end-users which Open edX versions are supported.

+In Tutor and its officially-supported plugins, certain features, API endpoints, and older depenency versions are periodically deprecated. Generally, warnings are added to the Changelogs and/or the command-line interface one major release before support for any behavior is removed. In order to keep track of pending removals in the source code, comments containing the string ``REMOVE-AFTER-VXX`` should be used, where ``<XX>`` is the last major version that must support the behavior. For example::
+
+    # This has been replaced with SOME_NEW_HOOK (REMOVE-AFTER-V25).
+    SOME_OLD_HOOK = Filter()
+
+indicates that this filter definition can be removed as soon as Tutor v26.0.0.
+
 .. _contributing:

 Contributing to Tutor
--- a/tests/commands/test_images.py
+++ b/tests/commands/test_images.py
@ -128,10 +128,8 @@ class ImagesTests(PluginsTestCase, TestCommandMixin):
            "service1",
        ]
        with temporary_root() as root:
-            utils.is_buildkit_enabled.cache_clear()
-            with patch.object(utils, "is_buildkit_enabled", return_value=False):
-                self.invoke_in_root(root, ["config", "save"])
-                result = self.invoke_in_root(root, build_args)
+            self.invoke_in_root(root, ["config", "save"])
+            result = self.invoke_in_root(root, build_args)
        self.assertIsNone(result.exception)
        self.assertEqual(0, result.exit_code)
        image_build.assert_called()
@ -146,7 +144,9 @@ class ImagesTests(PluginsTestCase, TestCommandMixin):
                "host",
                "--target",
                "target",
+                "--output=type=docker",
                "docker_args",
+                "--cache-from=type=registry,ref=service1:1.0.0-cache",
            ],
            list(image_build.call_args[0][1:]),
        )
--- a/tutor/commands/images.py
+++ b/tutor/commands/images.py
@ -156,7 +156,7 @@ def images_command() -> None:
    # Export image to docker. This is necessary to make the image available to docker-compose.
    # The `--load` option is a shorthand for `--output=type=docker`.
    default="type=docker",
-    help="Same as `docker build --output=...`. This option will only be used when BuildKit is enabled.",
+    help="Same as `docker build --output=...`.",
 )
@click.option(
    "-a",
@ -211,7 +211,7 @@ def build(
        command_args += ["--add-host", add_host]
    if target:
        command_args += ["--target", target]
-    if utils.is_buildkit_enabled() and docker_output:
+    if docker_output:
        command_args.append(f"--output={docker_output}")
    if docker_args:
        command_args += docker_args
@ -223,27 +223,19 @@ def build(
            image_build_args = [*command_args, *custom_args]

            # Registry cache
-            if utils.is_buildkit_enabled():
-                if not no_registry_cache:
-                    image_build_args.append(
-                        f"--cache-from=type=registry,ref={tag}-cache"
-                    )
-                if cache_to_registry:
-                    image_build_args.append(
-                        f"--cache-to=type=registry,mode=max,ref={tag}-cache"
-                    )
+            if not no_registry_cache:
+                image_build_args.append(f"--cache-from=type=registry,ref={tag}-cache")
+            if cache_to_registry:
+                image_build_args.append(
+                    f"--cache-to=type=registry,mode=max,ref={tag}-cache"
+                )

            # Build contexts
            for host_path, stage_name in build_contexts.get(name, []):
-                if utils.is_buildkit_enabled():
-                    fmt.echo_info(
-                        f"Adding {host_path} to the build context '{stage_name}' of image '{image}'"
-                    )
-                    image_build_args.append(f"--build-context={stage_name}={host_path}")
-                else:
-                    fmt.echo_alert(
-                        f"Unable to add {host_path} to the build context '{stage_name}' of image '{host_path}' because BuildKit is disabled."
-                    )
+                fmt.echo_info(
+                    f"Adding {host_path} to the build context '{stage_name}' of image '{image}'"
+                )
+                image_build_args.append(f"--build-context={stage_name}={host_path}")

            # Build
            images.build(
--- a/tutor/env.py
+++ b/tutor/env.py
@ -54,7 +54,9 @@ def _prepare_environment() -> None:
            ("HOST_USER_ID", utils.get_user_id()),
            ("TUTOR_APP", __app__.replace("-", "_")),
            ("TUTOR_VERSION", __version__),
-            ("is_buildkit_enabled", utils.is_buildkit_enabled),
+            # BuildKit used to be optional. Now, it's always enabled.
+            # This constant is just for temporary backwards compatibility (REMOVE-AFTER-V16).
+            ("is_buildkit_enabled", lambda: True),
        ],
    )

--- a/tutor/hooks/catalog.py
+++ b/tutor/hooks/catalog.py
@ -251,8 +251,7 @@ class Filters:
    #:   names must be prefixed with the plugin name in all-caps.
    CONFIG_UNIQUE: Filter[list[tuple[str, Any]], []] = Filter()

-    #: Use this filter to modify the ``docker build`` command. For instance, to replace
-    #: the ``build`` subcommand by ``buildx build``.
+    #: Use this filter to modify the ``docker build`` command.
    #:
    #: :parameter list[str] command: the full build command, including options and
    #:   arguments. Note that these arguments do not include the leading ``docker`` command.
@ -335,7 +334,7 @@ class Filters:
    #: - ``HOST_USER_ID``: the numerical ID of the user on the host.
    #: - ``TUTOR_APP``: the app name ("tutor" by default), used to determine the dev/local project names.
    #: - ``TUTOR_VERSION``: the current version of Tutor.
-    #: - ``is_buildkit_enabled``: a boolean function that indicates whether BuildKit is available on the host.
+    #: - ``is_buildkit_enabled``: a deprecated function which always returns ``True`` now. Will be removed after Quince.
    #: - ``iter_values_named``: a function to iterate on variables that start or end with a given string.
    #: - ``iter_mounts``: a function that yields compose-compatible bind-mounts for any given service.
    #: - ``patch``: a function to incorporate extra content into a template.
--- a/tutor/images.py
+++ b/tutor/images.py
@ -4,8 +4,9 @@ from tutor import fmt, hooks, utils
 def build(path: str, tag: str, *args: str) -> None:
    fmt.echo_info(f"Building image {tag}")
    build_command = ["build", f"--tag={tag}", *args, path]
-    if utils.is_buildkit_enabled():
-        build_command.insert(0, "buildx")
+    # `buildx` can be removed once Tutor requires Docker v23+. At that point, BuildKit will be
+    # enabled by default for all Docker users.
+    build_command.insert(0, "buildx")
    command = hooks.Filters.DOCKER_BUILD_COMMAND.apply(build_command)
    utils.docker(*command)

--- a/tutor/templates/build/openedx/Dockerfile
+++ b/tutor/templates/build/openedx/Dockerfile
@ -1,11 +1,11 @@
-{% if is_buildkit_enabled() %}# syntax=docker/dockerfile:1.4{% endif %}
+# syntax=docker/dockerfile:1.4
 ###### Minimal image with base system requirements for most stages
 FROM docker.io/ubuntu:20.04 as minimal
 LABEL maintainer="Overhang.io <contact@overhang.io>"

 ENV DEBIAN_FRONTEND=noninteractive
-RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \
-    --mount=type=cache,target=/var/lib/apt,sharing=locked{% endif %} \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
    apt update && \
    apt install -y build-essential curl git language-pack-en
 ENV LC_ALL en_US.UTF-8
@ -14,8 +14,9 @@ ENV LC_ALL en_US.UTF-8
 ###### Install python with pyenv in /opt/pyenv and create virtualenv in /openedx/venv
 FROM minimal as python
 # https://github.com/pyenv/pyenv/wiki/Common-build-problems#prerequisites
-RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \
-    --mount=type=cache,target=/var/lib/apt,sharing=locked {% endif %}apt update && \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    apt update && \
    apt install -y libssl-dev zlib1g-dev libbz2-dev \
    libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \
    xz-utils tk-dev libffi-dev liblzma-dev python-openssl git
@ -77,12 +78,14 @@ ENV PATH /openedx/venv/bin:${PATH}
 ENV VIRTUAL_ENV /openedx/venv/
 ENV XDG_CACHE_HOME /openedx/.cache

-RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \
-    --mount=type=cache,target=/var/lib/apt,sharing=locked {% endif %}apt update \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    apt update \
    && apt install -y software-properties-common libmysqlclient-dev libxmlsec1-dev libgeos-dev

 # Install the right version of pip/setuptools
-RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install \
+RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \
+    pip install \
    # https://pypi.org/project/setuptools/
    # https://pypi.org/project/pip/
    # https://pypi.org/project/wheel/
@ -92,14 +95,13 @@ RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,
 RUN pip install https://github.com/overhangio/py2neo/releases/download/2021.2.3/py2neo-2021.2.3.tar.gz

 # Install base requirements
-{% if not is_buildkit_enabled() %}
-COPY --from=edx-platform /requirements/edx/base.txt /openedx/edx-platform/requirements/edx/base.txt
-{% endif %}
-RUN {% if is_buildkit_enabled() %}--mount=type=bind,from=edx-platform,source=/requirements/edx/base.txt,target=/openedx/edx-platform/requirements/edx/base.txt \
-    --mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install -r /openedx/edx-platform/requirements/edx/base.txt
+RUN --mount=type=bind,from=edx-platform,source=/requirements/edx/base.txt,target=/openedx/edx-platform/requirements/edx/base.txt \
+    --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \
+    pip install -r /openedx/edx-platform/requirements/edx/base.txt

 # Install extra requirements
-RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install \
+RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \
+    pip install \
    # Use redis as a django cache https://pypi.org/project/django-redis/
    django-redis==5.2.0 \
    # uwsgi server https://pypi.org/project/uWSGI/
@ -109,11 +111,14 @@ RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,

 # Install private requirements: this is useful for installing custom xblocks.
 COPY ./requirements/ /openedx/requirements
-RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}cd /openedx/requirements/ \
+RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \
+  cd /openedx/requirements/ \
  && touch ./private.txt \
  && pip install -r ./private.txt

-{% for extra_requirements in OPENEDX_EXTRA_PIP_REQUIREMENTS %}RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install '{{ extra_requirements }}'
+{% for extra_requirements in OPENEDX_EXTRA_PIP_REQUIREMENTS %}
+RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \
+    pip install '{{ extra_requirements }}'
 {% endfor %}

 ###### Install nodejs with nodeenv in /openedx/nodeenv
@ -129,21 +134,19 @@ RUN nodeenv /openedx/nodeenv --node=16.14.0 --prebuilt
 # Install nodejs requirements
 ARG NPM_REGISTRY={{ NPM_REGISTRY }}
 WORKDIR /openedx/edx-platform
-{% if not is_buildkit_enabled() %}
-COPY --from=edx-platform /package.json /openedx/edx-platform/package.json
-COPY --from=edx-platform /package-lock.json /openedx/edx-platform/package-lock.json
-{% endif %}
-RUN {% if is_buildkit_enabled() %}--mount=type=bind,from=edx-platform,source=/package.json,target=/openedx/edx-platform/package.json \
+RUN --mount=type=bind,from=edx-platform,source=/package.json,target=/openedx/edx-platform/package.json \
    --mount=type=bind,from=edx-platform,source=/package-lock.json,target=/openedx/edx-platform/package-lock.json \
    --mount=type=bind,from=edx-platform,source=/scripts/copy-node-modules.sh,target=/openedx/edx-platform/scripts/copy-node-modules.sh \
-    --mount=type=cache,target=/root/.npm,sharing=shared {% endif %}npm clean-install --no-audit --registry=$NPM_REGISTRY
+    --mount=type=cache,target=/root/.npm,sharing=shared \
+    npm clean-install --no-audit --registry=$NPM_REGISTRY

 ###### Production image with system and python requirements
 FROM minimal as production

 # Install system requirements
-RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \
-    --mount=type=cache,target=/var/lib/apt,sharing=locked {% endif %}apt update \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    apt update \
    && apt install -y gettext gfortran graphviz graphviz-dev libffi-dev libfreetype6-dev libgeos-dev libjpeg8-dev liblapack-dev libmysqlclient-dev libpng-dev libsqlite3-dev libxmlsec1-dev lynx mysql-client ntp pkg-config rdfind

 # From then on, run as unprivileged "app" user
@ -154,7 +157,7 @@ RUN useradd --no-log-init --home-dir /openedx --create-home --shell /bin/bash --
 USER ${APP_USER_ID}

 # https://hub.docker.com/r/powerman/dockerize/tags
-COPY {% if is_buildkit_enabled() %}--link {% endif %}--from=docker.io/powerman/dockerize:0.19.0 /usr/local/bin/dockerize /usr/local/bin/dockerize
+COPY --link --from=docker.io/powerman/dockerize:0.19.0 /usr/local/bin/dockerize /usr/local/bin/dockerize
 COPY --chown=app:app --from=edx-platform / /openedx/edx-platform
 COPY --chown=app:app --from=locales /openedx/locale /openedx/locale
 COPY --chown=app:app --from=python /opt/pyenv /opt/pyenv
@ -248,16 +251,19 @@ FROM production as development

 # Install useful system requirements (as root)
 USER root
-RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \
-    --mount=type=cache,target=/var/lib/apt,sharing=locked {% endif %}apt update && \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    apt update && \
    apt install -y vim iputils-ping dnsutils telnet
 USER app

 # Install dev python requirements
-RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install -r requirements/edx/development.txt
+RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \
+    pip install -r requirements/edx/development.txt
 # https://pypi.org/project/ipdb/
 # https://pypi.org/project/ipython
-RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install ipdb==0.13.13 ipython==8.12.0
+RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \
+    pip install ipdb==0.13.13 ipython==8.12.0

 # Add ipdb as default PYTHONBREAKPOINT
 ENV PYTHONBREAKPOINT=ipdb.set_trace
--- a/tutor/utils.py
+++ b/tutor/utils.py
@ -173,25 +173,6 @@ def docker(*command: str) -> int:
    return execute("docker", *command)


-@lru_cache(maxsize=None)
-def is_buildkit_enabled() -> bool:
-    """
-    A helper function to determine whether we can run `docker buildx` with BuildKit.
-    """
-    # First, we respect the DOCKER_BUILDKIT environment variable
-    enabled_by_env = {
-        "1": True,
-        "0": False,
-    }.get(os.environ.get("DOCKER_BUILDKIT", ""))
-    if enabled_by_env is not None:
-        return enabled_by_env
-    try:
-        subprocess.run(["docker", "buildx", "version"], capture_output=True, check=True)
-        return True
-    except subprocess.CalledProcessError:
-        return False
-
-
 def docker_compose(*command: str) -> int:
    return execute("docker", "compose", *command)