feat!: assume BuildKit is available

changelog.d/20230818_112124_kyle_buildkit.md

@@ -0,0 +1,2 @@
+- [Deprecation] The template variable ``is_buildkit_enabled``, which now always returns True, is deprecated. Plugin authors should assume BuildKit is enabled and remove the variable from their templates (by @kdmccormick).
+- 💥[Deprecation] Tutor no longer supports the legacy Docker builder, which was previously available by setting ``DOCKER_BUILDKIT=0`` in the host environment. Going forward, Tutor will always use BuildKit (a.k.a. ``docker buildx`` in Docker v19-v22, or just ``docker build`` in Docker v23). This transition will improve build performance and should be seamless for Tutor users who are running a supported Docker version (by @kdmccormick).

docs/tutor.rst

@@ -93,6 +93,13 @@ An optional BRANCH suffix may be appended to the release name to indicate that e
 
 `Officially-supported plugins <https://overhang.io/tutor/plugins>`__ follow the same versioning pattern. As a third-party plugin developer, you are encouraged to use the same pattern to make it immediately clear to your end-users which Open edX versions are supported.
 
+In Tutor and its officially-supported plugins, certain features, API endpoints, and older depenency versions are periodically deprecated. Generally, warnings are added to the Changelogs and/or the command-line interface one major release before support for any behavior is removed. In order to keep track of pending removals in the source code, comments containing the string ``REMOVE-AFTER-VXX`` should be used, where ``<XX>`` is the last major version that must support the behavior. For example::
+
+    # This has been replaced with SOME_NEW_HOOK (REMOVE-AFTER-V25).
+    SOME_OLD_HOOK = Filter()
+
+indicates that this filter definition can be removed as soon as Tutor v26.0.0.
+
 .. _contributing:
 
 Contributing to Tutor

tests/commands/test_images.py

@@ -128,10 +128,8 @@ class ImagesTests(PluginsTestCase, TestCommandMixin):
             "service1",
         ]
         with temporary_root() as root:
-            utils.is_buildkit_enabled.cache_clear()
+            self.invoke_in_root(root, ["config", "save"])
-            with patch.object(utils, "is_buildkit_enabled", return_value=False):
+            result = self.invoke_in_root(root, build_args)
-                self.invoke_in_root(root, ["config", "save"])
-                result = self.invoke_in_root(root, build_args)
         self.assertIsNone(result.exception)
         self.assertEqual(0, result.exit_code)
         image_build.assert_called()
@@ -146,7 +144,9 @@ class ImagesTests(PluginsTestCase, TestCommandMixin):
                 "host",
                 "--target",
                 "target",
+                "--output=type=docker",
                 "docker_args",
+                "--cache-from=type=registry,ref=service1:1.0.0-cache",
             ],
             list(image_build.call_args[0][1:]),
         )

tutor/commands/images.py

@@ -156,7 +156,7 @@ def images_command() -> None:
     # Export image to docker. This is necessary to make the image available to docker-compose.
     # The `--load` option is a shorthand for `--output=type=docker`.
     default="type=docker",
-    help="Same as `docker build --output=...`. This option will only be used when BuildKit is enabled.",
+    help="Same as `docker build --output=...`.",
 )
 @click.option(
     "-a",
@@ -211,7 +211,7 @@ def build(
         command_args += ["--add-host", add_host]
     if target:
         command_args += ["--target", target]
-    if utils.is_buildkit_enabled() and docker_output:
+    if docker_output:
         command_args.append(f"--output={docker_output}")
     if docker_args:
         command_args += docker_args
@@ -223,27 +223,19 @@ def build(
             image_build_args = [*command_args, *custom_args]
 
             # Registry cache
-            if utils.is_buildkit_enabled():
+            if not no_registry_cache:
-                if not no_registry_cache:
+                image_build_args.append(f"--cache-from=type=registry,ref={tag}-cache")
-                    image_build_args.append(
+            if cache_to_registry:
-                        f"--cache-from=type=registry,ref={tag}-cache"
+                image_build_args.append(
-                    )
+                    f"--cache-to=type=registry,mode=max,ref={tag}-cache"
-                if cache_to_registry:
+                )
-                    image_build_args.append(
-                        f"--cache-to=type=registry,mode=max,ref={tag}-cache"
-                    )
 
             # Build contexts
             for host_path, stage_name in build_contexts.get(name, []):
-                if utils.is_buildkit_enabled():
+                fmt.echo_info(
-                    fmt.echo_info(
+                    f"Adding {host_path} to the build context '{stage_name}' of image '{image}'"
-                        f"Adding {host_path} to the build context '{stage_name}' of image '{image}'"
+                )
-                    )
+                image_build_args.append(f"--build-context={stage_name}={host_path}")
-                    image_build_args.append(f"--build-context={stage_name}={host_path}")
-                else:
-                    fmt.echo_alert(
-                        f"Unable to add {host_path} to the build context '{stage_name}' of image '{host_path}' because BuildKit is disabled."
-                    )
 
             # Build
             images.build(

tutor/env.py

@@ -54,7 +54,9 @@ def _prepare_environment() -> None:
             ("HOST_USER_ID", utils.get_user_id()),
             ("TUTOR_APP", __app__.replace("-", "_")),
             ("TUTOR_VERSION", __version__),
-            ("is_buildkit_enabled", utils.is_buildkit_enabled),
+            # BuildKit used to be optional. Now, it's always enabled.
+            # This constant is just for temporary backwards compatibility (REMOVE-AFTER-V16).
+            ("is_buildkit_enabled", lambda: True),
         ],
     )
 

tutor/hooks/catalog.py

@@ -251,8 +251,7 @@ class Filters:
     #:   names must be prefixed with the plugin name in all-caps.
     CONFIG_UNIQUE: Filter[list[tuple[str, Any]], []] = Filter()
 
-    #: Use this filter to modify the ``docker build`` command. For instance, to replace
+    #: Use this filter to modify the ``docker build`` command.
-    #: the ``build`` subcommand by ``buildx build``.
     #:
     #: :parameter list[str] command: the full build command, including options and
     #:   arguments. Note that these arguments do not include the leading ``docker`` command.
@@ -335,7 +334,7 @@ class Filters:
     #: - ``HOST_USER_ID``: the numerical ID of the user on the host.
     #: - ``TUTOR_APP``: the app name ("tutor" by default), used to determine the dev/local project names.
     #: - ``TUTOR_VERSION``: the current version of Tutor.
-    #: - ``is_buildkit_enabled``: a boolean function that indicates whether BuildKit is available on the host.
+    #: - ``is_buildkit_enabled``: a deprecated function which always returns ``True`` now. Will be removed after Quince.
     #: - ``iter_values_named``: a function to iterate on variables that start or end with a given string.
     #: - ``iter_mounts``: a function that yields compose-compatible bind-mounts for any given service.
     #: - ``patch``: a function to incorporate extra content into a template.

tutor/images.py

@@ -4,8 +4,9 @@ from tutor import fmt, hooks, utils
 def build(path: str, tag: str, *args: str) -> None:
     fmt.echo_info(f"Building image {tag}")
     build_command = ["build", f"--tag={tag}", *args, path]
-    if utils.is_buildkit_enabled():
+    # `buildx` can be removed once Tutor requires Docker v23+. At that point, BuildKit will be
-        build_command.insert(0, "buildx")
+    # enabled by default for all Docker users.
+    build_command.insert(0, "buildx")
     command = hooks.Filters.DOCKER_BUILD_COMMAND.apply(build_command)
     utils.docker(*command)
 

tutor/templates/build/openedx/Dockerfile

@@ -1,11 +1,11 @@
-{% if is_buildkit_enabled() %}# syntax=docker/dockerfile:1.4{% endif %}
+# syntax=docker/dockerfile:1.4
 ###### Minimal image with base system requirements for most stages
 FROM docker.io/ubuntu:20.04 as minimal
 LABEL maintainer="Overhang.io <contact@overhang.io>"
 
 ENV DEBIAN_FRONTEND=noninteractive
-RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
-    --mount=type=cache,target=/var/lib/apt,sharing=locked{% endif %} \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
     apt update && \
     apt install -y build-essential curl git language-pack-en
 ENV LC_ALL en_US.UTF-8
@@ -14,8 +14,9 @@ ENV LC_ALL en_US.UTF-8
 ###### Install python with pyenv in /opt/pyenv and create virtualenv in /openedx/venv
 FROM minimal as python
 # https://github.com/pyenv/pyenv/wiki/Common-build-problems#prerequisites
-RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
-    --mount=type=cache,target=/var/lib/apt,sharing=locked {% endif %}apt update && \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    apt update && \
     apt install -y libssl-dev zlib1g-dev libbz2-dev \
     libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \
     xz-utils tk-dev libffi-dev liblzma-dev python-openssl git
@@ -77,12 +78,14 @@ ENV PATH /openedx/venv/bin:${PATH}
 ENV VIRTUAL_ENV /openedx/venv/
 ENV XDG_CACHE_HOME /openedx/.cache
 
-RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
-    --mount=type=cache,target=/var/lib/apt,sharing=locked {% endif %}apt update \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    apt update \
     && apt install -y software-properties-common libmysqlclient-dev libxmlsec1-dev libgeos-dev
 
 # Install the right version of pip/setuptools
-RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install \
+RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \
+    pip install \
     # https://pypi.org/project/setuptools/
     # https://pypi.org/project/pip/
     # https://pypi.org/project/wheel/
@@ -92,14 +95,13 @@ RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,
 RUN pip install https://github.com/overhangio/py2neo/releases/download/2021.2.3/py2neo-2021.2.3.tar.gz
 
 # Install base requirements
-{% if not is_buildkit_enabled() %}
+RUN --mount=type=bind,from=edx-platform,source=/requirements/edx/base.txt,target=/openedx/edx-platform/requirements/edx/base.txt \
-COPY --from=edx-platform /requirements/edx/base.txt /openedx/edx-platform/requirements/edx/base.txt
+    --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \
-{% endif %}
+    pip install -r /openedx/edx-platform/requirements/edx/base.txt
-RUN {% if is_buildkit_enabled() %}--mount=type=bind,from=edx-platform,source=/requirements/edx/base.txt,target=/openedx/edx-platform/requirements/edx/base.txt \
-    --mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install -r /openedx/edx-platform/requirements/edx/base.txt
 
 # Install extra requirements
-RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install \
+RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \
+    pip install \
     # Use redis as a django cache https://pypi.org/project/django-redis/
     django-redis==5.2.0 \
     # uwsgi server https://pypi.org/project/uWSGI/
@@ -109,11 +111,14 @@ RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,
 
 # Install private requirements: this is useful for installing custom xblocks.
 COPY ./requirements/ /openedx/requirements
-RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}cd /openedx/requirements/ \
+RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \
+  cd /openedx/requirements/ \
   && touch ./private.txt \
   && pip install -r ./private.txt
 
-{% for extra_requirements in OPENEDX_EXTRA_PIP_REQUIREMENTS %}RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install '{{ extra_requirements }}'
+{% for extra_requirements in OPENEDX_EXTRA_PIP_REQUIREMENTS %}
+RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \
+    pip install '{{ extra_requirements }}'
 {% endfor %}
 
 ###### Install nodejs with nodeenv in /openedx/nodeenv
@@ -129,21 +134,19 @@ RUN nodeenv /openedx/nodeenv --node=16.14.0 --prebuilt
 # Install nodejs requirements
 ARG NPM_REGISTRY={{ NPM_REGISTRY }}
 WORKDIR /openedx/edx-platform
-{% if not is_buildkit_enabled() %}
+RUN --mount=type=bind,from=edx-platform,source=/package.json,target=/openedx/edx-platform/package.json \
-COPY --from=edx-platform /package.json /openedx/edx-platform/package.json
-COPY --from=edx-platform /package-lock.json /openedx/edx-platform/package-lock.json
-{% endif %}
-RUN {% if is_buildkit_enabled() %}--mount=type=bind,from=edx-platform,source=/package.json,target=/openedx/edx-platform/package.json \
     --mount=type=bind,from=edx-platform,source=/package-lock.json,target=/openedx/edx-platform/package-lock.json \
     --mount=type=bind,from=edx-platform,source=/scripts/copy-node-modules.sh,target=/openedx/edx-platform/scripts/copy-node-modules.sh \
-    --mount=type=cache,target=/root/.npm,sharing=shared {% endif %}npm clean-install --no-audit --registry=$NPM_REGISTRY
+    --mount=type=cache,target=/root/.npm,sharing=shared \
+    npm clean-install --no-audit --registry=$NPM_REGISTRY
 
 ###### Production image with system and python requirements
 FROM minimal as production
 
 # Install system requirements
-RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
-    --mount=type=cache,target=/var/lib/apt,sharing=locked {% endif %}apt update \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    apt update \
     && apt install -y gettext gfortran graphviz graphviz-dev libffi-dev libfreetype6-dev libgeos-dev libjpeg8-dev liblapack-dev libmysqlclient-dev libpng-dev libsqlite3-dev libxmlsec1-dev lynx mysql-client ntp pkg-config rdfind
 
 # From then on, run as unprivileged "app" user
@@ -154,7 +157,7 @@ RUN useradd --no-log-init --home-dir /openedx --create-home --shell /bin/bash --
 USER ${APP_USER_ID}
 
 # https://hub.docker.com/r/powerman/dockerize/tags
-COPY {% if is_buildkit_enabled() %}--link {% endif %}--from=docker.io/powerman/dockerize:0.19.0 /usr/local/bin/dockerize /usr/local/bin/dockerize
+COPY --link --from=docker.io/powerman/dockerize:0.19.0 /usr/local/bin/dockerize /usr/local/bin/dockerize
 COPY --chown=app:app --from=edx-platform / /openedx/edx-platform
 COPY --chown=app:app --from=locales /openedx/locale /openedx/locale
 COPY --chown=app:app --from=python /opt/pyenv /opt/pyenv
@@ -248,16 +251,19 @@ FROM production as development
 
 # Install useful system requirements (as root)
 USER root
-RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
-    --mount=type=cache,target=/var/lib/apt,sharing=locked {% endif %}apt update && \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    apt update && \
     apt install -y vim iputils-ping dnsutils telnet
 USER app
 
 # Install dev python requirements
-RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install -r requirements/edx/development.txt
+RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \
+    pip install -r requirements/edx/development.txt
 # https://pypi.org/project/ipdb/
 # https://pypi.org/project/ipython
-RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install ipdb==0.13.13 ipython==8.12.0
+RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \
+    pip install ipdb==0.13.13 ipython==8.12.0
 
 # Add ipdb as default PYTHONBREAKPOINT
 ENV PYTHONBREAKPOINT=ipdb.set_trace

tutor/utils.py

@@ -173,25 +173,6 @@ def docker(*command: str) -> int:
     return execute("docker", *command)
 
 
-@lru_cache(maxsize=None)
-def is_buildkit_enabled() -> bool:
-    """
-    A helper function to determine whether we can run `docker buildx` with BuildKit.
-    """
-    # First, we respect the DOCKER_BUILDKIT environment variable
-    enabled_by_env = {
-        "1": True,
-        "0": False,
-    }.get(os.environ.get("DOCKER_BUILDKIT", ""))
-    if enabled_by_env is not None:
-        return enabled_by_env
-    try:
-        subprocess.run(["docker", "buildx", "version"], capture_output=True, check=True)
-        return True
-    except subprocess.CalledProcessError:
-        return False
-
-
 def docker_compose(*command: str) -> int:
     return execute("docker", "compose", *command)