diff --git a/changelog.d/20230818_112124_kyle_buildkit.md b/changelog.d/20230818_112124_kyle_buildkit.md new file mode 100644 index 0000000..7084b29 --- /dev/null +++ b/changelog.d/20230818_112124_kyle_buildkit.md @@ -0,0 +1,2 @@ +- [Deprecation] The template variable ``is_buildkit_enabled``, which now always returns True, is deprecated. Plugin authors should assume BuildKit is enabled and remove the variable from their templates (by @kdmccormick). +- 💥[Deprecation] Tutor no longer supports the legacy Docker builder, which was previously available by setting ``DOCKER_BUILDKIT=0`` in the host environment. Going forward, Tutor will always use BuildKit (a.k.a. ``docker buildx`` in Docker v19-v22, or just ``docker build`` in Docker v23). This transition will improve build performance and should be seamless for Tutor users who are running a supported Docker version (by @kdmccormick). diff --git a/docs/tutor.rst b/docs/tutor.rst index 4036ff8..25ec627 100644 --- a/docs/tutor.rst +++ b/docs/tutor.rst @@ -93,6 +93,13 @@ An optional BRANCH suffix may be appended to the release name to indicate that e `Officially-supported plugins `__ follow the same versioning pattern. As a third-party plugin developer, you are encouraged to use the same pattern to make it immediately clear to your end-users which Open edX versions are supported. +In Tutor and its officially-supported plugins, certain features, API endpoints, and older depenency versions are periodically deprecated. Generally, warnings are added to the Changelogs and/or the command-line interface one major release before support for any behavior is removed. In order to keep track of pending removals in the source code, comments containing the string ``REMOVE-AFTER-VXX`` should be used, where ```` is the last major version that must support the behavior. For example:: + + # This has been replaced with SOME_NEW_HOOK (REMOVE-AFTER-V25). + SOME_OLD_HOOK = Filter() + +indicates that this filter definition can be removed as soon as Tutor v26.0.0. + .. _contributing: Contributing to Tutor diff --git a/tests/commands/test_images.py b/tests/commands/test_images.py index 7b09577..32ddba8 100644 --- a/tests/commands/test_images.py +++ b/tests/commands/test_images.py @@ -128,10 +128,8 @@ class ImagesTests(PluginsTestCase, TestCommandMixin): "service1", ] with temporary_root() as root: - utils.is_buildkit_enabled.cache_clear() - with patch.object(utils, "is_buildkit_enabled", return_value=False): - self.invoke_in_root(root, ["config", "save"]) - result = self.invoke_in_root(root, build_args) + self.invoke_in_root(root, ["config", "save"]) + result = self.invoke_in_root(root, build_args) self.assertIsNone(result.exception) self.assertEqual(0, result.exit_code) image_build.assert_called() @@ -146,7 +144,9 @@ class ImagesTests(PluginsTestCase, TestCommandMixin): "host", "--target", "target", + "--output=type=docker", "docker_args", + "--cache-from=type=registry,ref=service1:1.0.0-cache", ], list(image_build.call_args[0][1:]), ) diff --git a/tutor/commands/images.py b/tutor/commands/images.py index d21e280..5dfe2b9 100644 --- a/tutor/commands/images.py +++ b/tutor/commands/images.py @@ -156,7 +156,7 @@ def images_command() -> None: # Export image to docker. This is necessary to make the image available to docker-compose. # The `--load` option is a shorthand for `--output=type=docker`. default="type=docker", - help="Same as `docker build --output=...`. This option will only be used when BuildKit is enabled.", + help="Same as `docker build --output=...`.", ) @click.option( "-a", @@ -211,7 +211,7 @@ def build( command_args += ["--add-host", add_host] if target: command_args += ["--target", target] - if utils.is_buildkit_enabled() and docker_output: + if docker_output: command_args.append(f"--output={docker_output}") if docker_args: command_args += docker_args @@ -223,27 +223,19 @@ def build( image_build_args = [*command_args, *custom_args] # Registry cache - if utils.is_buildkit_enabled(): - if not no_registry_cache: - image_build_args.append( - f"--cache-from=type=registry,ref={tag}-cache" - ) - if cache_to_registry: - image_build_args.append( - f"--cache-to=type=registry,mode=max,ref={tag}-cache" - ) + if not no_registry_cache: + image_build_args.append(f"--cache-from=type=registry,ref={tag}-cache") + if cache_to_registry: + image_build_args.append( + f"--cache-to=type=registry,mode=max,ref={tag}-cache" + ) # Build contexts for host_path, stage_name in build_contexts.get(name, []): - if utils.is_buildkit_enabled(): - fmt.echo_info( - f"Adding {host_path} to the build context '{stage_name}' of image '{image}'" - ) - image_build_args.append(f"--build-context={stage_name}={host_path}") - else: - fmt.echo_alert( - f"Unable to add {host_path} to the build context '{stage_name}' of image '{host_path}' because BuildKit is disabled." - ) + fmt.echo_info( + f"Adding {host_path} to the build context '{stage_name}' of image '{image}'" + ) + image_build_args.append(f"--build-context={stage_name}={host_path}") # Build images.build( diff --git a/tutor/env.py b/tutor/env.py index 9ced7d5..1baf54b 100644 --- a/tutor/env.py +++ b/tutor/env.py @@ -54,7 +54,9 @@ def _prepare_environment() -> None: ("HOST_USER_ID", utils.get_user_id()), ("TUTOR_APP", __app__.replace("-", "_")), ("TUTOR_VERSION", __version__), - ("is_buildkit_enabled", utils.is_buildkit_enabled), + # BuildKit used to be optional. Now, it's always enabled. + # This constant is just for temporary backwards compatibility (REMOVE-AFTER-V16). + ("is_buildkit_enabled", lambda: True), ], ) diff --git a/tutor/hooks/catalog.py b/tutor/hooks/catalog.py index 66ed97a..fa89015 100644 --- a/tutor/hooks/catalog.py +++ b/tutor/hooks/catalog.py @@ -251,8 +251,7 @@ class Filters: #: names must be prefixed with the plugin name in all-caps. CONFIG_UNIQUE: Filter[list[tuple[str, Any]], []] = Filter() - #: Use this filter to modify the ``docker build`` command. For instance, to replace - #: the ``build`` subcommand by ``buildx build``. + #: Use this filter to modify the ``docker build`` command. #: #: :parameter list[str] command: the full build command, including options and #: arguments. Note that these arguments do not include the leading ``docker`` command. @@ -335,7 +334,7 @@ class Filters: #: - ``HOST_USER_ID``: the numerical ID of the user on the host. #: - ``TUTOR_APP``: the app name ("tutor" by default), used to determine the dev/local project names. #: - ``TUTOR_VERSION``: the current version of Tutor. - #: - ``is_buildkit_enabled``: a boolean function that indicates whether BuildKit is available on the host. + #: - ``is_buildkit_enabled``: a deprecated function which always returns ``True`` now. Will be removed after Quince. #: - ``iter_values_named``: a function to iterate on variables that start or end with a given string. #: - ``iter_mounts``: a function that yields compose-compatible bind-mounts for any given service. #: - ``patch``: a function to incorporate extra content into a template. diff --git a/tutor/images.py b/tutor/images.py index 26f80b1..ebab4ec 100644 --- a/tutor/images.py +++ b/tutor/images.py @@ -4,8 +4,9 @@ from tutor import fmt, hooks, utils def build(path: str, tag: str, *args: str) -> None: fmt.echo_info(f"Building image {tag}") build_command = ["build", f"--tag={tag}", *args, path] - if utils.is_buildkit_enabled(): - build_command.insert(0, "buildx") + # `buildx` can be removed once Tutor requires Docker v23+. At that point, BuildKit will be + # enabled by default for all Docker users. + build_command.insert(0, "buildx") command = hooks.Filters.DOCKER_BUILD_COMMAND.apply(build_command) utils.docker(*command) diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index 8b775b5..0f0aa66 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -1,11 +1,11 @@ -{% if is_buildkit_enabled() %}# syntax=docker/dockerfile:1.4{% endif %} +# syntax=docker/dockerfile:1.4 ###### Minimal image with base system requirements for most stages FROM docker.io/ubuntu:20.04 as minimal LABEL maintainer="Overhang.io " ENV DEBIAN_FRONTEND=noninteractive -RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,target=/var/lib/apt,sharing=locked{% endif %} \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ apt update && \ apt install -y build-essential curl git language-pack-en ENV LC_ALL en_US.UTF-8 @@ -14,8 +14,9 @@ ENV LC_ALL en_US.UTF-8 ###### Install python with pyenv in /opt/pyenv and create virtualenv in /openedx/venv FROM minimal as python # https://github.com/pyenv/pyenv/wiki/Common-build-problems#prerequisites -RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,target=/var/lib/apt,sharing=locked {% endif %}apt update && \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt update && \ apt install -y libssl-dev zlib1g-dev libbz2-dev \ libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \ xz-utils tk-dev libffi-dev liblzma-dev python-openssl git @@ -77,12 +78,14 @@ ENV PATH /openedx/venv/bin:${PATH} ENV VIRTUAL_ENV /openedx/venv/ ENV XDG_CACHE_HOME /openedx/.cache -RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,target=/var/lib/apt,sharing=locked {% endif %}apt update \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt update \ && apt install -y software-properties-common libmysqlclient-dev libxmlsec1-dev libgeos-dev # Install the right version of pip/setuptools -RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install \ +RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ + pip install \ # https://pypi.org/project/setuptools/ # https://pypi.org/project/pip/ # https://pypi.org/project/wheel/ @@ -92,14 +95,13 @@ RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip, RUN pip install https://github.com/overhangio/py2neo/releases/download/2021.2.3/py2neo-2021.2.3.tar.gz # Install base requirements -{% if not is_buildkit_enabled() %} -COPY --from=edx-platform /requirements/edx/base.txt /openedx/edx-platform/requirements/edx/base.txt -{% endif %} -RUN {% if is_buildkit_enabled() %}--mount=type=bind,from=edx-platform,source=/requirements/edx/base.txt,target=/openedx/edx-platform/requirements/edx/base.txt \ - --mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install -r /openedx/edx-platform/requirements/edx/base.txt +RUN --mount=type=bind,from=edx-platform,source=/requirements/edx/base.txt,target=/openedx/edx-platform/requirements/edx/base.txt \ + --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ + pip install -r /openedx/edx-platform/requirements/edx/base.txt # Install extra requirements -RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install \ +RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ + pip install \ # Use redis as a django cache https://pypi.org/project/django-redis/ django-redis==5.2.0 \ # uwsgi server https://pypi.org/project/uWSGI/ @@ -109,11 +111,14 @@ RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip, # Install private requirements: this is useful for installing custom xblocks. COPY ./requirements/ /openedx/requirements -RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}cd /openedx/requirements/ \ +RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ + cd /openedx/requirements/ \ && touch ./private.txt \ && pip install -r ./private.txt -{% for extra_requirements in OPENEDX_EXTRA_PIP_REQUIREMENTS %}RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install '{{ extra_requirements }}' +{% for extra_requirements in OPENEDX_EXTRA_PIP_REQUIREMENTS %} +RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ + pip install '{{ extra_requirements }}' {% endfor %} ###### Install nodejs with nodeenv in /openedx/nodeenv @@ -129,21 +134,19 @@ RUN nodeenv /openedx/nodeenv --node=16.14.0 --prebuilt # Install nodejs requirements ARG NPM_REGISTRY={{ NPM_REGISTRY }} WORKDIR /openedx/edx-platform -{% if not is_buildkit_enabled() %} -COPY --from=edx-platform /package.json /openedx/edx-platform/package.json -COPY --from=edx-platform /package-lock.json /openedx/edx-platform/package-lock.json -{% endif %} -RUN {% if is_buildkit_enabled() %}--mount=type=bind,from=edx-platform,source=/package.json,target=/openedx/edx-platform/package.json \ +RUN --mount=type=bind,from=edx-platform,source=/package.json,target=/openedx/edx-platform/package.json \ --mount=type=bind,from=edx-platform,source=/package-lock.json,target=/openedx/edx-platform/package-lock.json \ --mount=type=bind,from=edx-platform,source=/scripts/copy-node-modules.sh,target=/openedx/edx-platform/scripts/copy-node-modules.sh \ - --mount=type=cache,target=/root/.npm,sharing=shared {% endif %}npm clean-install --no-audit --registry=$NPM_REGISTRY + --mount=type=cache,target=/root/.npm,sharing=shared \ + npm clean-install --no-audit --registry=$NPM_REGISTRY ###### Production image with system and python requirements FROM minimal as production # Install system requirements -RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,target=/var/lib/apt,sharing=locked {% endif %}apt update \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt update \ && apt install -y gettext gfortran graphviz graphviz-dev libffi-dev libfreetype6-dev libgeos-dev libjpeg8-dev liblapack-dev libmysqlclient-dev libpng-dev libsqlite3-dev libxmlsec1-dev lynx mysql-client ntp pkg-config rdfind # From then on, run as unprivileged "app" user @@ -154,7 +157,7 @@ RUN useradd --no-log-init --home-dir /openedx --create-home --shell /bin/bash -- USER ${APP_USER_ID} # https://hub.docker.com/r/powerman/dockerize/tags -COPY {% if is_buildkit_enabled() %}--link {% endif %}--from=docker.io/powerman/dockerize:0.19.0 /usr/local/bin/dockerize /usr/local/bin/dockerize +COPY --link --from=docker.io/powerman/dockerize:0.19.0 /usr/local/bin/dockerize /usr/local/bin/dockerize COPY --chown=app:app --from=edx-platform / /openedx/edx-platform COPY --chown=app:app --from=locales /openedx/locale /openedx/locale COPY --chown=app:app --from=python /opt/pyenv /opt/pyenv @@ -248,16 +251,19 @@ FROM production as development # Install useful system requirements (as root) USER root -RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,target=/var/lib/apt,sharing=locked {% endif %}apt update && \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt update && \ apt install -y vim iputils-ping dnsutils telnet USER app # Install dev python requirements -RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install -r requirements/edx/development.txt +RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ + pip install -r requirements/edx/development.txt # https://pypi.org/project/ipdb/ # https://pypi.org/project/ipython -RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install ipdb==0.13.13 ipython==8.12.0 +RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ + pip install ipdb==0.13.13 ipython==8.12.0 # Add ipdb as default PYTHONBREAKPOINT ENV PYTHONBREAKPOINT=ipdb.set_trace diff --git a/tutor/utils.py b/tutor/utils.py index 59adee4..2d8d3e5 100644 --- a/tutor/utils.py +++ b/tutor/utils.py @@ -173,25 +173,6 @@ def docker(*command: str) -> int: return execute("docker", *command) -@lru_cache(maxsize=None) -def is_buildkit_enabled() -> bool: - """ - A helper function to determine whether we can run `docker buildx` with BuildKit. - """ - # First, we respect the DOCKER_BUILDKIT environment variable - enabled_by_env = { - "1": True, - "0": False, - }.get(os.environ.get("DOCKER_BUILDKIT", "")) - if enabled_by_env is not None: - return enabled_by_env - try: - subprocess.run(["docker", "buildx", "version"], capture_output=True, check=True) - return True - except subprocess.CalledProcessError: - return False - - def docker_compose(*command: str) -> int: return execute("docker", "compose", *command)