From f42abe18ad333cb887a520b0def2b436097c2b28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Thu, 16 Mar 2023 09:41:21 +0100 Subject: [PATCH 1/4] =?UTF-8?q?feat:=20highlight=20enabled=20plugins=20wit?= =?UTF-8?q?h=20=E2=9C=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In the output of `plugins list` it's difficult to see which plugins are enabled at a glance. This change adds a more visible checkmark to the output. Ex: $ tutor plugins list NAME STATUS VERSION cairn ✅ enabled 15.0.3 discovery installed 15.0.0 ecommerce installed 15.0.1 forum installed 14.0.0 mfe ✅ enabled 15.0.5 minio installed 15.1.0 --- .../20230316_094112_regis_better_highlight_enabled_plugins.md | 1 + tutor/commands/plugins.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/20230316_094112_regis_better_highlight_enabled_plugins.md diff --git a/changelog.d/20230316_094112_regis_better_highlight_enabled_plugins.md b/changelog.d/20230316_094112_regis_better_highlight_enabled_plugins.md new file mode 100644 index 0000000..70e0402 --- /dev/null +++ b/changelog.d/20230316_094112_regis_better_highlight_enabled_plugins.md @@ -0,0 +1 @@ +- [Improvement] Better highlight enabled plugins in `tutor plugins list`. (by @regisb) diff --git a/tutor/commands/plugins.py b/tutor/commands/plugins.py index 2b9bdee..0ed848e 100644 --- a/tutor/commands/plugins.py +++ b/tutor/commands/plugins.py @@ -335,7 +335,7 @@ def plugin_status(name: str) -> str: Return the status of a plugin. Either: "enabled", "installed" or "not installed". """ if plugins.is_loaded(name): - return "enabled" + return "✅ enabled" if plugins.is_installed(name): return "installed" return "not installed" From ff5357cdc041a30b37774d08dc358bcaf5747b84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Mon, 13 Mar 2023 16:41:36 +0100 Subject: [PATCH 2/4] improvement: faster build with `npm ci` --- changelog.d/20230313_163942_regis_docker_optimize.md | 1 + tutor/templates/build/openedx/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/20230313_163942_regis_docker_optimize.md diff --git a/changelog.d/20230313_163942_regis_docker_optimize.md b/changelog.d/20230313_163942_regis_docker_optimize.md new file mode 100644 index 0000000..491e07e --- /dev/null +++ b/changelog.d/20230313_163942_regis_docker_optimize.md @@ -0,0 +1 @@ +- [Improvement] Faster build with `npm clean-install` instead of `npm install` in the openedx Docker image. This may change the version of npm packages installed next to edx-platform. (by @regisb) diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index b95e5b3..0140383 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -119,7 +119,7 @@ ARG NPM_REGISTRY={{ NPM_REGISTRY }} COPY --from=code /openedx/edx-platform/package.json /openedx/edx-platform/package.json COPY --from=code /openedx/edx-platform/package-lock.json /openedx/edx-platform/package-lock.json WORKDIR /openedx/edx-platform -RUN npm install --verbose --registry=$NPM_REGISTRY +RUN npm clean-install --verbose --registry=$NPM_REGISTRY ###### Production image with system and python requirements FROM minimal as production From 28dce8c51ae00b9f5ecfa9987dd0c153bbc1e4a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Mon, 13 Mar 2023 18:03:47 +0100 Subject: [PATCH 3/4] improvement: customisable `docker build` command This paves the way for `docker buildx build` and better caching. For instance, with this change you can try out the following plugin, which should make image building much faster in CI: https://gist.github.com/regisb/4049622ec4b48cbd48c89ec708dc5252 (not ready for production just yet, we still need to build and push the images) --- changelog.d/20230313_172348_regis_docker_optimize.md | 1 + tutor/commands/k8s.py | 2 +- tutor/hooks/catalog.py | 9 ++++++++- tutor/images.py | 9 ++++++--- 4 files changed, 16 insertions(+), 5 deletions(-) create mode 100644 changelog.d/20230313_172348_regis_docker_optimize.md diff --git a/changelog.d/20230313_172348_regis_docker_optimize.md b/changelog.d/20230313_172348_regis_docker_optimize.md new file mode 100644 index 0000000..811724c --- /dev/null +++ b/changelog.d/20230313_172348_regis_docker_optimize.md @@ -0,0 +1 @@ +- [Feature] Introduce the `DOCKER_BUILD_COMMAND` which makes it possible to customize the `docker build` command. (by @regisb) diff --git a/tutor/commands/k8s.py b/tutor/commands/k8s.py index d26d7cc..db601f4 100644 --- a/tutor/commands/k8s.py +++ b/tutor/commands/k8s.py @@ -1,6 +1,6 @@ from datetime import datetime from time import sleep -from typing import Any, List, Optional, Type, Iterable +from typing import Any, Iterable, List, Optional, Type import click diff --git a/tutor/hooks/catalog.py b/tutor/hooks/catalog.py index 498aebd..9db9d70 100644 --- a/tutor/hooks/catalog.py +++ b/tutor/hooks/catalog.py @@ -282,13 +282,20 @@ class Filters: "config:overrides" ) - #: Declare uniqaue configuration settings that must be saved in the user ``config.yml`` file. This is where + #: Declare unique configuration settings that must be saved in the user ``config.yml`` file. This is where #: you should declare passwords and randomly-generated values that are different from one environment to the next. #: #: :parameter list[tuple[str, ...]] items: list of (name, value) new settings. All #: names must be prefixed with the plugin name in all-caps. CONFIG_UNIQUE: Filter[list[tuple[str, Any]], []] = filters.get("config:unique") + #: Use this filter to modify the ``docker build`` command. For instance, to replace + #: the ``build`` subcommand by ``buildx build``. + #: + #: :parameter list[str] command: the full build command, including options and + #: arguments. Note that these arguments do not include the leading ``docker`` command. + DOCKER_BUILD_COMMAND: Filter[list[str], []] = filters.get("docker:build:command") + #: List of patches that should be inserted in a given location of the templates. The #: filter name must be formatted with the patch name. #: This filter is not so convenient and plugin developers will probably diff --git a/tutor/images.py b/tutor/images.py index b87b550..d0637fe 100644 --- a/tutor/images.py +++ b/tutor/images.py @@ -1,5 +1,5 @@ -from . import fmt, utils -from .types import Config, get_typed +from tutor import fmt, hooks, utils +from tutor.types import Config, get_typed def get_tag(config: Config, name: str) -> str: @@ -9,7 +9,10 @@ def get_tag(config: Config, name: str) -> str: def build(path: str, tag: str, *args: str) -> None: fmt.echo_info(f"Building image {tag}") - utils.docker("build", "-t", tag, *args, path) + command = hooks.Filters.DOCKER_BUILD_COMMAND.apply( + ["build", "-t", tag, *args, path] + ) + utils.docker(*command) def pull(tag: str) -> None: From fa9e86e188f0a70b84f8b688e83a32a65b7bea10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Mon, 13 Mar 2023 18:07:26 +0100 Subject: [PATCH 4/4] improvement: COPY dockerize for faster build Dockerize now ships with multi-arch Docker images, so we can just COPY the binary from these images. This allows us to skip an image layer. --- changelog.d/20230313_180716_regis_docker_optimize.md | 1 + tutor/templates/build/openedx/Dockerfile | 12 ++---------- 2 files changed, 3 insertions(+), 10 deletions(-) create mode 100644 changelog.d/20230313_180716_regis_docker_optimize.md diff --git a/changelog.d/20230313_180716_regis_docker_optimize.md b/changelog.d/20230313_180716_regis_docker_optimize.md new file mode 100644 index 0000000..98bac1a --- /dev/null +++ b/changelog.d/20230313_180716_regis_docker_optimize.md @@ -0,0 +1 @@ +- [Improvement] During openedx image build, copy `dockerize` utility from Docker registry for better efficiency. (by @regisb) diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index 0140383..e3d1633 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -21,15 +21,6 @@ RUN git clone https://github.com/pyenv/pyenv $PYENV_ROOT --branch v2.2.2 --depth RUN $PYENV_ROOT/bin/pyenv install $PYTHON_VERSION RUN $PYENV_ROOT/versions/$PYTHON_VERSION/bin/python -m venv /openedx/venv -###### Install Dockerize to wait for mysql DB availability -FROM minimal as dockerize -# https://github.com/powerman/dockerize/releases -ARG DOCKERIZE_VERSION=v0.16.0 -RUN dockerize_url="https://github.com/powerman/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-$(uname -m | sed 's@aarch@arm@')" \ - && echo "Downloading dockerize from $dockerize_url" \ - && curl --fail --location --output /usr/local/bin/dockerize $dockerize_url \ - && chmod a+x /usr/local/bin/dockerize - ###### Checkout edx-platform code FROM minimal as code ARG EDX_PLATFORM_REPOSITORY={{ EDX_PLATFORM_REPOSITORY }} @@ -136,7 +127,8 @@ RUN if [ "$APP_USER_ID" = 0 ]; then echo "app user may not be root" && false; fi RUN useradd --home-dir /openedx --create-home --shell /bin/bash --uid ${APP_USER_ID} app USER ${APP_USER_ID} -COPY --from=dockerize /usr/local/bin/dockerize /usr/local/bin/dockerize +# https://hub.docker.com/r/powerman/dockerize/tags +COPY --from=docker.io/powerman/dockerize:0.19.0 /usr/local/bin/dockerize /usr/local/bin/dockerize COPY --chown=app:app --from=code /openedx/edx-platform /openedx/edx-platform COPY --chown=app:app --from=locales /openedx/locale /openedx/locale COPY --chown=app:app --from=python /opt/pyenv /opt/pyenv