From 947b37524f0adb80afa2a28b408d50632660e075 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Tue, 2 May 2023 10:49:19 +0200 Subject: [PATCH] feat: auto build "openedx-dev" on "dev launch" To achieve that, we introduce a new IMAGES_BUILD_REQUIRED filter. --- changelog.d/20230412_100608_regis_palm.md | 1 + docs/dev.rst | 11 +-- docs/tutorials/arm64.rst | 2 +- tutor/commands/compose.py | 86 +++++++++++++++-------- tutor/commands/dev.py | 13 ++++ tutor/commands/images.py | 20 ++++-- tutor/commands/local.py | 2 + tutor/hooks/catalog.py | 19 +++-- 8 files changed, 106 insertions(+), 48 deletions(-) diff --git a/changelog.d/20230412_100608_regis_palm.md b/changelog.d/20230412_100608_regis_palm.md index 3d6e6c5..9e3c8d2 100644 --- a/changelog.d/20230412_100608_regis_palm.md +++ b/changelog.d/20230412_100608_regis_palm.md @@ -12,3 +12,4 @@ - 💥[Feature] The "openedx" Docker image is no longer built with docker-compose in development on `tutor dev start`. This used to be the case to make sure that it was always up-to-date, but it introduced a discrepancy in how images were build (`docker compose build` vs `docker build`). As a consequence: - The "openedx" Docker image in development can be built with `tutor images build openedx-dev`. - The `tutor dev/local start --skip-build` option is removed. It is replaced by opt-in `--build`. + - [Improvement] The `IMAGES_BUILD` filter now supports relative paths as strings, and not just as tuple of strings. diff --git a/docs/dev.rst b/docs/dev.rst index a429711..c22c10a 100644 --- a/docs/dev.rst +++ b/docs/dev.rst @@ -16,18 +16,19 @@ Then, optionally, tell Tutor to use a local fork of edx-platform.:: tutor config save --append MOUNTS=./edx-platform -Then, build the "openedx" Docker image for development and launch the developer platfornm setup process:: +Then, launch the developer platform setup process:: tutor images build openedx-dev tutor dev launch This will perform several tasks. It will: +* build the "openedx-dev" Docker image, which is based on the "openedx" production image but is `specialized for developer usage`_ (eventually with your fork), * stop any existing locally-running Tutor containers, * disable HTTPS, * set ``LMS_HOST`` to `local.overhang.io `_ (a convenience domain that simply `points at 127.0.0.1 `_), * prompt for a platform details (with suitable defaults), -* build an ``openedx-dev`` image, which is based ``openedx`` production image but is `specialized for developer usage`_, +* build an ``openedx-dev`` image, * start LMS, CMS, supporting services, and any plugged-in services, * ensure databases are created and migrated, and * run service initialization scripts, such as service user creation and Waffle configuration. @@ -121,9 +122,7 @@ Rebuilding the openedx-dev image The ``openedx-dev`` Docker image is based on the same ``openedx`` image used by ``tutor local ...`` to run LMS and CMS. However, it has a few differences to make it more convenient for developers: - The user that runs inside the container has the same UID as the user on the host, to avoid permission problems inside mounted volumes (and in particular in the edx-platform repository). - - Additional Python and system requirements are installed for convenient debugging: `ipython `__, `ipdb `__, vim, telnet. - - The edx-platform `development requirements `__ are installed. @@ -131,6 +130,10 @@ If you are using a custom ``openedx`` image, then you will need to rebuild ``ope tutor images build openedx-dev +Alternatively, the image will be automatically rebuilt every time you run:: + + tutor dev launch + .. _bind_mounts: diff --git a/docs/tutorials/arm64.rst b/docs/tutorials/arm64.rst index 60d73a8..23beed5 100644 --- a/docs/tutorials/arm64.rst +++ b/docs/tutorials/arm64.rst @@ -28,7 +28,7 @@ Then, build the "openedx" and "permissions" images:: If you want to use Tutor as an Open edX development environment, you should also build the development image:: - tutor images build openedx-dev + tutor images build openedx-dev # this will be automatically done by `tutor dev launch` From this point on, use Tutor as normal. For example, start Open edX and run migrations with:: diff --git a/tutor/commands/compose.py b/tutor/commands/compose.py index ab0bff1..288725f 100644 --- a/tutor/commands/compose.py +++ b/tutor/commands/compose.py @@ -11,7 +11,7 @@ from tutor import env as tutor_env from tutor import fmt, hooks from tutor import interactive as interactive_config from tutor import utils -from tutor.commands import jobs +from tutor.commands import images, jobs from tutor.commands.config import save as config_save_command from tutor.commands.context import BaseTaskContext from tutor.commands.upgrade import OPENEDX_RELEASE_NAMES @@ -72,6 +72,8 @@ class ComposeTaskRunner(BaseComposeTaskRunner): class BaseComposeContext(BaseTaskContext): + NAME: t.Literal["local", "dev"] + def job_runner(self, config: Config) -> ComposeTaskRunner: raise NotImplementedError @@ -79,14 +81,29 @@ class BaseComposeContext(BaseTaskContext): @click.command(help="Configure and run Open edX from scratch") @click.option("-I", "--non-interactive", is_flag=True, help="Run non-interactively") @click.option("-p", "--pullimages", is_flag=True, help="Update docker images") +@click.option("--skip-build", is_flag=True, help="Skip building Docker images") @click.pass_context def launch( context: click.Context, non_interactive: bool, pullimages: bool, + skip_build: bool, ) -> None: + context_name = context.obj.NAME + run_for_prod = context_name != "dev" + utils.warn_macos_docker_memory() - interactive_upgrade(context, not non_interactive) + interactive_upgrade(context, not non_interactive, run_for_prod) + interactive_configuration(context, not non_interactive, run_for_prod) + + config = tutor_config.load(context.obj.root) + + if not skip_build: + click.echo(fmt.title("Building Docker images")) + images_to_build = hooks.Filters.IMAGES_BUILD_REQUIRED.apply([], context_name) + if not images_to_build: + fmt.echo_info("No image to build") + context.invoke(images.build, image_names=images_to_build) click.echo(fmt.title("Stopping any existing platform")) context.invoke(stop) @@ -101,12 +118,9 @@ def launch( click.echo(fmt.title("Database creation and migrations")) context.invoke(do.commands["init"]) - config = tutor_config.load(context.obj.root) - project_name = context.obj.job_runner(config).project_name - # Print the urls of the user-facing apps public_app_hosts = "" - for host in hooks.Filters.APP_PUBLIC_HOSTS.iterate(project_name): + for host in hooks.Filters.APP_PUBLIC_HOSTS.iterate(context_name): public_app_host = tutor_env.render_str( config, "{% if ENABLE_HTTPS %}https{% else %}http{% endif %}://" + host ) @@ -119,7 +133,9 @@ def launch( ) -def interactive_upgrade(context: click.Context, interactive: bool) -> None: +def interactive_upgrade( + context: click.Context, interactive: bool, run_for_prod: bool +) -> None: """ Piece of code that is only used in launch. """ @@ -146,30 +162,38 @@ Are you sure you want to continue?""" from_release=run_upgrade_from_release, ) + # Update env and configuration + interactive_configuration(context, interactive, run_for_prod) + + # Post upgrade + if run_upgrade_from_release and interactive: + question = f"""Your platform is being upgraded from {run_upgrade_from_release.capitalize()}. + + If you run custom Docker images, you must rebuild them now by running the following command in a different shell: + + tutor images build all # list your custom images here + + See the documentation for more information: + + https://docs.tutor.overhang.io/install.html#upgrading-to-a-new-open-edx-release + + Press enter when you are ready to continue""" + click.confirm( + fmt.question(question), default=True, abort=True, prompt_suffix=" " + ) + + +def interactive_configuration( + context: click.Context, interactive: bool, run_for_prod: bool +) -> None: click.echo(fmt.title("Interactive platform configuration")) config = tutor_config.load_minimal(context.obj.root) if interactive: - interactive_config.ask_questions(config) + interactive_config.ask_questions(config, run_for_prod=run_for_prod) tutor_config.save_config_file(context.obj.root, config) config = tutor_config.load_full(context.obj.root) tutor_env.save(context.obj.root, config) - if run_upgrade_from_release and interactive: - question = f"""Your platform is being upgraded from {run_upgrade_from_release.capitalize()}. - -If you run custom Docker images, you must rebuild them now by running the following command in a different shell: - - tutor images build all # list your custom images here - -See the documentation for more information: - - https://docs.tutor.overhang.io/install.html#upgrading-to-a-new-open-edx-release - -Press enter when you are ready to continue""" - click.confirm( - fmt.question(question), default=True, abort=True, prompt_suffix=" " - ) - @click.command( short_help="Perform release-specific upgrade tasks", @@ -420,12 +444,14 @@ def _mount_edx_platform( return volumes -def _edx_platform_public_hosts(hosts: list[str], project_name: str) -> list[str]: - edx_platform_hosts = ["{{ LMS_HOST }}", "{{ CMS_HOST }}"] - if project_name == "dev": - edx_platform_hosts[0] += ":8000" - edx_platform_hosts[1] += ":8001" - hosts += edx_platform_hosts +@hooks.Filters.APP_PUBLIC_HOSTS.add() +def _edx_platform_public_hosts( + hosts: list[str], context_name: t.Literal["local", "dev"] +) -> list[str]: + if context_name == "dev": + hosts += ["{{ LMS_HOST }}:8000", "{{ CMS_HOST }}:8001"] + else: + hosts += ["{{ LMS_HOST }}", "{{ CMS_HOST }}"] return hosts diff --git a/tutor/commands/dev.py b/tutor/commands/dev.py index a60d7e7..659e303 100644 --- a/tutor/commands/dev.py +++ b/tutor/commands/dev.py @@ -1,5 +1,7 @@ from __future__ import annotations +import typing as t + import click from tutor import env as tutor_env @@ -30,6 +32,8 @@ class DevTaskRunner(compose.ComposeTaskRunner): class DevContext(compose.BaseComposeContext): + NAME = "dev" + def job_runner(self, config: Config) -> DevTaskRunner: return DevTaskRunner(self.root, config) @@ -51,4 +55,13 @@ def _stop_on_local_start(root: str, config: Config, project_name: str) -> None: runner.docker_compose("stop") +@hooks.Filters.IMAGES_BUILD_REQUIRED.add() +def _build_openedx_dev_on_launch( + image_names: list[str], context_name: t.Literal["local", "dev"] +) -> list[str]: + if context_name == "dev": + image_names.append("openedx-dev") + return image_names + + compose.add_commands(dev) diff --git a/tutor/commands/images.py b/tutor/commands/images.py index 1d7b8d6..9c74c65 100644 --- a/tutor/commands/images.py +++ b/tutor/commands/images.py @@ -20,22 +20,27 @@ BASE_IMAGE_NAMES = [ @hooks.Filters.IMAGES_BUILD.add() def _add_core_images_to_build( - build_images: list[tuple[str, tuple[str, ...], str, tuple[str, ...]]], + build_images: list[tuple[str, t.Union[str, tuple[str, ...]], str, tuple[str, ...]]], config: Config, -) -> list[tuple[str, tuple[str, ...], str, tuple[str, ...]]]: +) -> list[tuple[str, t.Union[str, tuple[str, ...]], str, tuple[str, ...]]]: """ Add base images to the list of Docker images to build on `tutor build all`. """ for image, tag in BASE_IMAGE_NAMES: build_images.append( - (image, ("build", image), tutor_config.get_typed(config, tag, str), ()) + ( + image, + os.path.join("build", image), + tutor_config.get_typed(config, tag, str), + (), + ) ) # Build openedx-dev image build_images.append( ( "openedx-dev", - ("build", "openedx"), + os.path.join("build", "openedx"), tutor_config.get_typed(config, "DOCKER_IMAGE_OPENEDX_DEV", str), ( "--target=development", @@ -188,7 +193,7 @@ def build( # Build images.build( - tutor_env.pathjoin(context.root, *path), + tutor_env.pathjoin(context.root, path), tag, *image_build_args, ) @@ -262,7 +267,7 @@ def printtag(context: Context, image_names: list[str]) -> None: def find_images_to_build( config: Config, image: str -) -> t.Iterator[tuple[str, tuple[str, ...], str, tuple[str, ...]]]: +) -> t.Iterator[tuple[str, str, str, tuple[str, ...]]]: """ Iterate over all images to build. @@ -272,10 +277,11 @@ def find_images_to_build( """ found = False for name, path, tag, args in hooks.Filters.IMAGES_BUILD.iterate(config): + relative_path = path if isinstance(path, str) else os.path.join(*path) if image in [name, "all"]: found = True tag = tutor_env.render_str(config, tag) - yield (name, path, tag, args) + yield (name, relative_path, tag, args) if not found: raise ImageNotFoundError(image) diff --git a/tutor/commands/local.py b/tutor/commands/local.py index 2b2781d..320edaa 100644 --- a/tutor/commands/local.py +++ b/tutor/commands/local.py @@ -28,6 +28,8 @@ class LocalTaskRunner(compose.ComposeTaskRunner): # pylint: disable=too-few-public-methods class LocalContext(compose.BaseComposeContext): + NAME = "local" + def job_runner(self, config: Config) -> LocalTaskRunner: return LocalTaskRunner(self.root, config) diff --git a/tutor/hooks/catalog.py b/tutor/hooks/catalog.py index 32a77d4..d59fdd0 100644 --- a/tutor/hooks/catalog.py +++ b/tutor/hooks/catalog.py @@ -7,7 +7,7 @@ from __future__ import annotations # The Tutor plugin system is licensed under the terms of the Apache 2.0 license. __license__ = "Apache 2.0" -from typing import Any, Callable, Iterable +from typing import Any, Callable, Iterable, Literal, Union import click @@ -169,8 +169,8 @@ class Filters: #: #: :parameter list[str] hostnames: items from this list are templates that will be #: rendered by the environment. - #: :parameter str project_name: compose project name, such as "local" or "dev". - APP_PUBLIC_HOSTS: Filter[list[str], [str]] = Filter() + #: :parameter str context_name: either "local" or "dev", depending on the calling context. + APP_PUBLIC_HOSTS: Filter[list[str], [Literal["local", "dev"]]] = Filter() #: List of command line interface (CLI) commands. #: @@ -341,18 +341,25 @@ class Filters: #: :parameter list[tuple[str, tuple[str, ...], str, tuple[str, ...]]] tasks: list of ``(name, path, tag, args)`` tuples. #: #: - ``name`` is the name of the image, as in ``tutor images build myimage``. - #: - ``path`` is the relative path to the folder that contains the Dockerfile. + #: - ``path`` is the relative path to the folder that contains the Dockerfile. This can be either a string or a tuple of strings. #: For instance ``("myplugin", "build", "myservice")`` indicates that the template will be read from - #: ``myplugin/build/myservice/Dockerfile`` + #: ``myplugin/build/myservice/Dockerfile``. This argument value would be equivalent to "myplugin/build/myservice". #: - ``tag`` is the Docker tag that will be applied to the image. It will be #: rendered at runtime with the user configuration. Thus, the image tag could #: be ``"{{ DOCKER_REGISTRY }}/myimage:{{ TUTOR_VERSION }}"``. #: - ``args`` is a list of arguments that will be passed to ``docker build ...``. #: :parameter Config config: user configuration. IMAGES_BUILD: Filter[ - list[tuple[str, tuple[str, ...], str, tuple[str, ...]]], [Config] + list[tuple[str, Union[str, tuple[str, ...]], str, tuple[str, ...]]], [Config] ] = Filter() + #: List of image names which must be built prior to launching the platform. These + #: images will be built on launch, in "dev" and "local" mode (but not in Kubernetes). + #: + #: :parameter list[str] names: list of image names. + #: :parameter str context_name: either "local" or "dev", depending on the calling context. + IMAGES_BUILD_REQUIRED: Filter[list[str], [Literal["local", "dev"]]] = Filter() + #: List of host directories to be automatically bind-mounted in Docker images at #: build time. For instance, this is useful to build Docker images using a custom #: repository on the host.