diff --git a/changelog.d/20230412_100608_regis_palm.md b/changelog.d/20230412_100608_regis_palm.md index 5d38382..3d6e6c5 100644 --- a/changelog.d/20230412_100608_regis_palm.md +++ b/changelog.d/20230412_100608_regis_palm.md @@ -7,6 +7,8 @@ - `APP` was previously a ContextTemplate, and is now a dictionary of contexts indexed by name. Developers who implemented this context should replace `Contexts.APP(...)` by `Contexts.app(...)`. - Removed the `ENV_PATCH` filter, which was for internal use only anyway. - The `PLUGIN_LOADED` ActionTemplate is now an Action which takes a single argument. (the plugin name) - - 💥[Refactor] We refactored the hooks API further by removing the static hook indexes and the hooks names. As a consequence: - - The syntactic sugar functions from the "filters" and "actions" modules were all removed: `get`, `add*`, `iterate*`, `apply*`, `do*`, etc. + - 💥[Refactor] We refactored the hooks API further by removing the static hook indexes and the hooks names. As a consequence, the syntactic sugar functions from the "filters" and "actions" modules were all removed: `get`, `add*`, `iterate*`, `apply*`, `do*`, etc. - 💥[Deprecation] The obsolete filters `COMMANDS_PRE_INIT` and `COMMANDS_INIT` have been removed. Plugin developers should instead use `CLI_DO_INIT_TASKS` (with suitable priorities). + - 💥[Feature] The "openedx" Docker image is no longer built with docker-compose in development on `tutor dev start`. This used to be the case to make sure that it was always up-to-date, but it introduced a discrepancy in how images were build (`docker compose build` vs `docker build`). As a consequence: + - The "openedx" Docker image in development can be built with `tutor images build openedx-dev`. + - The `tutor dev/local start --skip-build` option is removed. It is replaced by opt-in `--build`. diff --git a/docs/dev.rst b/docs/dev.rst index 684d806..a429711 100644 --- a/docs/dev.rst +++ b/docs/dev.rst @@ -12,14 +12,13 @@ First-time setup Firstly, either :ref:`install Tutor ` (for development against the named releases of Open edX) or :ref:`install Tutor Nightly ` (for development against Open edX's master branches). -Then, optionally, tell Tutor to use a local fork of edx-platform. In that case you will need to rebuild the "openedx" Docker image:: +Then, optionally, tell Tutor to use a local fork of edx-platform.:: tutor config save --append MOUNTS=./edx-platform - tutor images build openedx -Then, run one of the following in order to launch the developer platform setup process:: +Then, build the "openedx" Docker image for development and launch the developer platfornm setup process:: - # To use the edx-platform repository that is built into the image, run: + tutor images build openedx-dev tutor dev launch This will perform several tasks. It will: @@ -130,7 +129,7 @@ The ``openedx-dev`` Docker image is based on the same ``openedx`` image used by If you are using a custom ``openedx`` image, then you will need to rebuild ``openedx-dev`` every time you modify ``openedx``. To so, run:: - tutor dev dc build lms + tutor images build openedx-dev .. _bind_mounts: diff --git a/docs/tutorials/arm64.rst b/docs/tutorials/arm64.rst index 63c8a92..60d73a8 100644 --- a/docs/tutorials/arm64.rst +++ b/docs/tutorials/arm64.rst @@ -26,10 +26,9 @@ Then, build the "openedx" and "permissions" images:: tutor images build openedx permissions -.. TODO we don't want this instruction anymore +If you want to use Tutor as an Open edX development environment, you should also build the development image:: -If you want to use Tutor as an Open edX development environment, you should also build the development images:: - tutor dev dc build lms + tutor images build openedx-dev From this point on, use Tutor as normal. For example, start Open edX and run migrations with:: diff --git a/tutor/commands/compose.py b/tutor/commands/compose.py index c7a4f6d..ab0bff1 100644 --- a/tutor/commands/compose.py +++ b/tutor/commands/compose.py @@ -1,15 +1,21 @@ from __future__ import annotations import os +import typing as t import click from tutor import bindmount from tutor import config as tutor_config from tutor import env as tutor_env -from tutor import hooks, utils +from tutor import fmt, hooks +from tutor import interactive as interactive_config +from tutor import utils from tutor.commands import jobs +from tutor.commands.config import save as config_save_command from tutor.commands.context import BaseTaskContext +from tutor.commands.upgrade import OPENEDX_RELEASE_NAMES +from tutor.commands.upgrade.compose import upgrade_from from tutor.core.hooks import Filter # pylint: disable=unused-import from tutor.exceptions import TutorError from tutor.tasks import BaseComposeTaskRunner @@ -70,22 +76,143 @@ class BaseComposeContext(BaseTaskContext): raise NotImplementedError +@click.command(help="Configure and run Open edX from scratch") +@click.option("-I", "--non-interactive", is_flag=True, help="Run non-interactively") +@click.option("-p", "--pullimages", is_flag=True, help="Update docker images") +@click.pass_context +def launch( + context: click.Context, + non_interactive: bool, + pullimages: bool, +) -> None: + utils.warn_macos_docker_memory() + interactive_upgrade(context, not non_interactive) + + click.echo(fmt.title("Stopping any existing platform")) + context.invoke(stop) + + if pullimages: + click.echo(fmt.title("Docker image updates")) + context.invoke(dc_command, command="pull") + + click.echo(fmt.title("Starting the platform in detached mode")) + context.invoke(start, detach=True) + + click.echo(fmt.title("Database creation and migrations")) + context.invoke(do.commands["init"]) + + config = tutor_config.load(context.obj.root) + project_name = context.obj.job_runner(config).project_name + + # Print the urls of the user-facing apps + public_app_hosts = "" + for host in hooks.Filters.APP_PUBLIC_HOSTS.iterate(project_name): + public_app_host = tutor_env.render_str( + config, "{% if ENABLE_HTTPS %}https{% else %}http{% endif %}://" + host + ) + public_app_hosts += f" {public_app_host}\n" + if public_app_hosts: + fmt.echo_info( + f"""The platform is now running and can be accessed at the following urls: + +{public_app_hosts}""" + ) + + +def interactive_upgrade(context: click.Context, interactive: bool) -> None: + """ + Piece of code that is only used in launch. + """ + run_upgrade_from_release = tutor_env.should_upgrade_from_release(context.obj.root) + if run_upgrade_from_release is not None: + click.echo(fmt.title("Upgrading from an older release")) + if interactive: + to_release = tutor_env.get_current_open_edx_release_name() + question = f"""You are about to upgrade your Open edX platform from {run_upgrade_from_release.capitalize()} to {to_release.capitalize()} + +It is strongly recommended to make a backup before upgrading. To do so, run: + + tutor local stop # or 'tutor dev stop' in development + sudo rsync -avr "$(tutor config printroot)"/ /tmp/tutor-backup/ + +In case of problem, to restore your backup you will then have to run: sudo rsync -avr /tmp/tutor-backup/ "$(tutor config printroot)"/ + +Are you sure you want to continue?""" + click.confirm( + fmt.question(question), default=True, abort=True, prompt_suffix=" " + ) + context.invoke( + upgrade, + from_release=run_upgrade_from_release, + ) + + click.echo(fmt.title("Interactive platform configuration")) + config = tutor_config.load_minimal(context.obj.root) + if interactive: + interactive_config.ask_questions(config) + tutor_config.save_config_file(context.obj.root, config) + config = tutor_config.load_full(context.obj.root) + tutor_env.save(context.obj.root, config) + + if run_upgrade_from_release and interactive: + question = f"""Your platform is being upgraded from {run_upgrade_from_release.capitalize()}. + +If you run custom Docker images, you must rebuild them now by running the following command in a different shell: + + tutor images build all # list your custom images here + +See the documentation for more information: + + https://docs.tutor.overhang.io/install.html#upgrading-to-a-new-open-edx-release + +Press enter when you are ready to continue""" + click.confirm( + fmt.question(question), default=True, abort=True, prompt_suffix=" " + ) + + +@click.command( + short_help="Perform release-specific upgrade tasks", + help="Perform release-specific upgrade tasks. To perform a full upgrade remember to run `launch`.", +) +@click.option( + "--from", + "from_release", + type=click.Choice(OPENEDX_RELEASE_NAMES), +) +@click.pass_context +def upgrade(context: click.Context, from_release: t.Optional[str]) -> None: + fmt.echo_alert( + "This command only performs a partial upgrade of your Open edX platform. " + "To perform a full upgrade, you should run `tutor local launch` (or `tutor dev launch` " + "in development)." + ) + if from_release is None: + from_release = tutor_env.get_env_release(context.obj.root) + if from_release is None: + fmt.echo_info("Your environment is already up-to-date") + else: + upgrade_from(context, from_release) + # We update the environment to update the version + context.invoke(config_save_command) + + @click.command( short_help="Run all or a selection of services.", help="Run all or a selection of services. Docker images will be rebuilt where necessary.", ) -@click.option("--skip-build", is_flag=True, help="Skip image building") +@click.option("--build", is_flag=True, help="Build images on start") @click.option("-d", "--detach", is_flag=True, help="Start in daemon mode") @click.argument("services", metavar="service", nargs=-1) @click.pass_obj def start( context: BaseComposeContext, - skip_build: bool, + build: bool, detach: bool, services: list[str], ) -> None: command = ["up", "--remove-orphans"] - if not skip_build: + if build: command.append("--build") if detach: command.append("-d") @@ -293,10 +420,21 @@ def _mount_edx_platform( return volumes +def _edx_platform_public_hosts(hosts: list[str], project_name: str) -> list[str]: + edx_platform_hosts = ["{{ LMS_HOST }}", "{{ CMS_HOST }}"] + if project_name == "dev": + edx_platform_hosts[0] += ":8000" + edx_platform_hosts[1] += ":8001" + hosts += edx_platform_hosts + return hosts + + hooks.Filters.ENV_TEMPLATE_VARIABLES.add_item(("iter_mounts", bindmount.iter_mounts)) def add_commands(command_group: click.Group) -> None: + command_group.add_command(launch) + command_group.add_command(upgrade) command_group.add_command(start) command_group.add_command(stop) command_group.add_command(restart) diff --git a/tutor/commands/dev.py b/tutor/commands/dev.py index f5f5df2..a60d7e7 100644 --- a/tutor/commands/dev.py +++ b/tutor/commands/dev.py @@ -2,11 +2,8 @@ from __future__ import annotations import click -from tutor import config as tutor_config from tutor import env as tutor_env -from tutor import fmt, hooks -from tutor import interactive as interactive_config -from tutor import utils +from tutor import hooks from tutor.commands import compose from tutor.types import Config, get_typed @@ -43,51 +40,6 @@ def dev(context: click.Context) -> None: context.obj = DevContext(context.obj.root) -@click.command(help="Configure and run Open edX from scratch, for development") -@click.option("-I", "--non-interactive", is_flag=True, help="Run non-interactively") -@click.option("-p", "--pullimages", is_flag=True, help="Update docker images") -@click.pass_context -def launch( - context: click.Context, - non_interactive: bool, - pullimages: bool, -) -> None: - utils.warn_macos_docker_memory() - - click.echo(fmt.title("Interactive platform configuration")) - config = tutor_config.load_minimal(context.obj.root) - if not non_interactive: - interactive_config.ask_questions(config, run_for_prod=False) - tutor_config.save_config_file(context.obj.root, config) - config = tutor_config.load_full(context.obj.root) - tutor_env.save(context.obj.root, config) - - click.echo(fmt.title("Stopping any existing platform")) - context.invoke(compose.stop) - - if pullimages: - click.echo(fmt.title("Docker image updates")) - context.invoke(compose.dc_command, command="pull") - - click.echo(fmt.title("Starting the platform in detached mode")) - context.invoke(compose.start, detach=True) - - click.echo(fmt.title("Database creation and migrations")) - context.invoke(compose.do.commands["init"]) - - fmt.echo_info( - """The Open edX platform is now running in detached mode -Your Open edX platform is ready and can be accessed at the following urls: - {http}://{lms_host}:8000 - {http}://{cms_host}:8001 - """.format( - http="https" if config["ENABLE_HTTPS"] else "http", - lms_host=config["LMS_HOST"], - cms_host=config["CMS_HOST"], - ) - ) - - @hooks.Actions.COMPOSE_PROJECT_STARTED.add() def _stop_on_local_start(root: str, config: Config, project_name: str) -> None: """ @@ -99,5 +51,4 @@ def _stop_on_local_start(root: str, config: Config, project_name: str) -> None: runner.docker_compose("stop") -dev.add_command(launch) compose.add_commands(dev) diff --git a/tutor/commands/images.py b/tutor/commands/images.py index ca5e719..ba31deb 100644 --- a/tutor/commands/images.py +++ b/tutor/commands/images.py @@ -34,6 +34,20 @@ def _add_core_images_to_build( for image in BASE_IMAGE_NAMES: tag = images.get_tag(config, image) build_images.append((image, ("build", image), tag, ())) + + # Build openedx-dev image + build_images.append( + ( + "openedx-dev", + ("build", "openedx"), + images.get_tag(config, "openedx-dev"), + ( + "--target=development", + f"--build-arg=APP_USER_ID={utils.get_user_id() or 1000}", + ), + ) + ) + return build_images @@ -208,6 +222,7 @@ def _mount_edx_platform( """ if os.path.basename(path) == "edx-platform": volumes.append(("openedx", "edx-platform")) + volumes.append(("openedx-dev", "edx-platform")) return volumes diff --git a/tutor/commands/local.py b/tutor/commands/local.py index 9ead815..2b2781d 100644 --- a/tutor/commands/local.py +++ b/tutor/commands/local.py @@ -1,18 +1,10 @@ from __future__ import annotations -import typing as t - import click -from tutor import config as tutor_config from tutor import env as tutor_env -from tutor import fmt, hooks -from tutor import interactive as interactive_config -from tutor import utils +from tutor import hooks from tutor.commands import compose -from tutor.commands.config import save as config_save_command -from tutor.commands.upgrade import OPENEDX_RELEASE_NAMES -from tutor.commands.upgrade.local import upgrade_from from tutor.types import Config, get_typed @@ -46,114 +38,6 @@ def local(context: click.Context) -> None: context.obj = LocalContext(context.obj.root) -@click.command(help="Configure and run Open edX from scratch") -@click.option("-I", "--non-interactive", is_flag=True, help="Run non-interactively") -@click.option("-p", "--pullimages", is_flag=True, help="Update docker images") -@click.pass_context -def launch( - context: click.Context, - non_interactive: bool, - pullimages: bool, -) -> None: - utils.warn_macos_docker_memory() - - run_upgrade_from_release = tutor_env.should_upgrade_from_release(context.obj.root) - if run_upgrade_from_release is not None: - click.echo(fmt.title("Upgrading from an older release")) - if not non_interactive: - to_release = tutor_env.get_current_open_edx_release_name() - question = f"""You are about to upgrade your Open edX platform from {run_upgrade_from_release.capitalize()} to {to_release.capitalize()} - -It is strongly recommended to make a backup before upgrading. To do so, run: - - tutor local stop - sudo rsync -avr "$(tutor config printroot)"/ /tmp/tutor-backup/ - -In case of problem, to restore your backup you will then have to run: sudo rsync -avr /tmp/tutor-backup/ "$(tutor config printroot)"/ - -Are you sure you want to continue?""" - click.confirm( - fmt.question(question), default=True, abort=True, prompt_suffix=" " - ) - context.invoke( - upgrade, - from_release=run_upgrade_from_release, - ) - - click.echo(fmt.title("Interactive platform configuration")) - config = tutor_config.load_minimal(context.obj.root) - if not non_interactive: - interactive_config.ask_questions(config) - tutor_config.save_config_file(context.obj.root, config) - config = tutor_config.load_full(context.obj.root) - tutor_env.save(context.obj.root, config) - - if run_upgrade_from_release and not non_interactive: - question = f"""Your platform is being upgraded from {run_upgrade_from_release.capitalize()}. - -If you run custom Docker images, you must rebuild them now by running the following command in a different shell: - - tutor images build all # list your custom images here - -See the documentation for more information: - - https://docs.tutor.overhang.io/install.html#upgrading-to-a-new-open-edx-release - -Press enter when you are ready to continue""" - click.confirm( - fmt.question(question), default=True, abort=True, prompt_suffix=" " - ) - - click.echo(fmt.title("Stopping any existing platform")) - context.invoke(compose.stop) - if pullimages: - click.echo(fmt.title("Docker image updates")) - context.invoke(compose.dc_command, command="pull") - click.echo(fmt.title("Starting the platform in detached mode")) - context.invoke(compose.start, detach=True) - click.echo(fmt.title("Database creation and migrations")) - context.invoke(compose.do.commands["init"]) - - config = tutor_config.load(context.obj.root) - fmt.echo_info( - """The Open edX platform is now running in detached mode -Your Open edX platform is ready and can be accessed at the following urls: - - {http}://{lms_host} - {http}://{cms_host} - """.format( - http="https" if config["ENABLE_HTTPS"] else "http", - lms_host=config["LMS_HOST"], - cms_host=config["CMS_HOST"], - ) - ) - - -@click.command( - short_help="Perform release-specific upgrade tasks", - help="Perform release-specific upgrade tasks. To perform a full upgrade remember to run `launch`.", -) -@click.option( - "--from", - "from_release", - type=click.Choice(OPENEDX_RELEASE_NAMES), -) -@click.pass_context -def upgrade(context: click.Context, from_release: t.Optional[str]) -> None: - fmt.echo_alert( - "This command only performs a partial upgrade of your Open edX platform. " - "To perform a full upgrade, you should run `tutor local launch`." - ) - if from_release is None: - from_release = tutor_env.get_env_release(context.obj.root) - if from_release is None: - fmt.echo_info("Your environment is already up-to-date") - else: - upgrade_from(context, from_release) - # We update the environment to update the version - context.invoke(config_save_command) - - @hooks.Actions.COMPOSE_PROJECT_STARTED.add() def _stop_on_dev_start(root: str, config: Config, project_name: str) -> None: """ @@ -165,6 +49,4 @@ def _stop_on_dev_start(root: str, config: Config, project_name: str) -> None: runner.docker_compose("stop") -local.add_command(launch) -local.add_command(upgrade) compose.add_commands(local) diff --git a/tutor/commands/upgrade/local.py b/tutor/commands/upgrade/compose.py similarity index 100% rename from tutor/commands/upgrade/local.py rename to tutor/commands/upgrade/compose.py diff --git a/tutor/core/hooks/actions.py b/tutor/core/hooks/actions.py index 924c5ae..e081244 100644 --- a/tutor/core/hooks/actions.py +++ b/tutor/core/hooks/actions.py @@ -9,7 +9,6 @@ from weakref import WeakSet from typing_extensions import ParamSpec - from . import priorities from .contexts import Contextualized diff --git a/tutor/hooks/catalog.py b/tutor/hooks/catalog.py index 2a1e55f..32a77d4 100644 --- a/tutor/hooks/catalog.py +++ b/tutor/hooks/catalog.py @@ -163,6 +163,15 @@ class Filters: :py:class:`tutor.core.hooks.Filter` API. """ + #: Hostnames of user-facing applications. + #: + #: So far this filter is only used to inform the user of application urls after they have run ``launch``. + #: + #: :parameter list[str] hostnames: items from this list are templates that will be + #: rendered by the environment. + #: :parameter str project_name: compose project name, such as "local" or "dev". + APP_PUBLIC_HOSTS: Filter[list[str], [str]] = Filter() + #: List of command line interface (CLI) commands. #: #: :parameter list commands: commands are instances of ``click.Command``. They will diff --git a/tutor/templates/dev/docker-compose.yml b/tutor/templates/dev/docker-compose.yml index 96d045e..4b8cd38 100644 --- a/tutor/templates/dev/docker-compose.yml +++ b/tutor/templates/dev/docker-compose.yml @@ -3,12 +3,6 @@ version: "{{ DOCKER_COMPOSE_VERSION }}" x-openedx-service: &openedx-service image: {{ DOCKER_IMAGE_OPENEDX_DEV }} - build: - context: ../build/openedx/ - target: development - args: - # Note that we never build the openedx-dev image with root user ID, as it would simply fail. - APP_USER_ID: "{{ HOST_USER_ID or 1000 }}" stdin_open: true tty: true volumes: