diff --git a/CHANGELOG.md b/CHANGELOG.md index f1b8502..376da6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Note: Breaking changes between versions are indicated by "💥". ## Unreleased +- [Improvement] Add the `COMPOSE_PROJECT_STARTED` action and run `dev stop` on `local start` (and vice versa). +- [Feature] Introduce `local/dev copyfrom` command to copy contents from a container. - [Bugfix] Fix a race condition that could prevent a newly provisioned LMS container from starting due to a `FileExistsError` when creating data folders. - [Deprecation] Mark `tutor dev runserver` as deprecated in favor of `tutor dev start`. Since `start` now supports bind-mounting and breakpoint debugging, `runserver` is redundant and will be removed in a future release. - [Improvement] Allow breakpoint debugging when attached to a service via `tutor dev start SERVICE`. diff --git a/docs/dev.rst b/docs/dev.rst index 1d22f98..8524d3b 100644 --- a/docs/dev.rst +++ b/docs/dev.rst @@ -139,10 +139,23 @@ So, when should you *not* be using the implicit form? That would be when Tutor d .. note:: Remember to setup your edx-platform repository for development! See :ref:`edx_platform_dev_env`. +Copy files from containers to the local filesystem +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Sometimes, you may want to modify some of the files inside a container for which you don't have a copy on the host. A typical example is when you want to troubleshoot a Python dependency that is installed inside the application virtual environment. In such cases, you want to first copy the contents of the virtual environment from the container to the local filesystem. To that end, Tutor provides the ``tutor dev copyfrom`` command. First, copy the contents of the container folder to the local filesystem:: + + tutor dev copyfrom lms /openedx/venv ~ + +Then, bind-mount that folder back in the container with the ``--mount`` option (described :ref:`above `):: + + tutor dev start --mount lms:~/venv:/openedx/venv lms + +You can then edit the files in ``~/venv`` on your local filesystem and see the changes live in your container. + Bind-mount from the "volumes/" directory ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. warning:: Bind-mounting volumes with the ``bindmount`` command is no longer the default, recommended way of bind-mounting volumes from the host. Instead, see the :ref:`mount option `. +.. warning:: Bind-mounting volumes with the ``bindmount`` command is no longer the default, recommended way of bind-mounting volumes from the host. Instead, see the :ref:`mount option ` and the ``tutor dev/local copyfrom`` commands. Tutor makes it easy to create a bind-mount from an existing container. First, copy the contents of a container directory with the ``bindmount`` command. For instance, to copy the virtual environment of the "lms" container:: @@ -231,6 +244,7 @@ After running all these commands, your edx-platform repository will be ready for If LMS isn't running, this will start it in your terminal. If an LMS container is already running background, this command will stop it, recreate it, and attach your terminal to it. Later, to detach your terminal without stopping the container, just hit ``Ctrl+z``. + XBlock and edx-platform plugin development ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/tutorials/arm64.rst b/docs/tutorials/arm64.rst index 80be43f..b5b3099 100644 --- a/docs/tutorials/arm64.rst +++ b/docs/tutorials/arm64.rst @@ -52,7 +52,6 @@ From this point on, use Tutor as normal. For example, start Open edX and run mig Or for a development environment:: - tutor local stop tutor dev start -d tutor dev init diff --git a/tests/commands/test_compose.py b/tests/commands/test_compose.py index 74f5610..67862a6 100644 --- a/tests/commands/test_compose.py +++ b/tests/commands/test_compose.py @@ -1,6 +1,7 @@ import unittest from click.exceptions import ClickException + from tutor.commands import compose diff --git a/tests/commands/test_local.py b/tests/commands/test_local.py index bae0edf..abb8443 100644 --- a/tests/commands/test_local.py +++ b/tests/commands/test_local.py @@ -1,4 +1,9 @@ +import os +import tempfile import unittest +from unittest.mock import patch + +from tests.helpers import temporary_root from .base import TestCommandMixin @@ -18,3 +23,46 @@ class LocalTests(unittest.TestCase, TestCommandMixin): result = self.invoke(["local", "upgrade", "--help"]) self.assertIsNone(result.exception) self.assertEqual(0, result.exit_code) + + def test_copyfrom(self) -> None: + with temporary_root() as root: + with tempfile.TemporaryDirectory() as directory: + with patch("tutor.utils.docker_compose") as mock_docker_compose: + self.invoke_in_root(root, ["config", "save"]) + + # Copy to existing directory + result = self.invoke_in_root( + root, ["local", "copyfrom", "lms", "/openedx/venv", directory] + ) + self.assertIsNone(result.exception) + self.assertEqual(0, result.exit_code) + self.assertIn( + f"--volume={directory}:/tmp/mount", + mock_docker_compose.call_args[0], + ) + self.assertIn( + "cp --recursive --preserve /openedx/venv /tmp/mount", + mock_docker_compose.call_args[0], + ) + + # Copy to non-existing directory + result = self.invoke_in_root( + root, + [ + "local", + "copyfrom", + "lms", + "/openedx/venv", + os.path.join(directory, "venv2"), + ], + ) + self.assertIsNone(result.exception) + self.assertEqual(0, result.exit_code) + self.assertIn( + f"--volume={directory}:/tmp/mount", + mock_docker_compose.call_args[0], + ) + self.assertIn( + "cp --recursive --preserve /openedx/venv /tmp/mount/venv2", + mock_docker_compose.call_args[0], + ) diff --git a/tutor/commands/compose.py b/tutor/commands/compose.py index 58896d7..ea2c0a3 100644 --- a/tutor/commands/compose.py +++ b/tutor/commands/compose.py @@ -7,12 +7,10 @@ import click from tutor import bindmounts from tutor import config as tutor_config from tutor import env as tutor_env -from tutor import fmt, jobs, utils -from tutor import serialize +from tutor import fmt, hooks, jobs, serialize, utils +from tutor.commands.context import BaseJobContext from tutor.exceptions import TutorError from tutor.types import Config -from tutor.commands.context import BaseJobContext -from tutor import hooks class ComposeJobRunner(jobs.BaseComposeJobRunner): @@ -26,6 +24,12 @@ class ComposeJobRunner(jobs.BaseComposeJobRunner): """ Run docker-compose with the right yml files. """ + if "start" in command or "up" in command or "restart" in command: + # Note that we don't trigger the action on "run". That's because we + # don't want to trigger the action for every initialization script. + hooks.Actions.COMPOSE_PROJECT_STARTED.do( + self.root, self.config, self.project_name + ) self.__update_docker_compose_tmp() args = [] for docker_compose_path in self.docker_compose_files: @@ -326,15 +330,16 @@ def run( name="bindmount", help="Copy the contents of a container directory to a ready-to-bind-mount host directory", ) -@click.argument( - "service", -) +@click.argument("service") @click.argument("path") @click.pass_obj def bindmount_command(context: BaseComposeContext, service: str, path: str) -> None: """ This command is made obsolete by the --mount arguments. """ + fmt.echo_alert( + "The 'bindmount' command is deprecated and will be removed in a later release. Use 'copyfrom' instead." + ) config = tutor_config.load(context.root) host_path = bindmounts.create(context.job_runner(config), service, path) fmt.echo_info( @@ -343,6 +348,51 @@ def bindmount_command(context: BaseComposeContext, service: str, path: str) -> N ) +@click.command( + name="copyfrom", + help="Copy files/folders from a container directory to the local filesystem.", +) +@click.argument("service") +@click.argument("container_path") +@click.argument( + "host_path", + type=click.Path(dir_okay=True, file_okay=False, resolve_path=True), +) +@click.pass_obj +def copyfrom( + context: BaseComposeContext, service: str, container_path: str, host_path: str +) -> None: + # Path management + container_root_path = "/tmp/mount" + container_dst_path = container_root_path + if not os.path.exists(host_path): + # Emulate cp semantics, where if the destination path does not exist + # then we copy to its parent and rename to the destination folder + container_dst_path += "/" + os.path.basename(host_path) + host_path = os.path.dirname(host_path) + if not os.path.exists(host_path): + raise TutorError( + f"Cannot create directory {host_path}. No such file or directory." + ) + + # cp/mv commands + command = f"cp --recursive --preserve {container_path} {container_dst_path}" + config = tutor_config.load(context.root) + runner = context.job_runner(config) + runner.docker_compose( + "run", + "--rm", + "--no-deps", + "--user=0", + f"--volume={host_path}:{container_root_path}", + service, + "sh", + "-e", + "-c", + command, + ) + + @click.command( short_help="Run a command in a running container", help=( @@ -490,6 +540,7 @@ def add_commands(command_group: click.Group) -> None: command_group.add_command(settheme) command_group.add_command(dc_command) command_group.add_command(run) + command_group.add_command(copyfrom) command_group.add_command(bindmount_command) command_group.add_command(execute) command_group.add_command(logs) diff --git a/tutor/commands/dev.py b/tutor/commands/dev.py index 657aeef..a3778cb 100644 --- a/tutor/commands/dev.py +++ b/tutor/commands/dev.py @@ -2,13 +2,13 @@ import typing as t import click -from .. import config as tutor_config -from .. import env as tutor_env -from .. import exceptions, fmt -from .. import interactive as interactive_config -from .. import utils -from ..types import Config, get_typed -from . import compose +from tutor import config as tutor_config +from tutor import env as tutor_env +from tutor import exceptions, fmt, hooks +from tutor import interactive as interactive_config +from tutor import utils +from tutor.commands import compose +from tutor.types import Config, get_typed class DevJobRunner(compose.ComposeJobRunner): @@ -128,6 +128,17 @@ def runserver( context.invoke(compose.run, mounts=mounts, args=args) +@hooks.Actions.COMPOSE_PROJECT_STARTED.add() +def _stop_on_local_start(root: str, config: Config, project_name: str) -> None: + """ + Stop the dev platform as soon as a platform with a different project name is + started. + """ + runner = DevJobRunner(root, config) + if project_name != runner.project_name: + runner.docker_compose("stop") + + dev.add_command(quickstart) dev.add_command(runserver) compose.add_commands(dev) diff --git a/tutor/commands/k8s.py b/tutor/commands/k8s.py index b4f42ce..14d5e42 100644 --- a/tutor/commands/k8s.py +++ b/tutor/commands/k8s.py @@ -6,8 +6,9 @@ import click from tutor import config as tutor_config from tutor import env as tutor_env +from tutor import exceptions, fmt from tutor import interactive as interactive_config -from tutor import exceptions, fmt, jobs, serialize, utils +from tutor import jobs, serialize, utils from tutor.commands.config import save as config_save_command from tutor.commands.context import BaseJobContext from tutor.commands.upgrade.k8s import upgrade_from diff --git a/tutor/commands/local.py b/tutor/commands/local.py index f659e7e..f731049 100644 --- a/tutor/commands/local.py +++ b/tutor/commands/local.py @@ -4,8 +4,9 @@ import click from tutor import config as tutor_config from tutor import env as tutor_env -from tutor import exceptions, fmt, utils +from tutor import exceptions, fmt, hooks from tutor import interactive as interactive_config +from tutor import utils from tutor.commands import compose from tutor.commands.config import save as config_save_command from tutor.commands.upgrade.local import upgrade_from @@ -165,6 +166,17 @@ def upgrade(context: click.Context, from_release: t.Optional[str]) -> None: context.invoke(config_save_command) +@hooks.Actions.COMPOSE_PROJECT_STARTED.add() +def _stop_on_dev_start(root: str, config: Config, project_name: str) -> None: + """ + Stop the local platform as soon as a platform with a different project name is + started. + """ + runner = LocalJobRunner(root, config) + if project_name != runner.project_name: + runner.docker_compose("stop") + + local.add_command(quickstart) local.add_command(upgrade) compose.add_commands(local) diff --git a/tutor/commands/plugins.py b/tutor/commands/plugins.py index b15fe51..5438ec1 100644 --- a/tutor/commands/plugins.py +++ b/tutor/commands/plugins.py @@ -1,6 +1,6 @@ import os -import urllib.request import typing as t +import urllib.request import click diff --git a/tutor/hooks/consts.py b/tutor/hooks/consts.py index 929f95f..f017944 100644 --- a/tutor/hooks/consts.py +++ b/tutor/hooks/consts.py @@ -25,6 +25,13 @@ class Actions: # Do stuff here """ + #: Triggered whenever a "docker-compose start", "up" or "restart" command is executed. + #: + #: :parameter: str root: project root. + #: :parameter: dict[str, ...] config: project configuration. + #: :parameter: str name: docker-compose project name. + COMPOSE_PROJECT_STARTED = actions.get("compose:project:started") + #: Called whenever the core project is ready to run. This action is called as soon #: as possible. This is the right time to discover plugins, for instance. In #: particular, we auto-discover the following plugins: