From 27449f4068186472c991ad50ff845d493504e25e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Wed, 20 Apr 2022 22:24:17 +0200 Subject: [PATCH] feat: add `dev/local copyfrom` commands `copyfrom` copies data from a container to the local filesystem. It's similar to bindmount, but less clunky, and more intuitive. Also, it plays along great with `--mount`. Eventually we'll just get rid of the `bindmount` command and the `--volume` option. --- CHANGELOG.md | 1 + docs/dev.rst | 16 ++++++++- tests/commands/test_compose.py | 1 + tests/commands/test_local.py | 48 +++++++++++++++++++++++++++ tutor/commands/compose.py | 59 ++++++++++++++++++++++++++++++---- tutor/commands/k8s.py | 3 +- tutor/commands/local.py | 3 +- tutor/commands/plugins.py | 2 +- 8 files changed, 122 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f1b8502..0b34793 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ Note: Breaking changes between versions are indicated by "💥". ## Unreleased +- [Feature] Introduce `local/dev copyfrom` command to copy contents from a container. - [Bugfix] Fix a race condition that could prevent a newly provisioned LMS container from starting due to a `FileExistsError` when creating data folders. - [Deprecation] Mark `tutor dev runserver` as deprecated in favor of `tutor dev start`. Since `start` now supports bind-mounting and breakpoint debugging, `runserver` is redundant and will be removed in a future release. - [Improvement] Allow breakpoint debugging when attached to a service via `tutor dev start SERVICE`. diff --git a/docs/dev.rst b/docs/dev.rst index 1d22f98..8524d3b 100644 --- a/docs/dev.rst +++ b/docs/dev.rst @@ -139,10 +139,23 @@ So, when should you *not* be using the implicit form? That would be when Tutor d .. note:: Remember to setup your edx-platform repository for development! See :ref:`edx_platform_dev_env`. +Copy files from containers to the local filesystem +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Sometimes, you may want to modify some of the files inside a container for which you don't have a copy on the host. A typical example is when you want to troubleshoot a Python dependency that is installed inside the application virtual environment. In such cases, you want to first copy the contents of the virtual environment from the container to the local filesystem. To that end, Tutor provides the ``tutor dev copyfrom`` command. First, copy the contents of the container folder to the local filesystem:: + + tutor dev copyfrom lms /openedx/venv ~ + +Then, bind-mount that folder back in the container with the ``--mount`` option (described :ref:`above `):: + + tutor dev start --mount lms:~/venv:/openedx/venv lms + +You can then edit the files in ``~/venv`` on your local filesystem and see the changes live in your container. + Bind-mount from the "volumes/" directory ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. warning:: Bind-mounting volumes with the ``bindmount`` command is no longer the default, recommended way of bind-mounting volumes from the host. Instead, see the :ref:`mount option `. +.. warning:: Bind-mounting volumes with the ``bindmount`` command is no longer the default, recommended way of bind-mounting volumes from the host. Instead, see the :ref:`mount option ` and the ``tutor dev/local copyfrom`` commands. Tutor makes it easy to create a bind-mount from an existing container. First, copy the contents of a container directory with the ``bindmount`` command. For instance, to copy the virtual environment of the "lms" container:: @@ -231,6 +244,7 @@ After running all these commands, your edx-platform repository will be ready for If LMS isn't running, this will start it in your terminal. If an LMS container is already running background, this command will stop it, recreate it, and attach your terminal to it. Later, to detach your terminal without stopping the container, just hit ``Ctrl+z``. + XBlock and edx-platform plugin development ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/tests/commands/test_compose.py b/tests/commands/test_compose.py index 74f5610..67862a6 100644 --- a/tests/commands/test_compose.py +++ b/tests/commands/test_compose.py @@ -1,6 +1,7 @@ import unittest from click.exceptions import ClickException + from tutor.commands import compose diff --git a/tests/commands/test_local.py b/tests/commands/test_local.py index bae0edf..40a135c 100644 --- a/tests/commands/test_local.py +++ b/tests/commands/test_local.py @@ -1,4 +1,9 @@ +import os +import tempfile import unittest +from unittest.mock import patch + +from tests.helpers import temporary_root from .base import TestCommandMixin @@ -18,3 +23,46 @@ class LocalTests(unittest.TestCase, TestCommandMixin): result = self.invoke(["local", "upgrade", "--help"]) self.assertIsNone(result.exception) self.assertEqual(0, result.exit_code) + + def test_copyfrom(self) -> None: + with temporary_root() as root: + with tempfile.TemporaryDirectory() as directory: + with patch("tutor.utils.docker_compose") as mock_docker_compose: + self.invoke_in_root(root, ["config", "save"]) + + # Copy to existing directory + result = self.invoke_in_root( + root, ["local", "copyfrom", "lms", "/openedx/venv", directory] + ) + self.assertIsNone(result.exception) + self.assertEqual(0, result.exit_code) + self.assertIn( + f"--volume={directory}:/tmp/mount", + mock_docker_compose.call_args.args, + ) + self.assertIn( + "cp --recursive --preserve /openedx/venv /tmp/mount", + mock_docker_compose.call_args.args, + ) + + # Copy to non-existing directory + result = self.invoke_in_root( + root, + [ + "local", + "copyfrom", + "lms", + "/openedx/venv", + os.path.join(directory, "venv2"), + ], + ) + self.assertIsNone(result.exception) + self.assertEqual(0, result.exit_code) + self.assertIn( + f"--volume={directory}:/tmp/mount", + mock_docker_compose.call_args.args, + ) + self.assertIn( + "cp --recursive --preserve /openedx/venv /tmp/mount/venv2", + mock_docker_compose.call_args.args, + ) diff --git a/tutor/commands/compose.py b/tutor/commands/compose.py index 58896d7..f19f8bf 100644 --- a/tutor/commands/compose.py +++ b/tutor/commands/compose.py @@ -7,12 +7,10 @@ import click from tutor import bindmounts from tutor import config as tutor_config from tutor import env as tutor_env -from tutor import fmt, jobs, utils -from tutor import serialize +from tutor import fmt, hooks, jobs, serialize, utils +from tutor.commands.context import BaseJobContext from tutor.exceptions import TutorError from tutor.types import Config -from tutor.commands.context import BaseJobContext -from tutor import hooks class ComposeJobRunner(jobs.BaseComposeJobRunner): @@ -326,15 +324,16 @@ def run( name="bindmount", help="Copy the contents of a container directory to a ready-to-bind-mount host directory", ) -@click.argument( - "service", -) +@click.argument("service") @click.argument("path") @click.pass_obj def bindmount_command(context: BaseComposeContext, service: str, path: str) -> None: """ This command is made obsolete by the --mount arguments. """ + fmt.echo_alert( + "The 'bindmount' command is deprecated and will be removed in a later release. Use 'copyfrom' instead." + ) config = tutor_config.load(context.root) host_path = bindmounts.create(context.job_runner(config), service, path) fmt.echo_info( @@ -343,6 +342,51 @@ def bindmount_command(context: BaseComposeContext, service: str, path: str) -> N ) +@click.command( + name="copyfrom", + help="Copy files/folders from a container directory to the local filesystem.", +) +@click.argument("service") +@click.argument("container_path") +@click.argument( + "host_path", + type=click.Path(dir_okay=True, file_okay=False, resolve_path=True), +) +@click.pass_obj +def copyfrom( + context: BaseComposeContext, service: str, container_path: str, host_path: str +) -> None: + # Path management + container_root_path = "/tmp/mount" + container_dst_path = container_root_path + if not os.path.exists(host_path): + # Emulate cp semantics, where if the destination path does not exist + # then we copy to its parent and rename to the destination folder + container_dst_path += "/" + os.path.basename(host_path) + host_path = os.path.dirname(host_path) + if not os.path.exists(host_path): + raise TutorError( + f"Cannot create directory {host_path}. No such file or directory." + ) + + # cp/mv commands + command = f"cp --recursive --preserve {container_path} {container_dst_path}" + config = tutor_config.load(context.root) + runner = context.job_runner(config) + runner.docker_compose( + "run", + "--rm", + "--no-deps", + "--user=0", + f"--volume={host_path}:{container_root_path}", + service, + "sh", + "-e", + "-c", + command, + ) + + @click.command( short_help="Run a command in a running container", help=( @@ -490,6 +534,7 @@ def add_commands(command_group: click.Group) -> None: command_group.add_command(settheme) command_group.add_command(dc_command) command_group.add_command(run) + command_group.add_command(copyfrom) command_group.add_command(bindmount_command) command_group.add_command(execute) command_group.add_command(logs) diff --git a/tutor/commands/k8s.py b/tutor/commands/k8s.py index b4f42ce..14d5e42 100644 --- a/tutor/commands/k8s.py +++ b/tutor/commands/k8s.py @@ -6,8 +6,9 @@ import click from tutor import config as tutor_config from tutor import env as tutor_env +from tutor import exceptions, fmt from tutor import interactive as interactive_config -from tutor import exceptions, fmt, jobs, serialize, utils +from tutor import jobs, serialize, utils from tutor.commands.config import save as config_save_command from tutor.commands.context import BaseJobContext from tutor.commands.upgrade.k8s import upgrade_from diff --git a/tutor/commands/local.py b/tutor/commands/local.py index f659e7e..a7848e2 100644 --- a/tutor/commands/local.py +++ b/tutor/commands/local.py @@ -4,8 +4,9 @@ import click from tutor import config as tutor_config from tutor import env as tutor_env -from tutor import exceptions, fmt, utils +from tutor import exceptions, fmt from tutor import interactive as interactive_config +from tutor import utils from tutor.commands import compose from tutor.commands.config import save as config_save_command from tutor.commands.upgrade.local import upgrade_from diff --git a/tutor/commands/plugins.py b/tutor/commands/plugins.py index b15fe51..5438ec1 100644 --- a/tutor/commands/plugins.py +++ b/tutor/commands/plugins.py @@ -1,6 +1,6 @@ import os -import urllib.request import typing as t +import urllib.request import click