Merge remote-tracking branch 'origin/master' into nightly

CHANGELOG.md

@@ -4,6 +4,8 @@ Note: Breaking changes between versions are indicated by "💥".
 
 ## Unreleased
 
+- [Improvement] Add the `COMPOSE_PROJECT_STARTED` action and run `dev stop` on `local start` (and vice versa).
+- [Feature] Introduce `local/dev copyfrom` command to copy contents from a container.
 - [Bugfix] Fix a race condition that could prevent a newly provisioned LMS container from starting due to a `FileExistsError` when creating data folders.
 - [Deprecation] Mark `tutor dev runserver` as deprecated in favor of `tutor dev start`. Since `start` now supports bind-mounting and breakpoint debugging, `runserver` is redundant and will be removed in a future release.
 - [Improvement] Allow breakpoint debugging when attached to a service via `tutor dev start SERVICE`.

docs/dev.rst

@@ -139,10 +139,23 @@ So, when should you *not* be using the implicit form? That would be when Tutor d
 
 .. note:: Remember to setup your edx-platform repository for development! See :ref:`edx_platform_dev_env`.
 
+Copy files from containers to the local filesystem
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Sometimes, you may want to modify some of the files inside a container for which you don't have a copy on the host. A typical example is when you want to troubleshoot a Python dependency that is installed inside the application virtual environment. In such cases, you want to first copy the contents of the virtual environment from the container to the local filesystem. To that end, Tutor provides the ``tutor dev copyfrom`` command. First, copy the contents of the container folder to the local filesystem::
+
+    tutor dev copyfrom lms /openedx/venv ~
+
+Then, bind-mount that folder back in the container with the ``--mount`` option (described :ref:`above <mount_option>`)::
+
+    tutor dev start --mount lms:~/venv:/openedx/venv lms
+
+You can then edit the files in ``~/venv`` on your local filesystem and see the changes live in your container.
+
 Bind-mount from the "volumes/" directory
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. warning:: Bind-mounting volumes with the ``bindmount`` command is no longer the default, recommended way of bind-mounting volumes from the host. Instead, see the :ref:`mount option <mount_option>`.
+.. warning:: Bind-mounting volumes with the ``bindmount`` command is no longer the default, recommended way of bind-mounting volumes from the host. Instead, see the :ref:`mount option <mount_option>` and the ``tutor dev/local copyfrom`` commands.
 
 Tutor makes it easy to create a bind-mount from an existing container. First, copy the contents of a container directory with the ``bindmount`` command. For instance, to copy the virtual environment of the "lms" container::
 
@@ -231,6 +244,7 @@ After running all these commands, your edx-platform repository will be ready for
 
 If LMS isn't running, this will start it in your terminal. If an LMS container is already running background, this command will stop it, recreate it, and attach your terminal to it. Later, to detach your terminal without stopping the container, just hit ``Ctrl+z``.
 
+
 XBlock and edx-platform plugin development
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

docs/tutorials/arm64.rst

@@ -52,7 +52,6 @@ From this point on, use Tutor as normal. For example, start Open edX and run mig
 
 Or for a development environment::
 
-    tutor local stop
     tutor dev start -d
     tutor dev init
 

tests/commands/test_compose.py

@@ -1,6 +1,7 @@
 import unittest
 
 from click.exceptions import ClickException
+
 from tutor.commands import compose
 
 

tests/commands/test_local.py

@@ -1,4 +1,9 @@
+import os
+import tempfile
 import unittest
+from unittest.mock import patch
+
+from tests.helpers import temporary_root
 
 from .base import TestCommandMixin
 
@@ -18,3 +23,46 @@ class LocalTests(unittest.TestCase, TestCommandMixin):
         result = self.invoke(["local", "upgrade", "--help"])
         self.assertIsNone(result.exception)
         self.assertEqual(0, result.exit_code)
+
+    def test_copyfrom(self) -> None:
+        with temporary_root() as root:
+            with tempfile.TemporaryDirectory() as directory:
+                with patch("tutor.utils.docker_compose") as mock_docker_compose:
+                    self.invoke_in_root(root, ["config", "save"])
+
+                    # Copy to existing directory
+                    result = self.invoke_in_root(
+                        root, ["local", "copyfrom", "lms", "/openedx/venv", directory]
+                    )
+                    self.assertIsNone(result.exception)
+                    self.assertEqual(0, result.exit_code)
+                    self.assertIn(
+                        f"--volume={directory}:/tmp/mount",
+                        mock_docker_compose.call_args[0],
+                    )
+                    self.assertIn(
+                        "cp --recursive --preserve /openedx/venv /tmp/mount",
+                        mock_docker_compose.call_args[0],
+                    )
+
+                    # Copy to non-existing directory
+                    result = self.invoke_in_root(
+                        root,
+                        [
+                            "local",
+                            "copyfrom",
+                            "lms",
+                            "/openedx/venv",
+                            os.path.join(directory, "venv2"),
+                        ],
+                    )
+                    self.assertIsNone(result.exception)
+                    self.assertEqual(0, result.exit_code)
+                    self.assertIn(
+                        f"--volume={directory}:/tmp/mount",
+                        mock_docker_compose.call_args[0],
+                    )
+                    self.assertIn(
+                        "cp --recursive --preserve /openedx/venv /tmp/mount/venv2",
+                        mock_docker_compose.call_args[0],
+                    )

tutor/commands/compose.py

@@ -7,12 +7,10 @@ import click
 from tutor import bindmounts
 from tutor import config as tutor_config
 from tutor import env as tutor_env
-from tutor import fmt, jobs, utils
-from tutor import serialize
+from tutor import fmt, hooks, jobs, serialize, utils
+from tutor.commands.context import BaseJobContext
 from tutor.exceptions import TutorError
 from tutor.types import Config
-from tutor.commands.context import BaseJobContext
-from tutor import hooks
 
 
 class ComposeJobRunner(jobs.BaseComposeJobRunner):
@@ -26,6 +24,12 @@ class ComposeJobRunner(jobs.BaseComposeJobRunner):
         """
         Run docker-compose with the right yml files.
         """
+        if "start" in command or "up" in command or "restart" in command:
+            # Note that we don't trigger the action on "run". That's because we
+            # don't want to trigger the action for every initialization script.
+            hooks.Actions.COMPOSE_PROJECT_STARTED.do(
+                self.root, self.config, self.project_name
+            )
         self.__update_docker_compose_tmp()
         args = []
         for docker_compose_path in self.docker_compose_files:
@@ -326,15 +330,16 @@ def run(
     name="bindmount",
     help="Copy the contents of a container directory to a ready-to-bind-mount host directory",
 )
-@click.argument(
-    "service",
-)
+@click.argument("service")
 @click.argument("path")
 @click.pass_obj
 def bindmount_command(context: BaseComposeContext, service: str, path: str) -> None:
     """
     This command is made obsolete by the --mount arguments.
     """
+    fmt.echo_alert(
+        "The 'bindmount' command is deprecated and will be removed in a later release. Use 'copyfrom' instead."
+    )
     config = tutor_config.load(context.root)
     host_path = bindmounts.create(context.job_runner(config), service, path)
     fmt.echo_info(
@@ -343,6 +348,51 @@ def bindmount_command(context: BaseComposeContext, service: str, path: str) -> N
     )
 
 
+@click.command(
+    name="copyfrom",
+    help="Copy files/folders from a container directory to the local filesystem.",
+)
+@click.argument("service")
+@click.argument("container_path")
+@click.argument(
+    "host_path",
+    type=click.Path(dir_okay=True, file_okay=False, resolve_path=True),
+)
+@click.pass_obj
+def copyfrom(
+    context: BaseComposeContext, service: str, container_path: str, host_path: str
+) -> None:
+    # Path management
+    container_root_path = "/tmp/mount"
+    container_dst_path = container_root_path
+    if not os.path.exists(host_path):
+        # Emulate cp semantics, where if the destination path does not exist
+        # then we copy to its parent and rename to the destination folder
+        container_dst_path += "/" + os.path.basename(host_path)
+        host_path = os.path.dirname(host_path)
+    if not os.path.exists(host_path):
+        raise TutorError(
+            f"Cannot create directory {host_path}. No such file or directory."
+        )
+
+    # cp/mv commands
+    command = f"cp --recursive --preserve {container_path} {container_dst_path}"
+    config = tutor_config.load(context.root)
+    runner = context.job_runner(config)
+    runner.docker_compose(
+        "run",
+        "--rm",
+        "--no-deps",
+        "--user=0",
+        f"--volume={host_path}:{container_root_path}",
+        service,
+        "sh",
+        "-e",
+        "-c",
+        command,
+    )
+
+
 @click.command(
     short_help="Run a command in a running container",
     help=(
@@ -490,6 +540,7 @@ def add_commands(command_group: click.Group) -> None:
     command_group.add_command(settheme)
     command_group.add_command(dc_command)
     command_group.add_command(run)
+    command_group.add_command(copyfrom)
     command_group.add_command(bindmount_command)
     command_group.add_command(execute)
     command_group.add_command(logs)

tutor/commands/dev.py

@@ -2,13 +2,13 @@ import typing as t
 
 import click
 
-from .. import config as tutor_config
-from .. import env as tutor_env
-from .. import exceptions, fmt
-from .. import interactive as interactive_config
-from .. import utils
-from ..types import Config, get_typed
-from . import compose
+from tutor import config as tutor_config
+from tutor import env as tutor_env
+from tutor import exceptions, fmt, hooks
+from tutor import interactive as interactive_config
+from tutor import utils
+from tutor.commands import compose
+from tutor.types import Config, get_typed
 
 
 class DevJobRunner(compose.ComposeJobRunner):
@@ -128,6 +128,17 @@ def runserver(
     context.invoke(compose.run, mounts=mounts, args=args)
 
 
+@hooks.Actions.COMPOSE_PROJECT_STARTED.add()
+def _stop_on_local_start(root: str, config: Config, project_name: str) -> None:
+    """
+    Stop the dev platform as soon as a platform with a different project name is
+    started.
+    """
+    runner = DevJobRunner(root, config)
+    if project_name != runner.project_name:
+        runner.docker_compose("stop")
+
+
 dev.add_command(quickstart)
 dev.add_command(runserver)
 compose.add_commands(dev)

tutor/commands/k8s.py

@@ -6,8 +6,9 @@ import click
 
 from tutor import config as tutor_config
 from tutor import env as tutor_env
+from tutor import exceptions, fmt
 from tutor import interactive as interactive_config
-from tutor import exceptions, fmt, jobs, serialize, utils
+from tutor import jobs, serialize, utils
 from tutor.commands.config import save as config_save_command
 from tutor.commands.context import BaseJobContext
 from tutor.commands.upgrade.k8s import upgrade_from

tutor/commands/local.py

@@ -4,8 +4,9 @@ import click
 
 from tutor import config as tutor_config
 from tutor import env as tutor_env
-from tutor import exceptions, fmt, utils
+from tutor import exceptions, fmt, hooks
 from tutor import interactive as interactive_config
+from tutor import utils
 from tutor.commands import compose
 from tutor.commands.config import save as config_save_command
 from tutor.commands.upgrade.local import upgrade_from
@@ -165,6 +166,17 @@ def upgrade(context: click.Context, from_release: t.Optional[str]) -> None:
     context.invoke(config_save_command)
 
 
+@hooks.Actions.COMPOSE_PROJECT_STARTED.add()
+def _stop_on_dev_start(root: str, config: Config, project_name: str) -> None:
+    """
+    Stop the local platform as soon as a platform with a different project name is
+    started.
+    """
+    runner = LocalJobRunner(root, config)
+    if project_name != runner.project_name:
+        runner.docker_compose("stop")
+
+
 local.add_command(quickstart)
 local.add_command(upgrade)
 compose.add_commands(local)

tutor/commands/plugins.py

@@ -1,6 +1,6 @@
 import os
-import urllib.request
 import typing as t
+import urllib.request
 
 import click
 

tutor/hooks/consts.py

@@ -25,6 +25,13 @@ class Actions:
             # Do stuff here
     """
 
+    #: Triggered whenever a "docker-compose start", "up" or "restart" command is executed.
+    #:
+    #: :parameter: str root: project root.
+    #: :parameter: dict[str, ...] config: project configuration.
+    #: :parameter: str name: docker-compose project name.
+    COMPOSE_PROJECT_STARTED = actions.get("compose:project:started")
+
     #: Called whenever the core project is ready to run. This action is called as soon
     #: as possible. This is the right time to discover plugins, for instance. In
     #: particular, we auto-discover the following plugins: