feat: make it possible to override jobs in dev

Previously, job declarations were always loaded from local/docker-compose.yml
and local/docker-compose.jobs.yml. This meant that it was not possible to
override job declarations in dev mode. It is now the case, with
dev/docker-compose.jobs.yml and dev/docker-compose.jobs.override.yml. Neither
of these files exist yet... But who knows? we might need this feature one day.
In any case the code is much cleaner now.

CHANGELOG.md

@@ -4,6 +4,7 @@ Note: Breaking changes between versions are indicated by "💥".
 
 ## Unreleased
 
+- [Feature] Make it possible to override job configuration in development: if they exist, `dev/docker-compose.jobs.yml` and `dev/docker-compose.jobs.override.yml` will be loaded when running jobs.
 - [Improvement] Faster `tutor local start` by building only necessary images.
 
 ## v12.1.5 (2021-10-25)

tutor/commands/compose.py

@@ -15,6 +15,24 @@ from .context import BaseJobContext
 
 
 class ComposeJobRunner(jobs.BaseComposeJobRunner):
+    def __init__(self, root: str, config: Config):
+        super().__init__(root, config)
+        self.project_name = ""
+        self.docker_compose_files: List[str] = []
+        self.docker_compose_job_files: List[str] = []
+
+    def docker_compose(self, *command: str) -> int:
+        """
+        Run docker-compose with the right yml files.
+        """
+        args = []
+        for docker_compose_path in self.docker_compose_files:
+            if os.path.exists(docker_compose_path):
+                args += ["-f", docker_compose_path]
+        return utils.docker_compose(
+            *args, "--project-name", self.project_name, *command
+        )
+
     def run_job(self, service: str, command: str) -> int:
         """
         Run the "{{ service }}-job" service from local/docker-compose.jobs.yml with the
@@ -22,15 +40,11 @@ class ComposeJobRunner(jobs.BaseComposeJobRunner):
         service does not exist, run the service from good old regular
         docker-compose.yml.
         """
-        run_command = [
-            "-f",
-            tutor_env.pathjoin(self.root, "local", "docker-compose.jobs.yml"),
-        ]
-        override_path = tutor_env.pathjoin(
-            self.root, "local", "docker-compose.jobs.override.yml"
-        )
-        if os.path.exists(override_path):
-            run_command += ["-f", override_path]
+        run_command = []
+        for docker_compose_path in self.docker_compose_job_files:
+            path = tutor_env.pathjoin(self.root, docker_compose_path)
+            if os.path.exists(path):
+                run_command += ["-f", path]
         run_command += ["run", "--rm"]
         if not utils.is_a_tty():
             run_command += ["-T"]

tutor/commands/dev.py

@@ -1,4 +1,3 @@
-import os
 from typing import List
 
 import click
@@ -6,35 +5,29 @@ import click
 from .. import config as tutor_config
 from .. import env as tutor_env
 from .. import fmt
-from ..types import Config
-from .. import utils
+from ..types import Config, get_typed
 from . import compose
 
 
 class DevJobRunner(compose.ComposeJobRunner):
-    def docker_compose(self, *command: str) -> int:
+    def __init__(self, root: str, config: Config):
         """
-        Run docker-compose with dev arguments.
+        Load docker-compose files from dev/ and local/
         """
-        args = []
-        for folder in ["local", "dev"]:
-            # Add docker-compose.yml and docker-compose.override.yml (if it exists)
-            # from "local" and "dev" folders (but not docker-compose.prod.yml)
-            args += [
-                "-f",
-                tutor_env.pathjoin(self.root, folder, "docker-compose.yml"),
-            ]
-            override_path = tutor_env.pathjoin(
-                self.root, folder, "docker-compose.override.yml"
-            )
-            if os.path.exists(override_path):
-                args += ["-f", override_path]
-        return utils.docker_compose(
-            *args,
-            "--project-name",
-            str(self.config["DEV_PROJECT_NAME"]),
-            *command,
-        )
+        super().__init__(root, config)
+        self.project_name = get_typed(self.config, "DEV_PROJECT_NAME", str)
+        self.docker_compose_files += [
+            tutor_env.pathjoin(self.root, "local", "docker-compose.yml"),
+            tutor_env.pathjoin(self.root, "dev", "docker-compose.yml"),
+            tutor_env.pathjoin(self.root, "local", "docker-compose.override.yml"),
+            tutor_env.pathjoin(self.root, "dev", "docker-compose.override.yml"),
+        ]
+        self.docker_compose_job_files += [
+            tutor_env.pathjoin(self.root, "local", "docker-compose.jobs.yml"),
+            tutor_env.pathjoin(self.root, "dev", "docker-compose.jobs.yml"),
+            tutor_env.pathjoin(self.root, "local", "docker-compose.jobs.override.yml"),
+            tutor_env.pathjoin(self.root, "dev", "docker-compose.jobs.override.yml"),
+        ]
 
 
 class DevContext(compose.BaseComposeContext):

tutor/commands/local.py

@@ -1,4 +1,3 @@
-import os
 from time import sleep
 
 import click
@@ -6,7 +5,7 @@ import click
 from .. import config as tutor_config
 from .. import env as tutor_env
 from .. import fmt
-from ..types import get_typed, Config
+from ..types import Config, get_typed
 from .. import utils
 from .. import exceptions
 from . import compose
@@ -14,26 +13,21 @@ from .config import save as config_save_command
 
 
 class LocalJobRunner(compose.ComposeJobRunner):
-    def docker_compose(self, *command: str) -> int:
+    def __init__(self, root: str, config: Config):
         """
-        Run docker-compose with local and production yml files.
+        Load docker-compose files from local/.
         """
-        args = []
-        override_path = tutor_env.pathjoin(
-            self.root, "local", "docker-compose.override.yml"
-        )
-        if os.path.exists(override_path):
-            args += ["-f", override_path]
-        return utils.docker_compose(
-            "-f",
+        super().__init__(root, config)
+        self.project_name = get_typed(self.config, "LOCAL_PROJECT_NAME", str)
+        self.docker_compose_files += [
             tutor_env.pathjoin(self.root, "local", "docker-compose.yml"),
-            "-f",
             tutor_env.pathjoin(self.root, "local", "docker-compose.prod.yml"),
-            *args,
-            "--project-name",
-            get_typed(self.config, "LOCAL_PROJECT_NAME", str),
-            *command
-        )
+            tutor_env.pathjoin(self.root, "local", "docker-compose.override.yml"),
+        ]
+        self.docker_compose_job_files += [
+            tutor_env.pathjoin(self.root, "local", "docker-compose.jobs.yml"),
+            tutor_env.pathjoin(self.root, "local", "docker-compose.jobs.override.yml"),
+        ]
 
 
 class LocalContext(compose.BaseComposeContext):

tutor/templates/dev/docker-compose.jobs.yml

@@ -0,0 +1,3 @@
+version: "3.7"
+services: {% if not patch("dev-docker-compose-jobs-services") %}{}{% endif %}
+    {{ patch("dev-docker-compose-jobs-services")|indent(4) }}