feat: make it possible to override local docker-compose job

Previously, it was possible to override docker-compose services, but not jobs. This requirement has appeared because some people need to override project-wide MTU settings.

See: https://discuss.overhang.io/t/problem-fetching-saml-idp-metadata/1330/23

CHANGELOG.md

@@ -4,6 +4,8 @@ Note: Breaking changes between versions are indicated by "💥".
 
 ## Unreleased
 
+- 💥[Feature] Make it possible to override local job configuration. This deprecates the older model for running jobs which dates back from a long time ago.
+
 ## v12.0.4 (2021-08-12)
 
 - [Security] Apply security patch [28442](https://github.com/edx/edx-platform/pull/28442).

docs/local.rst

@@ -280,3 +280,5 @@ You might want to customize the docker-compose services listed in ``$(tutor conf
     vim $(tutor config printroot)/env/local/docker-compose.override.yml
 
 The values in this file will override the values from ``docker-compose.yml`` and ``docker-compose.prod.yml``, as explained in the `docker-compose documentation <https://docs.docker.com/compose/extends/#adding-and-overriding-configuration>`__.
+
+Similarly, the job service configuration can be overridden by creating a ``docker-compose.jobs.override.yml`` file in that same folder.

tutor/commands/compose.py

@@ -10,7 +10,6 @@ from .. import env as tutor_env
 from ..exceptions import TutorError
 from .. import fmt
 from .. import jobs
-from .. import serialize
 from ..types import Config
 from .. import utils
 from .context import Context
@@ -33,49 +32,24 @@ class ComposeJobRunner(jobs.BaseJobRunner):
         service does not exist, run the service from good old regular
         docker-compose.yml.
         """
-        jobs_path = tutor_env.pathjoin(self.root, "local", "docker-compose.jobs.yml")
-        job_service_name = "{}-job".format(service)
-        opts = [] if utils.is_a_tty() else ["-T"]
-        if job_service_name in serialize.load(open(jobs_path).read())["services"]:
-            return self.docker_compose_func(
-                self.root,
-                self.config,
-                "-f",
-                jobs_path,
-                "run",
-                *opts,
-                "--rm",
-                job_service_name,
-                "sh",
-                "-e",
-                "-c",
-                command,
-            )
-        fmt.echo_alert(
-            (
-                "The '{job_service_name}' service does not exist in {jobs_path}. "
-                "This might be caused by an older plugin. Tutor switched to a job "
-                "runner model for running one-time commands, such as database"
-                " initialisation. For the record, this is the command that we are "
-                "running:\n"
-                "\n"
-                "    {command}\n"
-                "\n"
-                "Old-style job running will be deprecated soon. Please inform "
-                "your plugin maintainer!"
-            ).format(
-                job_service_name=job_service_name,
-                jobs_path=jobs_path,
-                command=command.replace("\n", "\n    "),
-            )
+        run_command = [
+            "-f",
+            tutor_env.pathjoin(self.root, "local", "docker-compose.jobs.yml"),
+        ]
+        override_path = tutor_env.pathjoin(
+            self.root, "local", "docker-compose.jobs.override.yml"
         )
+        if os.path.exists(override_path):
+            run_command += ["-f", override_path]
+        run_command += ["run", "--rm"]
+        if not utils.is_a_tty():
+            run_command += ["-T"]
+        job_service_name = "{}-job".format(service)
         return self.docker_compose_func(
             self.root,
             self.config,
-            "run",
-            *opts,
-            "--rm",
-            service,
+            *run_command,
+            job_service_name,
             "sh",
             "-e",
             "-c",

tutor/commands/k8s.py

@@ -58,7 +58,7 @@ class K8sJobRunner(jobs.BaseJobRunner):
                 )
             if job_name == name:
                 return job
-        raise ValueError("Could not find job '{}'".format(name))
+        raise exceptions.TutorError("Could not find job '{}'".format(name))
 
     def active_job_names(self) -> List[str]:
         """
@@ -75,27 +75,7 @@ class K8sJobRunner(jobs.BaseJobRunner):
 
     def run_job(self, service: str, command: str) -> int:
         job_name = "{}-job".format(service)
-        try:
-            job = self.load_job(job_name)
-        except ValueError:
-            message = (
-                "The '{job_name}' kubernetes job does not exist in the list of job "
-                "runners. This might be caused by an older plugin. Tutor switched to a"
-                " job runner model for running one-time commands, such as database"
-                " initialisation. For the record, this is the command that we are "
-                "running:\n"
-                "\n"
-                "    {command}\n"
-                "\n"
-                "Old-style job running will be deprecated soon. Please inform "
-                "your plugin maintainer!"
-            ).format(
-                job_name=job_name,
-                command=command.replace("\n", "\n    "),
-            )
-            fmt.echo_alert(message)
-            wait_for_pod_ready(self.config, service)
-            return kubectl_exec(self.config, service, command)
+        job = self.load_job(job_name)
         # Create a unique job name to make it deduplicate jobs and make it easier to
         # find later. Logs of older jobs will remain available for some time.
         job_name += "-" + datetime.now().strftime("%Y%m%d%H%M%S")