From d3bfc4fb87c8377a1a81a5b9c95b47bb8a0973d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Mon, 6 Sep 2021 16:20:36 +0200 Subject: [PATCH] feat: make it possible to override local docker-compose job Previously, it was possible to override docker-compose services, but not jobs. This requirement has appeared because some people need to override project-wide MTU settings. See: https://discuss.overhang.io/t/problem-fetching-saml-idp-metadata/1330/23 --- CHANGELOG.md | 2 ++ docs/local.rst | 2 ++ tutor/commands/compose.py | 54 ++++++++++----------------------------- tutor/commands/k8s.py | 24 ++--------------- 4 files changed, 20 insertions(+), 62 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a32b37..bfad27f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Note: Breaking changes between versions are indicated by "💥". ## Unreleased +- 💥[Feature] Make it possible to override local job configuration. This deprecates the older model for running jobs which dates back from a long time ago. + ## v12.0.4 (2021-08-12) - [Security] Apply security patch [28442](https://github.com/edx/edx-platform/pull/28442). diff --git a/docs/local.rst b/docs/local.rst index 8f93962..48fe784 100644 --- a/docs/local.rst +++ b/docs/local.rst @@ -280,3 +280,5 @@ You might want to customize the docker-compose services listed in ``$(tutor conf vim $(tutor config printroot)/env/local/docker-compose.override.yml The values in this file will override the values from ``docker-compose.yml`` and ``docker-compose.prod.yml``, as explained in the `docker-compose documentation `__. + +Similarly, the job service configuration can be overridden by creating a ``docker-compose.jobs.override.yml`` file in that same folder. diff --git a/tutor/commands/compose.py b/tutor/commands/compose.py index be75583..48022f7 100644 --- a/tutor/commands/compose.py +++ b/tutor/commands/compose.py @@ -10,7 +10,6 @@ from .. import env as tutor_env from ..exceptions import TutorError from .. import fmt from .. import jobs -from .. import serialize from ..types import Config from .. import utils from .context import Context @@ -33,49 +32,24 @@ class ComposeJobRunner(jobs.BaseJobRunner): service does not exist, run the service from good old regular docker-compose.yml. """ - jobs_path = tutor_env.pathjoin(self.root, "local", "docker-compose.jobs.yml") - job_service_name = "{}-job".format(service) - opts = [] if utils.is_a_tty() else ["-T"] - if job_service_name in serialize.load(open(jobs_path).read())["services"]: - return self.docker_compose_func( - self.root, - self.config, - "-f", - jobs_path, - "run", - *opts, - "--rm", - job_service_name, - "sh", - "-e", - "-c", - command, - ) - fmt.echo_alert( - ( - "The '{job_service_name}' service does not exist in {jobs_path}. " - "This might be caused by an older plugin. Tutor switched to a job " - "runner model for running one-time commands, such as database" - " initialisation. For the record, this is the command that we are " - "running:\n" - "\n" - " {command}\n" - "\n" - "Old-style job running will be deprecated soon. Please inform " - "your plugin maintainer!" - ).format( - job_service_name=job_service_name, - jobs_path=jobs_path, - command=command.replace("\n", "\n "), - ) + run_command = [ + "-f", + tutor_env.pathjoin(self.root, "local", "docker-compose.jobs.yml"), + ] + override_path = tutor_env.pathjoin( + self.root, "local", "docker-compose.jobs.override.yml" ) + if os.path.exists(override_path): + run_command += ["-f", override_path] + run_command += ["run", "--rm"] + if not utils.is_a_tty(): + run_command += ["-T"] + job_service_name = "{}-job".format(service) return self.docker_compose_func( self.root, self.config, - "run", - *opts, - "--rm", - service, + *run_command, + job_service_name, "sh", "-e", "-c", diff --git a/tutor/commands/k8s.py b/tutor/commands/k8s.py index 9fd51c7..743645f 100644 --- a/tutor/commands/k8s.py +++ b/tutor/commands/k8s.py @@ -58,7 +58,7 @@ class K8sJobRunner(jobs.BaseJobRunner): ) if job_name == name: return job - raise ValueError("Could not find job '{}'".format(name)) + raise exceptions.TutorError("Could not find job '{}'".format(name)) def active_job_names(self) -> List[str]: """ @@ -75,27 +75,7 @@ class K8sJobRunner(jobs.BaseJobRunner): def run_job(self, service: str, command: str) -> int: job_name = "{}-job".format(service) - try: - job = self.load_job(job_name) - except ValueError: - message = ( - "The '{job_name}' kubernetes job does not exist in the list of job " - "runners. This might be caused by an older plugin. Tutor switched to a" - " job runner model for running one-time commands, such as database" - " initialisation. For the record, this is the command that we are " - "running:\n" - "\n" - " {command}\n" - "\n" - "Old-style job running will be deprecated soon. Please inform " - "your plugin maintainer!" - ).format( - job_name=job_name, - command=command.replace("\n", "\n "), - ) - fmt.echo_alert(message) - wait_for_pod_ready(self.config, service) - return kubectl_exec(self.config, service, command) + job = self.load_job(job_name) # Create a unique job name to make it deduplicate jobs and make it easier to # find later. Logs of older jobs will remain available for some time. job_name += "-" + datetime.now().strftime("%Y%m%d%H%M%S")