christianlight/tutor
7
0
Fork 0
mirror of https://github.com/ChristianLight/tutor.git synced 2024-06-07 00:20:49 +00:00
Code Issues Releases Activity

Merge remote-tracking branch 'origin/master' into nightly

Browse Source
This commit is contained in:
Overhang.IO 2023-02-27 08:32:05 +00:00
commit c669565edf
6 changed files with 67 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
changelog.d/20230214_105510_keith_fix_jobs_merge.md Normal file
View File

@ -0,0 +1,12 @@
<!--
Create a changelog entry for every new user-facing change. Please respect the following instructions:
- Indicate breaking changes by prepending an explosion 💥 character.
- Prefix your changes with either [Bugfix], [Improvement], [Feature], [Security], [Deprecation].
- You may optionally append "(by @<author>)" at the end of the line, where "<author>" is either one (just one)
of your GitHub username, real name or affiliated organization. These affiliations will be displayed in
the release notes for every release.
-->
<!-- - 💥[Feature] Foobarize the blorginator. This breaks plugins by renaming the `FOO_DO` filter to `BAR_DO`. (by @regisb) -->
<!-- - [Improvement] This is a non-breaking change. Life is good. (by @billgates) -->
[Bugfix] `patchStrategicMerge` can now be applied to jobs (by @keithgg)

10
requirements/base.txt
View File

@ -16,7 +16,7 @@ charset-normalizer==3.0.1
# via requests
click==8.1.3
# via -r requirements/base.in
google-auth==2.16.0
google-auth==2.16.1
# via kubernetes
idna==3.4
# via requests
@ -24,11 +24,11 @@ importlib-metadata==6.0.0
# via click
jinja2==3.1.2
# via -r requirements/base.in
kubernetes==25.3.0
kubernetes==26.1.0
# via -r requirements/base.in
markupsafe==2.1.2
# via jinja2
mypy==1.0.0
mypy==1.0.1
# via -r requirements/base.in
mypy-extensions==1.0.0
# via mypy
@ -65,7 +65,7 @@ tomli==2.0.1
# via mypy
typed-ast==1.5.4
# via mypy
typing-extensions==4.4.0
typing-extensions==4.5.0
# via
# -r requirements/base.in
# importlib-metadata
@ -76,7 +76,7 @@ urllib3==1.26.14
# requests
websocket-client==1.5.1
# via kubernetes
zipp==3.12.1
zipp==3.15.0
# via importlib-metadata
# The following packages are considered to be unsafe in a requirements file:

38
requirements/dev.txt
View File

@ -8,7 +8,7 @@ altgraph==0.17.3
# via pyinstaller
appdirs==1.4.4
# via -r requirements/base.txt
astroid==2.14.1
astroid==2.14.2
# via pylint
attrs==22.2.0
# via scriv
@ -42,7 +42,7 @@ click==8.1.3
# scriv
click-log==0.4.0
# via scriv
coverage==7.1.0
coverage==7.2.1
# via -r requirements/dev.in
cryptography==39.0.1
# via secretstorage
@ -52,7 +52,7 @@ docutils==0.17.1
# via
# -r requirements/dev.in
# readme-renderer
google-auth==2.16.0
google-auth==2.16.1
# via
# -r requirements/base.txt
# kubernetes
@ -68,7 +68,7 @@ importlib-metadata==6.0.0
# keyring
# pyinstaller
# twine
importlib-resources==5.10.2
importlib-resources==5.12.0
# via keyring
isort==5.11.5
# via pylint
@ -84,11 +84,11 @@ jinja2==3.1.2
# scriv
keyring==23.13.1
# via twine
kubernetes==25.3.0
kubernetes==26.1.0
# via -r requirements/base.txt
lazy-object-proxy==1.9.0
# via astroid
markdown-it-py==2.1.0
markdown-it-py==2.2.0
# via rich
markupsafe==2.1.2
# via
@ -100,7 +100,7 @@ mdurl==0.1.2
# via markdown-it-py
more-itertools==9.0.0
# via jaraco-classes
mypy==1.0.0
mypy==1.0.1
# via -r requirements/base.txt
mypy-extensions==1.0.0
# via
@ -142,11 +142,11 @@ pygments==2.14.0
# via
# readme-renderer
# rich
pyinstaller==5.7.0
pyinstaller==5.8.0
# via -r requirements/dev.in
pyinstaller-hooks-contrib==2022.15
pyinstaller-hooks-contrib==2023.0
# via pyinstaller
pylint==2.16.1
pylint==2.16.2
# via -r requirements/dev.in
pyproject-hooks==1.0.0
# via build
@ -182,7 +182,7 @@ rsa==4.9
# via
# -r requirements/base.txt
# google-auth
scriv==1.2.0
scriv==1.2.1
# via -r requirements/dev.in
secretstorage==3.3.3
# via keyring
@ -211,15 +211,13 @@ typed-ast==1.5.4
# astroid
# black
# mypy
types-docutils==0.19.1.3
# via
# -r requirements/dev.in
# types-setuptools
types-pyyaml==6.0.12.4
types-docutils==0.19.1.6
# via -r requirements/dev.in
types-setuptools==67.2.0.1
types-pyyaml==6.0.12.8
# via -r requirements/dev.in
typing-extensions==4.4.0
types-setuptools==67.4.0.3
# via -r requirements/dev.in
typing-extensions==4.5.0
# via
# -r requirements/base.txt
# astroid
@ -244,9 +242,9 @@ websocket-client==1.5.1
# kubernetes
wheel==0.38.4
# via pip-tools
wrapt==1.14.1
wrapt==1.15.0
# via astroid
zipp==3.12.1
zipp==3.15.0
# via
# -r requirements/base.txt
# importlib-metadata

10
requirements/docs.txt
View File

@ -32,7 +32,7 @@ docutils==0.18.1
# sphinx
# sphinx-click
# sphinx-rtd-theme
google-auth==2.16.0
google-auth==2.16.1
# via
# -r requirements/base.txt
# kubernetes
@ -51,13 +51,13 @@ jinja2==3.1.2
# via
# -r requirements/base.txt
# sphinx
kubernetes==25.3.0
kubernetes==26.1.0
# via -r requirements/base.txt
markupsafe==2.1.2
# via
# -r requirements/base.txt
# jinja2
mypy==1.0.0
mypy==1.0.1
# via -r requirements/base.txt
mypy-extensions==1.0.0
# via
@ -145,7 +145,7 @@ typed-ast==1.5.4
# via
# -r requirements/base.txt
# mypy
typing-extensions==4.4.0
typing-extensions==4.5.0
# via
# -r requirements/base.txt
# importlib-metadata
@ -159,7 +159,7 @@ websocket-client==1.5.1
# via
# -r requirements/base.txt
# kubernetes
zipp==3.12.1
zipp==3.15.0
# via
# -r requirements/base.txt
# importlib-metadata

31
tutor/commands/k8s.py
View File

@ -1,6 +1,6 @@
from datetime import datetime
from time import sleep
from typing import Any, List, Optional, Type
from typing import Any, List, Optional, Type, Iterable
import click
@ -64,11 +64,12 @@ class K8sTaskRunner(BaseTaskRunner):
"""
def run_task(self, service: str, command: str) -> int:
job_name = f"{service}-job"
job = self.load_job(job_name)
canonical_job_name = f"{service}-job"
all_jobs = list(self._load_jobs())
job = self._find_job(canonical_job_name, all_jobs)
# Create a unique job name to make it deduplicate jobs and make it easier to
# find later. Logs of older jobs will remain available for some time.
job_name += "-" + datetime.now().strftime("%Y%m%d%H%M%S")
job_name = canonical_job_name + "-" + datetime.now().strftime("%Y%m%d%H%M%S")
# Wait until all other jobs are completed
while True:
@ -98,11 +99,12 @@ class K8sTaskRunner(BaseTaskRunner):
job["spec"]["template"]["spec"]["containers"][0]["args"] = container_args
job["spec"]["backoffLimit"] = 1
job["spec"]["ttlSecondsAfterFinished"] = 3600
# Save patched job to "jobs.yml" file
with open(
tutor_env.pathjoin(self.root, "k8s", "jobs.yml"), "w", encoding="utf-8"
) as job_file:
serialize.dump(job, job_file)
serialize.dump_all(all_jobs, job_file)
# We cannot use the k8s API to create the job: configMap and volume names need
# to be found with the right suffixes.
kubectl_apply(
@ -143,8 +145,15 @@ class K8sTaskRunner(BaseTaskRunner):
"""
Find a given job definition in the rendered k8s/jobs.yml template.
"""
all_jobs = self.render("k8s", "jobs.yml")
for job in serialize.load_all(all_jobs):
return self._find_job(name, self._load_jobs())
def _find_job(self, name: str, all_jobs: Iterable[Any]) -> Any:
"""
Find the matching job definition in the in the list of jobs provided.
Returns the found job's manifest.
"""
for job in all_jobs:
job_name = job["metadata"]["name"]
if not isinstance(job_name, str):
raise exceptions.TutorError(
@ -154,6 +163,12 @@ class K8sTaskRunner(BaseTaskRunner):
return job
raise exceptions.TutorError(f"Could not find job '{name}'")
def _load_jobs(self) -> Iterable[Any]:
manifests = self.render("k8s", "jobs.yml")
for manifest in serialize.load_all(manifests):
if manifest["kind"] == "Job":
yield manifest
def active_job_names(self) -> List[str]:
"""
Return a list of active job names

4
tutor/serialize.py
View File

@ -17,6 +17,10 @@ def load_all(stream: str) -> t.Iterator[t.Any]:
return yaml.load_all(stream, Loader=yaml.SafeLoader)
def dump_all(documents: t.Sequence[t.Any], fileobj: TextIOWrapper) -> None:
yaml.safe_dump_all(documents, stream=fileobj, default_flow_style=False)
def dump(content: t.Any, fileobj: TextIOWrapper) -> None:
yaml.dump(content, stream=fileobj, default_flow_style=False)