christianlight
/
tutor
mirror of https://github.com/ChristianLight/tutor.git
7
0
Fork 0
Code Issues Releases Activity

refactor: deduplicate jobs code 

createuser, importdemocourse and settheme were 100% duplicated code between
k8s.py and compose.py.
This commit is contained in:
Régis Behmo 2022-10-18 16:57:07 +02:00 committed by Régis Behmo
parent e734f52f07
commit b6dc65cc64
8 changed files with 216 additions and 283 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
tests/test_jobs.py → tests/commands/test_jobs.py
View File

@ -5,7 +5,7 @@ from unittest.mock import patch
from tests.helpers import TestContext, temporary_root
from tutor import config as tutor_config
from tutor import jobs
from tutor.commands import jobs
class JobsTests(unittest.TestCase):
@ -21,26 +21,21 @@ class JobsTests(unittest.TestCase):
self.assertTrue(output.endswith("All services initialised."))
def test_create_user_command_without_staff(self) -> None:
command = jobs.create_user_command("superuser", False, "username", "email")
command = jobs.create_user_template("superuser", False, "username", "email", "p4ssw0rd")
self.assertNotIn("--staff", command)
self.assertIn("set_password", command)
def test_create_user_command_with_staff(self) -> None:
command = jobs.create_user_command("superuser", True, "username", "email")
command = jobs.create_user_template("superuser", True, "username", "email", "p4ssw0rd")
self.assertIn("--staff", command)
def test_create_user_command_with_staff_with_password(self) -> None:
command = jobs.create_user_command(
"superuser", True, "username", "email", "command"
)
self.assertIn("set_password", command)
@patch("sys.stdout", new_callable=StringIO)
def test_import_demo_course(self, mock_stdout: StringIO) -> None:
with temporary_root() as root:
context = TestContext(root)
config = tutor_config.load_full(root)
runner = context.job_runner(config)
jobs.import_demo_course(runner)
runner.run_job_from_str("cms", jobs.import_demo_course_template())
output = mock_stdout.getvalue()
service = re.search(r"Service: (\w*)", output)
@ -60,7 +55,8 @@ class JobsTests(unittest.TestCase):
context = TestContext(root)
config = tutor_config.load_full(root)
runner = context.job_runner(config)
jobs.set_theme("sample_theme", ["domain1", "domain2"], runner)
command = jobs.set_theme_template("sample_theme", ["domain1", "domain2"])
runner.run_job_from_str("lms", command)
output = mock_stdout.getvalue()
service = re.search(r"Service: (\w*)", output)
@ -73,10 +69,3 @@ class JobsTests(unittest.TestCase):
.strip()
.startswith('echo "Loading settings $DJANGO_SETTINGS_MODULE"')
)
def test_get_all_openedx_domains(self) -> None:
with temporary_root() as root:
config = tutor_config.load_full(root)
domains = jobs.get_all_openedx_domains(config)
self.assertTrue(domains)
self.assertEqual(6, len(domains))

68
tutor/commands/compose.py
View File

@ -7,13 +7,15 @@ import click
from tutor import config as tutor_config
from tutor import env as tutor_env
from tutor import fmt, hooks, jobs, serialize, utils
from tutor import fmt, hooks, serialize, utils
from tutor.commands import jobs
from tutor.commands.context import BaseJobContext
from tutor.exceptions import TutorError
from tutor.jobs import BaseComposeJobRunner
from tutor.types import Config
class ComposeJobRunner(jobs.BaseComposeJobRunner):
class ComposeJobRunner(BaseComposeJobRunner):
def __init__(self, root: str, config: Config):
super().__init__(root, config)
self.project_name = ""
@ -310,64 +312,6 @@ def init(
jobs.initialise(runner, limit_to=limit)
@click.command(help="Create an Open edX user and interactively set their password")
@click.option("--superuser", is_flag=True, help="Make superuser")
@click.option("--staff", is_flag=True, help="Make staff user")
@click.option(
"-p",
"--password",
help="Specify password from the command line. If undefined, you will be prompted to input a password",
)
@click.argument("name")
@click.argument("email")
@click.pass_obj
def createuser(
context: BaseComposeContext,
superuser: str,
staff: bool,
password: str,
name: str,
email: str,
) -> None:
config = tutor_config.load(context.root)
runner = context.job_runner(config)
command = jobs.create_user_command(superuser, staff, name, email, password=password)
runner.run_job("lms", command)
@click.command(
help="Assign a theme to the LMS and the CMS. To reset to the default theme , use 'default' as the theme name."
)
@click.option(
"-d",
"--domain",
"domains",
multiple=True,
help=(
"Limit the theme to these domain names. By default, the theme is "
"applied to the LMS and the CMS, both in development and production mode"
),
)
@click.argument("theme_name")
@click.pass_obj
def settheme(
context: BaseComposeContext, domains: t.List[str], theme_name: str
) -> None:
config = tutor_config.load(context.root)
runner = context.job_runner(config)
domains = domains or jobs.get_all_openedx_domains(config)
jobs.set_theme(theme_name, domains, runner)
@click.command(help="Import the demo course")
@click.pass_obj
def importdemocourse(context: BaseComposeContext) -> None:
config = tutor_config.load(context.root)
runner = context.job_runner(config)
fmt.echo_info("Importing demo course")
jobs.import_demo_course(runner)
@click.command(
short_help="Run a command in a new container",
help=(
@ -527,12 +471,10 @@ def add_commands(command_group: click.Group) -> None:
command_group.add_command(restart)
command_group.add_command(reboot)
command_group.add_command(init)
command_group.add_command(createuser)
command_group.add_command(importdemocourse)
command_group.add_command(settheme)
command_group.add_command(dc_command)
command_group.add_command(run)
command_group.add_command(copyfrom)
command_group.add_command(execute)
command_group.add_command(logs)
command_group.add_command(status)
jobs.add_commands(command_group)

187
tutor/commands/jobs.py Normal file
View File

@ -0,0 +1,187 @@
"""
Common jobs that must be added both to local, dev and k8s commands.
"""
import typing as t
import click
from tutor import config as tutor_config
from tutor import fmt, hooks, jobs
from .context import BaseJobContext
BASE_OPENEDX_COMMAND = """
echo "Loading settings $DJANGO_SETTINGS_MODULE"
"""
@hooks.Actions.CORE_READY.add()
def _add_core_init_tasks() -> None:
"""
Declare core init scripts at runtime.
The context is important, because it allows us to select the init scripts based on
the --limit argument.
"""
with hooks.Contexts.APP("mysql").enter():
hooks.Filters.COMMANDS_INIT.add_item(("mysql", ("hooks", "mysql", "init")))
with hooks.Contexts.APP("lms").enter():
hooks.Filters.COMMANDS_INIT.add_item(("lms", ("hooks", "lms", "init")))
with hooks.Contexts.APP("cms").enter():
hooks.Filters.COMMANDS_INIT.add_item(("cms", ("hooks", "cms", "init")))
def initialise(runner: jobs.BaseJobRunner, limit_to: t.Optional[str] = None) -> None:
fmt.echo_info("Initialising all services...")
filter_context = hooks.Contexts.APP(limit_to).name if limit_to else None
# Pre-init tasks
iter_pre_init_tasks: t.Iterator[
t.Tuple[str, t.Iterable[str]]
] = hooks.Filters.COMMANDS_PRE_INIT.iterate(context=filter_context)
for service, path in iter_pre_init_tasks:
fmt.echo_info(f"Running pre-init task: {'/'.join(path)}")
runner.run_job_from_template(service, *path)
# Init tasks
iter_init_tasks: t.Iterator[
t.Tuple[str, t.Iterable[str]]
] = hooks.Filters.COMMANDS_INIT.iterate(context=filter_context)
for service, path in iter_init_tasks:
fmt.echo_info(f"Running init task: {'/'.join(path)}")
runner.run_job_from_template(service, *path)
fmt.echo_info("All services initialised.")
@click.command(help="Create an Open edX user and interactively set their password")
@click.option("--superuser", is_flag=True, help="Make superuser")
@click.option("--staff", is_flag=True, help="Make staff user")
@click.option(
"-p",
"--password",
help="Specify password from the command line. If undefined, you will be prompted to input a password",
prompt=True,
hide_input=True,
)
@click.argument("name")
@click.argument("email")
@click.pass_obj
def createuser(
context: BaseJobContext,
superuser: str,
staff: bool,
password: str,
name: str,
email: str,
) -> None:
run_job(
context, "lms", create_user_template(superuser, staff, name, email, password)
)
@click.command(help="Import the demo course")
@click.pass_obj
def importdemocourse(context: BaseJobContext) -> None:
run_job(context, "cms", import_demo_course_template())
@click.command(
help="Assign a theme to the LMS and the CMS. To reset to the default theme , use 'default' as the theme name."
)
@click.option(
"-d",
"--domain",
"domains",
multiple=True,
help=(
"Limit the theme to these domain names. By default, the theme is "
"applied to the LMS and the CMS, both in development and production mode"
),
)
@click.argument("theme_name")
@click.pass_obj
def settheme(context: BaseJobContext, domains: t.List[str], theme_name: str) -> None:
run_job(context, "lms", set_theme_template(theme_name, domains))
def run_job(context: BaseJobContext, service: str, command: str) -> None:
config = tutor_config.load(context.root)
runner = context.job_runner(config)
runner.run_job_from_str(service, command)
def create_user_template(
superuser: str, staff: bool, username: str, email: str, password: str
) -> str:
opts = ""
if superuser:
opts += " --superuser"
if staff:
opts += " --staff"
return (
BASE_OPENEDX_COMMAND
+ f"""
./manage.py lms manage_user {opts} {username} {email}
./manage.py lms shell -c "
from django.contrib.auth import get_user_model
u = get_user_model().objects.get(username='{username}')
u.set_password('{password}')
u.save()"
"""
)
def import_demo_course_template() -> str:
return (
BASE_OPENEDX_COMMAND
+ """
# Import demo course
git clone https://github.com/openedx/edx-demo-course --branch {{ OPENEDX_COMMON_VERSION }} --depth 1 ../edx-demo-course
python ./manage.py cms import ../data ../edx-demo-course
# Re-index courses
./manage.py cms reindex_course --all --setup"""
)
def set_theme_template(theme_name: str, domain_names: t.List[str]) -> str:
"""
For each domain, get or create a Site object and assign the selected theme.
"""
# Note that there are no double quotes " in this piece of code
python_command = """
import sys
from django.contrib.sites.models import Site
def assign_theme(name, domain):
print('Assigning theme', name, 'to', domain)
if len(domain) > 50:
sys.stderr.write(
'Assigning a theme to a site with a long (> 50 characters) domain name.'
' The displayed site name will be truncated to 50 characters.\\n'
)
site, _ = Site.objects.get_or_create(domain=domain)
if not site.name:
name_max_length = Site._meta.get_field('name').max_length
site.name = domain[:name_max_length]
site.save()
site.themes.all().delete()
site.themes.create(theme_dir_name=name)
"""
domain_names = domain_names or [
"{{ LMS_HOST }}",
"{{ LMS_HOST }}:8000",
"{{ CMS_HOST }}",
"{{ CMS_HOST }}:8001",
"{{ PREVIEW_LMS_HOST }}",
"{{ PREVIEW_LMS_HOST }}:8000",
]
for domain_name in domain_names:
python_command += f"assign_theme('{theme_name}', '{domain_name}')\n"
return BASE_OPENEDX_COMMAND + f'./manage.py lms shell -c "{python_command}"'
def add_commands(command_group: click.Group) -> None:
for job_command in [createuser, importdemocourse, settheme]:
command_group.add_command(job_command)

68
tutor/commands/k8s.py
View File

@ -8,10 +8,12 @@ from tutor import config as tutor_config
from tutor import env as tutor_env
from tutor import exceptions, fmt
from tutor import interactive as interactive_config
from tutor import jobs, serialize, utils
from tutor import serialize, utils
from tutor.commands import jobs
from tutor.commands.config import save as config_save_command
from tutor.commands.context import BaseJobContext
from tutor.commands.upgrade.k8s import upgrade_from
from tutor.jobs import BaseJobRunner
from tutor.types import Config, get_typed
@ -46,7 +48,7 @@ class K8sClients:
return self._core_api
class K8sJobRunner(jobs.BaseJobRunner):
class K8sJobRunner(BaseJobRunner):
def load_job(self, name: str) -> Any:
all_jobs = self.render("k8s", "jobs.yml")
for job in serialize.load_all(all_jobs):
@ -370,64 +372,6 @@ def scale(context: K8sContext, deployment: str, replicas: int) -> None:
)
@click.command(help="Create an Open edX user and interactively set their password")
@click.option("--superuser", is_flag=True, help="Make superuser")
@click.option("--staff", is_flag=True, help="Make staff user")
@click.option(
"-p",
"--password",
help="Specify password from the command line. If undefined, you will be prompted to input a password",
prompt=True,
hide_input=True,
)
@click.argument("name")
@click.argument("email")
@click.pass_obj
def createuser(
context: K8sContext,
superuser: str,
staff: bool,
password: str,
name: str,
email: str,
) -> None:
config = tutor_config.load(context.root)
command = jobs.create_user_command(superuser, staff, name, email, password=password)
runner = context.job_runner(config)
runner.run_job("lms", command)
@click.command(help="Import the demo course")
@click.pass_obj
def importdemocourse(context: K8sContext) -> None:
fmt.echo_info("Importing demo course")
config = tutor_config.load(context.root)
runner = context.job_runner(config)
jobs.import_demo_course(runner)
@click.command(
help="Assign a theme to the LMS and the CMS. To reset to the default theme , use 'default' as the theme name."
)
@click.option(
"-d",
"--domain",
"domains",
multiple=True,
help=(
"Limit the theme to these domain names. By default, the theme is "
"applied to the LMS and the CMS, both in development and production mode"
),
)
@click.argument("theme_name")
@click.pass_obj
def settheme(context: K8sContext, domains: List[str], theme_name: str) -> None:
config = tutor_config.load(context.root)
runner = context.job_runner(config)
domains = domains or jobs.get_all_openedx_domains(config)
jobs.set_theme(theme_name, domains, runner)
@click.command(
name="exec",
help="Execute a command in a pod of the given application",
@ -590,12 +534,10 @@ k8s.add_command(reboot)
k8s.add_command(delete)
k8s.add_command(init)
k8s.add_command(scale)
k8s.add_command(createuser)
k8s.add_command(importdemocourse)
k8s.add_command(settheme)
k8s.add_command(exec_command)
k8s.add_command(logs)
k8s.add_command(wait)
k8s.add_command(upgrade)
k8s.add_command(apply_command)
k8s.add_command(status)
jobs.add_commands(k8s)

3
tutor/config.py
View File

@ -223,10 +223,9 @@ def upgrade_obsolete(config: Config) -> None:
]:
if name in config:
config[name.replace("ACTIVATE_", "RUN_")] = config.pop(name)
# Replace RUN_CADDY by ENABLE_WEB_PROXY
# Replace nginx by caddy
if "RUN_CADDY" in config:
config["ENABLE_WEB_PROXY"] = config.pop("RUN_CADDY")
# Replace RUN_CADDY by ENABLE_WEB_PROXY
if "NGINX_HTTP_PORT" in config:
config["CADDY_HTTP_PORT"] = config.pop("NGINX_HTTP_PORT")

3
tutor/env.py
View File

@ -69,8 +69,7 @@ class JinjaEnvironment(jinja2.Environment):
class Renderer:
def __init__(self, config: t.Optional[Config] = None):
config = config or {}
self.config = deepcopy(config)
self.config = deepcopy(config or {})
self.template_roots = hooks.Filters.ENV_TEMPLATE_ROOTS.apply([TEMPLATES_ROOT])
# Create environment with extra filters and globals

137
tutor/jobs.py
View File

@ -1,16 +1,13 @@
import typing as t
from tutor import env, fmt, hooks
from tutor.types import Config, get_typed
BASE_OPENEDX_COMMAND = """
echo "Loading settings $DJANGO_SETTINGS_MODULE"
"""
from tutor import env
from tutor.types import Config
class BaseJobRunner:
"""
A job runner is responsible for getting a certain task to complete.
A job runner is responsible for running bash commands in the right context.
Commands may be loaded from string or template files. The `run_job` method must be
implemented by child classes.
"""
def __init__(self, root: str, config: Config):
@ -21,6 +18,10 @@ class BaseJobRunner:
command = self.render(*path)
self.run_job(service, command)
def run_job_from_str(self, service: str, command: str) -> None:
rendered = env.render_str(self.config, command).strip()
self.run_job(service, rendered)
def render(self, *path: str) -> str:
rendered = env.render_file(self.config, *path).strip()
if isinstance(rendered, bytes):
@ -39,121 +40,3 @@ class BaseJobRunner:
class BaseComposeJobRunner(BaseJobRunner):
def docker_compose(self, *command: str) -> int:
raise NotImplementedError
@hooks.Actions.CORE_READY.add()
def _add_core_init_tasks() -> None:
"""
Declare core init scripts at runtime.
The context is important, because it allows us to select the init scripts based on
the --limit argument.
"""
with hooks.Contexts.APP("mysql").enter():
hooks.Filters.COMMANDS_INIT.add_item(("mysql", ("hooks", "mysql", "init")))
with hooks.Contexts.APP("lms").enter():
hooks.Filters.COMMANDS_INIT.add_item(("lms", ("hooks", "lms", "init")))
with hooks.Contexts.APP("cms").enter():
hooks.Filters.COMMANDS_INIT.add_item(("cms", ("hooks", "cms", "init")))
def initialise(runner: BaseJobRunner, limit_to: t.Optional[str] = None) -> None:
fmt.echo_info("Initialising all services...")
filter_context = hooks.Contexts.APP(limit_to).name if limit_to else None
# Pre-init tasks
iter_pre_init_tasks: t.Iterator[
t.Tuple[str, t.Iterable[str]]
] = hooks.Filters.COMMANDS_PRE_INIT.iterate(context=filter_context)
for service, path in iter_pre_init_tasks:
fmt.echo_info(f"Running pre-init task: {'/'.join(path)}")
runner.run_job_from_template(service, *path)
# Init tasks
iter_init_tasks: t.Iterator[
t.Tuple[str, t.Iterable[str]]
] = hooks.Filters.COMMANDS_INIT.iterate(context=filter_context)
for service, path in iter_init_tasks:
fmt.echo_info(f"Running init task: {'/'.join(path)}")
runner.run_job_from_template(service, *path)
fmt.echo_info("All services initialised.")
def create_user_command(
superuser: str,
staff: bool,
username: str,
email: str,
password: t.Optional[str] = None,
) -> str:
command = BASE_OPENEDX_COMMAND
opts = ""
if superuser:
opts += " --superuser"
if staff:
opts += " --staff"
command += """
./manage.py lms manage_user {opts} {username} {email}
"""
if password:
command += """
./manage.py lms shell -c "from django.contrib.auth import get_user_model
u = get_user_model().objects.get(username='{username}')
u.set_password('{password}')
u.save()"
"""
else:
command += """
./manage.py lms changepassword {username}
"""
return command.format(opts=opts, username=username, email=email, password=password)
def import_demo_course(runner: BaseJobRunner) -> None:
runner.run_job_from_template("cms", "hooks", "cms", "importdemocourse")
def set_theme(
theme_name: str, domain_names: t.List[str], runner: BaseJobRunner
) -> None:
"""
For each domain, get or create a Site object and assign the selected theme.
"""
if not domain_names:
return
python_code = "from django.contrib.sites.models import Site"
for domain_name in domain_names:
if len(domain_name) > 50:
fmt.echo_alert(
"Assigning a theme to a site with a long (> 50 characters) domain name."
" The displayed site name will be truncated to 50 characters."
)
python_code += """
print('Assigning theme {theme_name} to {domain_name}...')
site, _ = Site.objects.get_or_create(domain='{domain_name}')
if not site.name:
name_max_length = Site._meta.get_field('name').max_length
name = '{domain_name}'[:name_max_length]
site.name = name
site.save()
site.themes.all().delete()
site.themes.create(theme_dir_name='{theme_name}')
""".format(
theme_name=theme_name, domain_name=domain_name
)
command = BASE_OPENEDX_COMMAND + f'./manage.py lms shell -c "{python_code}"'
runner.run_job("lms", command)
def get_all_openedx_domains(config: Config) -> t.List[str]:
return [
get_typed(config, "LMS_HOST", str),
get_typed(config, "LMS_HOST", str) + ":8000",
get_typed(config, "CMS_HOST", str),
get_typed(config, "CMS_HOST", str) + ":8001",
get_typed(config, "PREVIEW_LMS_HOST", str),
get_typed(config, "PREVIEW_LMS_HOST", str) + ":8000",
]

8
tutor/templates/hooks/cms/importdemocourse
View File

@ -1,8 +0,0 @@
echo "Loading settings $DJANGO_SETTINGS_MODULE"
# Import demo course
git clone https://github.com/openedx/edx-demo-course --branch {{ OPENEDX_COMMON_VERSION }} --depth 1 ../edx-demo-course
python ./manage.py cms import ../data ../edx-demo-course
# Re-index courses
./manage.py cms reindex_course --all --setup