2021-04-06 10:09:00 +00:00
|
|
|
from typing import Dict, Iterator, List, Optional, Tuple, Union
|
2021-02-25 08:09:14 +00:00
|
|
|
|
2021-04-06 10:09:00 +00:00
|
|
|
from . import env, fmt, plugins
|
2021-09-15 09:21:01 +00:00
|
|
|
from .types import Config, get_typed
|
2019-01-22 20:25:04 +00:00
|
|
|
|
2020-03-13 11:09:48 +00:00
|
|
|
BASE_OPENEDX_COMMAND = """
|
2020-06-23 17:11:35 +00:00
|
|
|
export DJANGO_SETTINGS_MODULE=$SERVICE_VARIANT.envs.$SETTINGS
|
2020-03-13 11:09:48 +00:00
|
|
|
echo "Loading settings $DJANGO_SETTINGS_MODULE"
|
|
|
|
"""
|
|
|
|
|
2019-05-05 09:45:24 +00:00
|
|
|
|
2021-03-13 18:46:44 +00:00
|
|
|
class BaseJobRunner:
|
2021-04-06 10:09:00 +00:00
|
|
|
def __init__(self, root: str, config: Config):
|
2019-05-21 09:56:49 +00:00
|
|
|
self.root = root
|
|
|
|
self.config = config
|
|
|
|
|
2021-02-25 08:09:14 +00:00
|
|
|
def run_job_from_template(self, service: str, *path: str) -> None:
|
2019-06-05 17:57:30 +00:00
|
|
|
command = self.render(*path)
|
Improve job running in local and k8s
Running jobs was previously done with "exec". This was because it
allowed us to avoid copying too much container specification information
from the docker-compose/deployments files to the jobs files. However,
this was limiting:
- In order to run a job, the corresponding container had to be running.
This was particularly painful in Kubernetes, where containers are
crashing as long as migrations are not correctly run.
- Containers in which we need to run jobs needed to be present in the
docker-compose/deployments files. This is unnecessary, for example when
mysql is disabled, or in the case of the certbot container.
Now, we create dedicated jobs files, both for local and k8s deployment.
This introduces a little redundancy, but not too much. Note that
dependent containers are not listed in the docker-compose.jobs.yml file,
so an actual platform is still supposed to be running when we launch the
jobs.
This also introduces a subtle change: now, jobs go through the container
entrypoint prior to running. This is probably a good thing, as it will
avoid forgetting about incorrect environment variables.
In k8s, we find ourselves interacting way too much with the kubectl
utility. Parsing output from the CLI is a pain. So we need to switch to
the native kubernetes client library.
2020-03-25 17:47:36 +00:00
|
|
|
self.run_job(service, command)
|
2019-05-21 09:56:49 +00:00
|
|
|
|
2021-02-25 08:09:14 +00:00
|
|
|
def render(self, *path: str) -> str:
|
|
|
|
rendered = env.render_file(self.config, *path).strip()
|
|
|
|
if isinstance(rendered, bytes):
|
|
|
|
raise TypeError("Cannot load job from binary file")
|
|
|
|
return rendered
|
2019-06-05 17:57:30 +00:00
|
|
|
|
2021-02-25 08:09:14 +00:00
|
|
|
def run_job(self, service: str, command: str) -> int:
|
2019-05-21 09:56:49 +00:00
|
|
|
raise NotImplementedError
|
|
|
|
|
2021-02-25 08:09:14 +00:00
|
|
|
def iter_plugin_hooks(
|
|
|
|
self, hook: str
|
|
|
|
) -> Iterator[Tuple[str, Union[Dict[str, str], List[str]]]]:
|
2019-06-06 19:58:21 +00:00
|
|
|
yield from plugins.iter_hooks(self.config, hook)
|
2019-06-05 13:43:51 +00:00
|
|
|
|
2019-05-21 09:56:49 +00:00
|
|
|
|
2021-02-25 08:09:14 +00:00
|
|
|
def initialise(runner: BaseJobRunner, limit_to: Optional[str] = None) -> None:
|
2019-06-05 17:28:06 +00:00
|
|
|
fmt.echo_info("Initialising all services...")
|
2020-06-01 20:38:04 +00:00
|
|
|
if limit_to is None or limit_to == "mysql":
|
|
|
|
runner.run_job_from_template("mysql", "hooks", "mysql", "init")
|
2019-07-05 06:19:23 +00:00
|
|
|
for plugin_name, hook in runner.iter_plugin_hooks("pre-init"):
|
2020-06-01 20:38:04 +00:00
|
|
|
if limit_to is None or limit_to == plugin_name:
|
|
|
|
for service in hook:
|
|
|
|
fmt.echo_info(
|
|
|
|
"Plugin {}: running pre-init for service {}...".format(
|
|
|
|
plugin_name, service
|
|
|
|
)
|
|
|
|
)
|
|
|
|
runner.run_job_from_template(
|
|
|
|
service, plugin_name, "hooks", service, "pre-init"
|
2019-07-05 06:31:28 +00:00
|
|
|
)
|
2019-07-03 14:09:33 +00:00
|
|
|
for service in ["lms", "cms", "forum"]:
|
2020-06-01 20:38:04 +00:00
|
|
|
if limit_to is None or limit_to == service:
|
|
|
|
fmt.echo_info("Initialising {}...".format(service))
|
|
|
|
runner.run_job_from_template(service, "hooks", service, "init")
|
2019-07-02 20:16:44 +00:00
|
|
|
for plugin_name, hook in runner.iter_plugin_hooks("init"):
|
2020-06-01 20:38:04 +00:00
|
|
|
if limit_to is None or limit_to == plugin_name:
|
|
|
|
for service in hook:
|
|
|
|
fmt.echo_info(
|
|
|
|
"Plugin {}: running init for service {}...".format(
|
|
|
|
plugin_name, service
|
|
|
|
)
|
|
|
|
)
|
|
|
|
runner.run_job_from_template(
|
|
|
|
service, plugin_name, "hooks", service, "init"
|
|
|
|
)
|
2019-06-05 17:28:06 +00:00
|
|
|
fmt.echo_info("All services initialised.")
|
2019-01-22 20:25:04 +00:00
|
|
|
|
2019-05-05 09:45:24 +00:00
|
|
|
|
2021-02-25 08:09:14 +00:00
|
|
|
def create_user_command(
|
|
|
|
superuser: str,
|
|
|
|
staff: bool,
|
|
|
|
username: str,
|
|
|
|
email: str,
|
|
|
|
password: Optional[str] = None,
|
|
|
|
) -> str:
|
2020-03-13 11:09:48 +00:00
|
|
|
command = BASE_OPENEDX_COMMAND
|
2019-11-14 08:27:28 +00:00
|
|
|
|
2019-06-05 13:43:51 +00:00
|
|
|
opts = ""
|
2019-03-22 17:50:16 +00:00
|
|
|
if superuser:
|
2019-06-05 13:43:51 +00:00
|
|
|
opts += " --superuser"
|
2019-03-22 17:50:16 +00:00
|
|
|
if staff:
|
2019-06-05 13:43:51 +00:00
|
|
|
opts += " --staff"
|
2019-11-14 08:27:28 +00:00
|
|
|
command += """
|
|
|
|
./manage.py lms manage_user {opts} {username} {email}
|
|
|
|
"""
|
2019-07-11 03:55:12 +00:00
|
|
|
if password:
|
2019-11-14 08:27:28 +00:00
|
|
|
command += """
|
2020-03-13 11:09:48 +00:00
|
|
|
./manage.py lms shell -c "from django.contrib.auth import get_user_model
|
|
|
|
u = get_user_model().objects.get(username='{username}')
|
|
|
|
u.set_password('{password}')
|
|
|
|
u.save()"
|
2019-11-14 08:27:28 +00:00
|
|
|
"""
|
2019-07-11 03:55:12 +00:00
|
|
|
else:
|
2019-11-14 08:27:28 +00:00
|
|
|
command += """
|
|
|
|
./manage.py lms changepassword {username}
|
|
|
|
"""
|
|
|
|
|
2019-07-11 03:55:12 +00:00
|
|
|
return command.format(opts=opts, username=username, email=email, password=password)
|
2019-01-22 20:25:04 +00:00
|
|
|
|
2019-05-05 09:45:24 +00:00
|
|
|
|
2021-02-25 08:09:14 +00:00
|
|
|
def import_demo_course(runner: BaseJobRunner) -> None:
|
Improve job running in local and k8s
Running jobs was previously done with "exec". This was because it
allowed us to avoid copying too much container specification information
from the docker-compose/deployments files to the jobs files. However,
this was limiting:
- In order to run a job, the corresponding container had to be running.
This was particularly painful in Kubernetes, where containers are
crashing as long as migrations are not correctly run.
- Containers in which we need to run jobs needed to be present in the
docker-compose/deployments files. This is unnecessary, for example when
mysql is disabled, or in the case of the certbot container.
Now, we create dedicated jobs files, both for local and k8s deployment.
This introduces a little redundancy, but not too much. Note that
dependent containers are not listed in the docker-compose.jobs.yml file,
so an actual platform is still supposed to be running when we launch the
jobs.
This also introduces a subtle change: now, jobs go through the container
entrypoint prior to running. This is probably a good thing, as it will
avoid forgetting about incorrect environment variables.
In k8s, we find ourselves interacting way too much with the kubectl
utility. Parsing output from the CLI is a pain. So we need to switch to
the native kubernetes client library.
2020-03-25 17:47:36 +00:00
|
|
|
runner.run_job_from_template("cms", "hooks", "cms", "importdemocourse")
|
2020-03-13 11:09:48 +00:00
|
|
|
|
|
|
|
|
2021-09-15 09:21:01 +00:00
|
|
|
def set_theme(theme_name: str, domain_names: List[str], runner: BaseJobRunner) -> None:
|
|
|
|
"""
|
|
|
|
For each domain, get or create a Site object and assign the selected theme.
|
|
|
|
"""
|
|
|
|
if not domain_names:
|
|
|
|
return
|
|
|
|
python_code = "from django.contrib.sites.models import Site"
|
|
|
|
for domain_name in domain_names:
|
|
|
|
python_code += """
|
|
|
|
print('Assigning theme {theme_name} to {domain_name}...')
|
2020-03-13 11:09:48 +00:00
|
|
|
site, _ = Site.objects.get_or_create(domain='{domain_name}')
|
|
|
|
if not site.name:
|
|
|
|
site.name = '{domain_name}'
|
|
|
|
site.save()
|
|
|
|
site.themes.all().delete()
|
2021-09-15 09:21:01 +00:00
|
|
|
site.themes.create(theme_dir_name='{theme_name}')
|
|
|
|
""".format(
|
|
|
|
theme_name=theme_name, domain_name=domain_name
|
|
|
|
)
|
|
|
|
command = BASE_OPENEDX_COMMAND + './manage.py lms shell -c "{python_code}"'.format(
|
|
|
|
python_code=python_code
|
|
|
|
)
|
Improve job running in local and k8s
Running jobs was previously done with "exec". This was because it
allowed us to avoid copying too much container specification information
from the docker-compose/deployments files to the jobs files. However,
this was limiting:
- In order to run a job, the corresponding container had to be running.
This was particularly painful in Kubernetes, where containers are
crashing as long as migrations are not correctly run.
- Containers in which we need to run jobs needed to be present in the
docker-compose/deployments files. This is unnecessary, for example when
mysql is disabled, or in the case of the certbot container.
Now, we create dedicated jobs files, both for local and k8s deployment.
This introduces a little redundancy, but not too much. Note that
dependent containers are not listed in the docker-compose.jobs.yml file,
so an actual platform is still supposed to be running when we launch the
jobs.
This also introduces a subtle change: now, jobs go through the container
entrypoint prior to running. This is probably a good thing, as it will
avoid forgetting about incorrect environment variables.
In k8s, we find ourselves interacting way too much with the kubectl
utility. Parsing output from the CLI is a pain. So we need to switch to
the native kubernetes client library.
2020-03-25 17:47:36 +00:00
|
|
|
runner.run_job("lms", command)
|
2021-09-15 09:21:01 +00:00
|
|
|
|
|
|
|
|
|
|
|
def get_all_openedx_domains(config: Config) -> List[str]:
|
|
|
|
return [
|
|
|
|
get_typed(config, "LMS_HOST", str),
|
|
|
|
get_typed(config, "LMS_HOST", str) + ":8000",
|
|
|
|
get_typed(config, "CMS_HOST", str),
|
|
|
|
get_typed(config, "CMS_HOST", str) + ":8001",
|
|
|
|
]
|