christianlight/tutor
7
0
Fork 0
mirror of https://github.com/ChristianLight/tutor.git synced 2024-06-07 00:20:49 +00:00
Code Issues Releases Activity

refactor: better runner inheritance architecture

Browse Source 
Before, custom `docker_compose_func` arguments had to be passed to job runners.
This was not very elegant. Also, it prevented us from loading custom job files
in development.

Here, we adopt a better object-oriented approach, where context classes are
ordered hierarchically.

This paves the way for loading `dev/docker-compose.jobs.yml` files in `tutor
dev init` commands -- which will be necessary to fix permissions in dev/local
mode.
This commit is contained in:
Régis Behmo 2021-09-27 11:48:17 +02:00 committed by Régis Behmo
parent 079fb1c9ec
commit 02536e0f9f
6 changed files with 136 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
tutor/bindmounts.py
View File

@ -1,22 +1,19 @@
import os
from typing import Callable, List, Tuple
from typing import List, Tuple
import click
from mypy_extensions import VarArg
from .exceptions import TutorError
from .types import Config
from .jobs import BaseComposeJobRunner
from .utils import get_user_id
def create(
root: str,
config: Config,
docker_compose_func: Callable[[str, Config, VarArg(str)], int],
runner: BaseComposeJobRunner,
service: str,
path: str,
) -> str:
volumes_root_path = get_root_path(root)
volumes_root_path = get_root_path(runner.root)
volume_name = get_name(path)
container_volumes_root_path = "/tmp/volumes"
command = """rm -rf {volumes_path}/{volume_name}
@ -33,9 +30,7 @@ chown -R {user_id} {volumes_path}/{volume_name}""".format(
if not os.path.exists(volumes_root_path):
os.makedirs(volumes_root_path)
docker_compose_func(
root,
config,
runner.docker_compose(
"run",
"--rm",
"--no-deps",

72
tutor/commands/compose.py
View File

@ -1,8 +1,7 @@
import os
from typing import Callable, List
from typing import List
import click
from mypy_extensions import VarArg
from .. import bindmounts
from .. import config as tutor_config
@ -12,19 +11,10 @@ from .. import fmt
from .. import jobs
from ..types import Config
from .. import utils
from .context import Context
from .context import BaseJobContext
class ComposeJobRunner(jobs.BaseJobRunner):
def __init__(
self,
root: str,
config: Config,
docker_compose_func: Callable[[str, Config, VarArg(str)], int],
):
super().__init__(root, config)
self.docker_compose_func = docker_compose_func
class ComposeJobRunner(jobs.BaseComposeJobRunner):
def run_job(self, service: str, command: str) -> int:
"""
Run the "{{ service }}-job" service from local/docker-compose.jobs.yml with the
@ -45,9 +35,7 @@ class ComposeJobRunner(jobs.BaseJobRunner):
if not utils.is_a_tty():
run_command += ["-T"]
job_service_name = "{}-job".format(service)
return self.docker_compose_func(
self.root,
self.config,
return self.docker_compose(
*run_command,
job_service_name,
"sh",
@ -57,6 +45,11 @@ class ComposeJobRunner(jobs.BaseJobRunner):
)
class BaseComposeContext(BaseJobContext):
def job_runner(self, config: Config) -> ComposeJobRunner:
raise NotImplementedError
@click.command(
short_help="Run all or a selection of services.",
help="Run all or a selection of services. Docker images will be rebuilt where necessary.",
@ -64,22 +57,22 @@ class ComposeJobRunner(jobs.BaseJobRunner):
@click.option("-d", "--detach", is_flag=True, help="Start in daemon mode")
@click.argument("services", metavar="service", nargs=-1)
@click.pass_obj
def start(context: Context, detach: bool, services: List[str]) -> None:
def start(context: BaseComposeContext, detach: bool, services: List[str]) -> None:
command = ["up", "--remove-orphans", "--build"]
if detach:
command.append("-d")
config = tutor_config.load(context.root)
# Start services
context.docker_compose(context.root, config, *command, *services)
config = tutor_config.load(context.root)
context.job_runner(config).docker_compose(*command, *services)
@click.command(help="Stop a running platform")
@click.argument("services", metavar="service", nargs=-1)
@click.pass_obj
def stop(context: Context, services: List[str]) -> None:
def stop(context: BaseComposeContext, services: List[str]) -> None:
config = tutor_config.load(context.root)
context.docker_compose(context.root, config, "stop", *services)
context.job_runner(config).docker_compose("stop", *services)
@click.command(
@ -103,7 +96,7 @@ fully stop the platform, use the 'reboot' command.""",
)
@click.argument("services", metavar="service", nargs=-1)
@click.pass_obj
def restart(context: Context, services: List[str]) -> None:
def restart(context: BaseComposeContext, services: List[str]) -> None:
config = tutor_config.load(context.root)
command = ["restart"]
if "all" in services:
@ -117,15 +110,15 @@ def restart(context: Context, services: List[str]) -> None:
command += ["cms", "cms-worker"]
else:
command.append(service)
context.docker_compose(context.root, config, *command)
context.job_runner(config).docker_compose(*command)
@click.command(help="Initialise all applications")
@click.option("-l", "--limit", help="Limit initialisation to this service or plugin")
@click.pass_obj
def init(context: Context, limit: str) -> None:
def init(context: BaseComposeContext, limit: str) -> None:
config = tutor_config.load(context.root)
runner = ComposeJobRunner(context.root, config, context.docker_compose)
runner = context.job_runner(config)
jobs.initialise(runner, limit_to=limit)
@ -141,10 +134,15 @@ def init(context: Context, limit: str) -> None:
@click.argument("email")
@click.pass_obj
def createuser(
context: Context, superuser: str, staff: bool, password: str, name: str, email: str
context: BaseComposeContext,
superuser: str,
staff: bool,
password: str,
name: str,
email: str,
) -> None:
config = tutor_config.load(context.root)
runner = ComposeJobRunner(context.root, config, context.docker_compose)
runner = context.job_runner(config)
command = jobs.create_user_command(superuser, staff, name, email, password=password)
runner.run_job("lms", command)
@ -164,18 +162,18 @@ def createuser(
)
@click.argument("theme_name")
@click.pass_obj
def settheme(context: Context, domains: List[str], theme_name: str) -> None:
def settheme(context: BaseComposeContext, domains: List[str], theme_name: str) -> None:
config = tutor_config.load(context.root)
runner = ComposeJobRunner(context.root, config, context.docker_compose)
runner = context.job_runner(config)
domains = domains or jobs.get_all_openedx_domains(config)
jobs.set_theme(theme_name, domains, runner)
@click.command(help="Import the demo course")
@click.pass_obj
def importdemocourse(context: Context) -> None:
def importdemocourse(context: BaseComposeContext) -> None:
config = tutor_config.load(context.root)
runner = ComposeJobRunner(context.root, config, context.docker_compose)
runner = context.job_runner(config)
fmt.echo_info("Importing demo course")
jobs.import_demo_course(runner)
@ -207,11 +205,9 @@ def run(context: click.Context, args: List[str]) -> None:
)
@click.argument("path")
@click.pass_obj
def bindmount_command(context: Context, service: str, path: str) -> None:
def bindmount_command(context: BaseComposeContext, service: str, path: str) -> None:
config = tutor_config.load(context.root)
host_path = bindmounts.create(
context.root, config, context.docker_compose, service, path
)
host_path = bindmounts.create(context.job_runner(config), service, path)
fmt.echo_info(
"Bind-mount volume created at {}. You can now use it in all `local` and `dev` commands with the `--volume={}` option.".format(
host_path, path
@ -265,7 +261,7 @@ def logs(context: click.Context, follow: bool, tail: bool, service: str) -> None
@click.argument("command")
@click.argument("args", nargs=-1)
@click.pass_obj
def dc_command(context: Context, command: str, args: List[str]) -> None:
def dc_command(context: BaseComposeContext, command: str, args: List[str]) -> None:
config = tutor_config.load(context.root)
volumes, non_volume_args = bindmounts.parse_volumes(args)
volume_args = []
@ -282,9 +278,7 @@ def dc_command(context: Context, command: str, args: List[str]) -> None:
)
volume_arg = "{}:{}".format(host_bind_path, volume_arg)
volume_args += ["--volume", volume_arg]
context.docker_compose(
context.root, config, command, *volume_args, *non_volume_args
)
context.job_runner(config).docker_compose(command, *volume_args, *non_volume_args)
def add_commands(command_group: click.Group) -> None:

31
tutor/commands/context.py
View File

@ -1,15 +1,30 @@
from ..jobs import BaseJobRunner
from ..types import Config
def unimplemented_docker_compose(root: str, config: Config, *command: str) -> int:
raise NotImplementedError
# pylint: disable=too-few-public-methods
class Context:
"""
Context object that is passed to all subcommands.
The project `root` is passed to all subcommands of `tutor`; that's because
it is defined as an argument of the top-level command. For instance:
tutor --root=... local run ...
"""
def __init__(self, root: str) -> None:
self.root = root
self.docker_compose_func = unimplemented_docker_compose
def docker_compose(self, root: str, config: Config, *command: str) -> int:
return self.docker_compose_func(root, config, *command)
class BaseJobContext(Context):
"""
Specialized context that subcommands may use.
For instance `dev`, `local` and `k8s` define custom runners to run jobs.
"""
def job_runner(self, config: Config) -> BaseJobRunner:
"""
Return a runner capable of running docker-compose/kubectl commands.
"""
raise NotImplementedError

57
tutor/commands/dev.py
View File

@ -9,36 +9,43 @@ from .. import fmt
from ..types import Config
from .. import utils
from . import compose
from .context import Context
def docker_compose(root: str, config: Config, *command: str) -> int:
"""
Run docker-compose with dev arguments.
"""
args = []
for folder in ["local", "dev"]:
# Add docker-compose.yml and docker-compose.override.yml (if it exists)
# from "local" and "dev" folders (but not docker-compose.prod.yml)
args += [
"-f",
tutor_env.pathjoin(root, folder, "docker-compose.yml"),
]
override_path = tutor_env.pathjoin(root, folder, "docker-compose.override.yml")
if os.path.exists(override_path):
args += ["-f", override_path]
return utils.docker_compose(
*args,
"--project-name",
str(config["DEV_PROJECT_NAME"]),
*command,
)
class DevJobRunner(compose.ComposeJobRunner):
def docker_compose(self, *command: str) -> int:
"""
Run docker-compose with dev arguments.
"""
args = []
for folder in ["local", "dev"]:
# Add docker-compose.yml and docker-compose.override.yml (if it exists)
# from "local" and "dev" folders (but not docker-compose.prod.yml)
args += [
"-f",
tutor_env.pathjoin(self.root, folder, "docker-compose.yml"),
]
override_path = tutor_env.pathjoin(
self.root, folder, "docker-compose.override.yml"
)
if os.path.exists(override_path):
args += ["-f", override_path]
return utils.docker_compose(
*args,
"--project-name",
str(self.config["DEV_PROJECT_NAME"]),
*command,
)
class DevContext(compose.BaseComposeContext):
def job_runner(self, config: Config) -> DevJobRunner:
return DevJobRunner(self.root, config)
@click.group(help="Run Open edX locally with development settings")
@click.pass_obj
def dev(context: Context) -> None:
context.docker_compose_func = docker_compose
@click.pass_context
def dev(context: click.Context) -> None:
context.obj = DevContext(context.obj.root)
@click.command(

51
tutor/commands/local.py
View File

@ -11,33 +11,40 @@ from .. import utils
from .. import exceptions
from . import compose
from .config import save as config_save_command
from .context import Context
def docker_compose(root: str, config: Config, *command: str) -> int:
"""
Run docker-compose with local and production yml files.
"""
args = []
override_path = tutor_env.pathjoin(root, "local", "docker-compose.override.yml")
if os.path.exists(override_path):
args += ["-f", override_path]
return utils.docker_compose(
"-f",
tutor_env.pathjoin(root, "local", "docker-compose.yml"),
"-f",
tutor_env.pathjoin(root, "local", "docker-compose.prod.yml"),
*args,
"--project-name",
get_typed(config, "LOCAL_PROJECT_NAME", str),
*command
)
class LocalJobRunner(compose.ComposeJobRunner):
def docker_compose(self, *command: str) -> int:
"""
Run docker-compose with local and production yml files.
"""
args = []
override_path = tutor_env.pathjoin(
self.root, "local", "docker-compose.override.yml"
)
if os.path.exists(override_path):
args += ["-f", override_path]
return utils.docker_compose(
"-f",
tutor_env.pathjoin(self.root, "local", "docker-compose.yml"),
"-f",
tutor_env.pathjoin(self.root, "local", "docker-compose.prod.yml"),
*args,
"--project-name",
get_typed(self.config, "LOCAL_PROJECT_NAME", str),
*command
)
class LocalContext(compose.BaseComposeContext):
def job_runner(self, config: Config) -> LocalJobRunner:
return LocalJobRunner(self.root, config)
@click.group(help="Run Open edX locally with docker-compose")
@click.pass_obj
def local(context: Context) -> None:
context.docker_compose_func = docker_compose
@click.pass_context
def local(context: click.Context) -> None:
context.obj = LocalContext(context.obj.root)
@click.command(help="Configure and run Open edX from scratch")

14
tutor/jobs.py
View File

@ -10,6 +10,10 @@ echo "Loading settings $DJANGO_SETTINGS_MODULE"
class BaseJobRunner:
"""
A job runner is responsible for getting a certain task to complete.
"""
def __init__(self, root: str, config: Config):
self.root = root
self.config = config
@ -25,6 +29,11 @@ class BaseJobRunner:
return rendered
def run_job(self, service: str, command: str) -> int:
"""
Given a (potentially large) string command, run it with the
corresponding service. Implementations will differ depending on the
deployment strategy.
"""
raise NotImplementedError
def iter_plugin_hooks(
@ -33,6 +42,11 @@ class BaseJobRunner:
yield from plugins.iter_hooks(self.config, hook)
class BaseComposeJobRunner(BaseJobRunner):
def docker_compose(self, *command: str) -> int:
raise NotImplementedError
def initialise(runner: BaseJobRunner, limit_to: Optional[str] = None) -> None:
fmt.echo_info("Initialising all services...")
if limit_to is None or limit_to == "mysql":