christianlight/tutor
6
0
Fork 0
mirror of https://github.com/ChristianLight/tutor.git synced 2025-01-12 01:45:39 +00:00
Code Issues Releases Activity

Merge remote-tracking branch 'origin/master' into nightly

Browse Source
This commit is contained in:
Overhang.IO 2021-11-01 16:40:25 +00:00
commit c9bde8b1ec
8 changed files with 155 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGELOG.md
View File

@ -4,6 +4,9 @@ Note: Breaking changes between versions are indicated by "💥".
## Unreleased
- [Feature] Make it possible to override job configuration in development: if they exist, `dev/docker-compose.jobs.yml` and `dev/docker-compose.jobs.override.yml` will be loaded when running jobs.
- [Improvement] Faster `tutor local start` by building only necessary images.
## v12.1.5 (2021-10-25)
- 💥[Improvement] Change the `settheme` command such that, by default, a custom theme is assigned to the LMS and the CMS, both in production and development mode.

15
tutor/bindmounts.py
View File

@ -1,22 +1,19 @@
import os
from typing import Callable, List, Tuple
from typing import List, Tuple
import click
from mypy_extensions import VarArg
from .exceptions import TutorError
from .types import Config
from .jobs import BaseComposeJobRunner
from .utils import get_user_id
def create(
root: str,
config: Config,
docker_compose_func: Callable[[str, Config, VarArg(str)], int],
runner: BaseComposeJobRunner,
service: str,
path: str,
) -> str:
volumes_root_path = get_root_path(root)
volumes_root_path = get_root_path(runner.root)
volume_name = get_name(path)
container_volumes_root_path = "/tmp/volumes"
command = """rm -rf {volumes_path}/{volume_name}
@ -33,9 +30,7 @@ chown -R {user_id} {volumes_path}/{volume_name}""".format(
if not os.path.exists(volumes_root_path):
os.makedirs(volumes_root_path)
docker_compose_func(
root,
config,
runner.docker_compose(
"run",
"--rm",
"--no-deps",

104
tutor/commands/compose.py
View File

@ -1,8 +1,7 @@
import os
from typing import Callable, List
from typing import List
import click
from mypy_extensions import VarArg
from .. import bindmounts
from .. import config as tutor_config
@ -12,18 +11,27 @@ from .. import fmt
from .. import jobs
from ..types import Config
from .. import utils
from .context import Context
from .context import BaseJobContext
class ComposeJobRunner(jobs.BaseJobRunner):
def __init__(
self,
root: str,
config: Config,
docker_compose_func: Callable[[str, Config, VarArg(str)], int],
):
class ComposeJobRunner(jobs.BaseComposeJobRunner):
def __init__(self, root: str, config: Config):
super().__init__(root, config)
self.docker_compose_func = docker_compose_func
self.project_name = ""
self.docker_compose_files: List[str] = []
self.docker_compose_job_files: List[str] = []
def docker_compose(self, *command: str) -> int:
"""
Run docker-compose with the right yml files.
"""
args = []
for docker_compose_path in self.docker_compose_files:
if os.path.exists(docker_compose_path):
args += ["-f", docker_compose_path]
return utils.docker_compose(
*args, "--project-name", self.project_name, *command
)
def run_job(self, service: str, command: str) -> int:
"""
@ -32,22 +40,16 @@ class ComposeJobRunner(jobs.BaseJobRunner):
service does not exist, run the service from good old regular
docker-compose.yml.
"""
run_command = [
"-f",
tutor_env.pathjoin(self.root, "local", "docker-compose.jobs.yml"),
]
override_path = tutor_env.pathjoin(
self.root, "local", "docker-compose.jobs.override.yml"
)
if os.path.exists(override_path):
run_command += ["-f", override_path]
run_command = []
for docker_compose_path in self.docker_compose_job_files:
path = tutor_env.pathjoin(self.root, docker_compose_path)
if os.path.exists(path):
run_command += ["-f", path]
run_command += ["run", "--rm"]
if not utils.is_a_tty():
run_command += ["-T"]
job_service_name = "{}-job".format(service)
return self.docker_compose_func(
self.root,
self.config,
return self.docker_compose(
*run_command,
job_service_name,
"sh",
@ -57,6 +59,11 @@ class ComposeJobRunner(jobs.BaseJobRunner):
)
class BaseComposeContext(BaseJobContext):
def job_runner(self, config: Config) -> ComposeJobRunner:
raise NotImplementedError
@click.command(
short_help="Run all or a selection of services.",
help="Run all or a selection of services. Docker images will be rebuilt where necessary.",
@ -64,24 +71,22 @@ class ComposeJobRunner(jobs.BaseJobRunner):
@click.option("-d", "--detach", is_flag=True, help="Start in daemon mode")
@click.argument("services", metavar="service", nargs=-1)
@click.pass_obj
def start(context: Context, detach: bool, services: List[str]) -> None:
command = ["up", "--remove-orphans"]
def start(context: BaseComposeContext, detach: bool, services: List[str]) -> None:
command = ["up", "--remove-orphans", "--build"]
if detach:
command.append("-d")
config = tutor_config.load(context.root)
# Rebuild Docker images with a `build: ...` context.
context.docker_compose(context.root, config, "build")
# Start services
context.docker_compose(context.root, config, *command, *services)
config = tutor_config.load(context.root)
context.job_runner(config).docker_compose(*command, *services)
@click.command(help="Stop a running platform")
@click.argument("services", metavar="service", nargs=-1)
@click.pass_obj
def stop(context: Context, services: List[str]) -> None:
def stop(context: BaseComposeContext, services: List[str]) -> None:
config = tutor_config.load(context.root)
context.docker_compose(context.root, config, "stop", *services)
context.job_runner(config).docker_compose("stop", *services)
@click.command(
@ -105,7 +110,7 @@ fully stop the platform, use the 'reboot' command.""",
)
@click.argument("services", metavar="service", nargs=-1)
@click.pass_obj
def restart(context: Context, services: List[str]) -> None:
def restart(context: BaseComposeContext, services: List[str]) -> None:
config = tutor_config.load(context.root)
command = ["restart"]
if "all" in services:
@ -119,15 +124,15 @@ def restart(context: Context, services: List[str]) -> None:
command += ["cms", "cms-worker"]
else:
command.append(service)
context.docker_compose(context.root, config, *command)
context.job_runner(config).docker_compose(*command)
@click.command(help="Initialise all applications")
@click.option("-l", "--limit", help="Limit initialisation to this service or plugin")
@click.pass_obj
def init(context: Context, limit: str) -> None:
def init(context: BaseComposeContext, limit: str) -> None:
config = tutor_config.load(context.root)
runner = ComposeJobRunner(context.root, config, context.docker_compose)
runner = context.job_runner(config)
jobs.initialise(runner, limit_to=limit)
@ -143,10 +148,15 @@ def init(context: Context, limit: str) -> None:
@click.argument("email")
@click.pass_obj
def createuser(
context: Context, superuser: str, staff: bool, password: str, name: str, email: str
context: BaseComposeContext,
superuser: str,
staff: bool,
password: str,
name: str,
email: str,
) -> None:
config = tutor_config.load(context.root)
runner = ComposeJobRunner(context.root, config, context.docker_compose)
runner = context.job_runner(config)
command = jobs.create_user_command(superuser, staff, name, email, password=password)
runner.run_job("lms", command)
@ -166,18 +176,18 @@ def createuser(
)
@click.argument("theme_name")
@click.pass_obj
def settheme(context: Context, domains: List[str], theme_name: str) -> None:
def settheme(context: BaseComposeContext, domains: List[str], theme_name: str) -> None:
config = tutor_config.load(context.root)
runner = ComposeJobRunner(context.root, config, context.docker_compose)
runner = context.job_runner(config)
domains = domains or jobs.get_all_openedx_domains(config)
jobs.set_theme(theme_name, domains, runner)
@click.command(help="Import the demo course")
@click.pass_obj
def importdemocourse(context: Context) -> None:
def importdemocourse(context: BaseComposeContext) -> None:
config = tutor_config.load(context.root)
runner = ComposeJobRunner(context.root, config, context.docker_compose)
runner = context.job_runner(config)
fmt.echo_info("Importing demo course")
jobs.import_demo_course(runner)
@ -209,11 +219,9 @@ def run(context: click.Context, args: List[str]) -> None:
)
@click.argument("path")
@click.pass_obj
def bindmount_command(context: Context, service: str, path: str) -> None:
def bindmount_command(context: BaseComposeContext, service: str, path: str) -> None:
config = tutor_config.load(context.root)
host_path = bindmounts.create(
context.root, config, context.docker_compose, service, path
)
host_path = bindmounts.create(context.job_runner(config), service, path)
fmt.echo_info(
"Bind-mount volume created at {}. You can now use it in all `local` and `dev` commands with the `--volume={}` option.".format(
host_path, path
@ -267,7 +275,7 @@ def logs(context: click.Context, follow: bool, tail: bool, service: str) -> None
@click.argument("command")
@click.argument("args", nargs=-1)
@click.pass_obj
def dc_command(context: Context, command: str, args: List[str]) -> None:
def dc_command(context: BaseComposeContext, command: str, args: List[str]) -> None:
config = tutor_config.load(context.root)
volumes, non_volume_args = bindmounts.parse_volumes(args)
volume_args = []
@ -284,9 +292,7 @@ def dc_command(context: Context, command: str, args: List[str]) -> None:
)
volume_arg = "{}:{}".format(host_bind_path, volume_arg)
volume_args += ["--volume", volume_arg]
context.docker_compose(
context.root, config, command, *volume_args, *non_volume_args
)
context.job_runner(config).docker_compose(command, *volume_args, *non_volume_args)
def add_commands(command_group: click.Group) -> None:

31
tutor/commands/context.py
View File

@ -1,15 +1,30 @@
from ..jobs import BaseJobRunner
from ..types import Config
def unimplemented_docker_compose(root: str, config: Config, *command: str) -> int:
raise NotImplementedError
# pylint: disable=too-few-public-methods
class Context:
"""
Context object that is passed to all subcommands.
The project `root` is passed to all subcommands of `tutor`; that's because
it is defined as an argument of the top-level command. For instance:
tutor --root=... local run ...
"""
def __init__(self, root: str) -> None:
self.root = root
self.docker_compose_func = unimplemented_docker_compose
def docker_compose(self, root: str, config: Config, *command: str) -> int:
return self.docker_compose_func(root, config, *command)
class BaseJobContext(Context):
"""
Specialized context that subcommands may use.
For instance `dev`, `local` and `k8s` define custom runners to run jobs.
"""
def job_runner(self, config: Config) -> BaseJobRunner:
"""
Return a runner capable of running docker-compose/kubectl commands.
"""
raise NotImplementedError

54
tutor/commands/dev.py
View File

@ -1,4 +1,3 @@
import os
from typing import List
import click
@ -6,39 +5,40 @@ import click
from .. import config as tutor_config
from .. import env as tutor_env
from .. import fmt
from ..types import Config
from .. import utils
from ..types import Config, get_typed
from . import compose
from .context import Context
def docker_compose(root: str, config: Config, *command: str) -> int:
"""
Run docker-compose with dev arguments.
"""
args = []
for folder in ["local", "dev"]:
# Add docker-compose.yml and docker-compose.override.yml (if it exists)
# from "local" and "dev" folders (but not docker-compose.prod.yml)
args += [
"-f",
tutor_env.pathjoin(root, folder, "docker-compose.yml"),
class DevJobRunner(compose.ComposeJobRunner):
def __init__(self, root: str, config: Config):
"""
Load docker-compose files from dev/ and local/
"""
super().__init__(root, config)
self.project_name = get_typed(self.config, "DEV_PROJECT_NAME", str)
self.docker_compose_files += [
tutor_env.pathjoin(self.root, "local", "docker-compose.yml"),
tutor_env.pathjoin(self.root, "dev", "docker-compose.yml"),
tutor_env.pathjoin(self.root, "local", "docker-compose.override.yml"),
tutor_env.pathjoin(self.root, "dev", "docker-compose.override.yml"),
]
override_path = tutor_env.pathjoin(root, folder, "docker-compose.override.yml")
if os.path.exists(override_path):
args += ["-f", override_path]
return utils.docker_compose(
*args,
"--project-name",
str(config["DEV_PROJECT_NAME"]),
*command,
)
self.docker_compose_job_files += [
tutor_env.pathjoin(self.root, "local", "docker-compose.jobs.yml"),
tutor_env.pathjoin(self.root, "dev", "docker-compose.jobs.yml"),
tutor_env.pathjoin(self.root, "local", "docker-compose.jobs.override.yml"),
tutor_env.pathjoin(self.root, "dev", "docker-compose.jobs.override.yml"),
]
class DevContext(compose.BaseComposeContext):
def job_runner(self, config: Config) -> DevJobRunner:
return DevJobRunner(self.root, config)
@click.group(help="Run Open edX locally with development settings")
@click.pass_obj
def dev(context: Context) -> None:
context.docker_compose_func = docker_compose
@click.pass_context
def dev(context: click.Context) -> None:
context.obj = DevContext(context.obj.root)
@click.command(

49
tutor/commands/local.py
View File

@ -1,4 +1,3 @@
import os
from time import sleep
import click
@ -6,38 +5,40 @@ import click
from .. import config as tutor_config
from .. import env as tutor_env
from .. import fmt
from ..types import get_typed, Config
from ..types import Config, get_typed
from .. import utils
from .. import exceptions
from . import compose
from .config import save as config_save_command
from .context import Context
def docker_compose(root: str, config: Config, *command: str) -> int:
"""
Run docker-compose with local and production yml files.
"""
args = []
override_path = tutor_env.pathjoin(root, "local", "docker-compose.override.yml")
if os.path.exists(override_path):
args += ["-f", override_path]
return utils.docker_compose(
"-f",
tutor_env.pathjoin(root, "local", "docker-compose.yml"),
"-f",
tutor_env.pathjoin(root, "local", "docker-compose.prod.yml"),
*args,
"--project-name",
get_typed(config, "LOCAL_PROJECT_NAME", str),
*command
)
class LocalJobRunner(compose.ComposeJobRunner):
def __init__(self, root: str, config: Config):
"""
Load docker-compose files from local/.
"""
super().__init__(root, config)
self.project_name = get_typed(self.config, "LOCAL_PROJECT_NAME", str)
self.docker_compose_files += [
tutor_env.pathjoin(self.root, "local", "docker-compose.yml"),
tutor_env.pathjoin(self.root, "local", "docker-compose.prod.yml"),
tutor_env.pathjoin(self.root, "local", "docker-compose.override.yml"),
]
self.docker_compose_job_files += [
tutor_env.pathjoin(self.root, "local", "docker-compose.jobs.yml"),
tutor_env.pathjoin(self.root, "local", "docker-compose.jobs.override.yml"),
]
class LocalContext(compose.BaseComposeContext):
def job_runner(self, config: Config) -> LocalJobRunner:
return LocalJobRunner(self.root, config)
@click.group(help="Run Open edX locally with docker-compose")
@click.pass_obj
def local(context: Context) -> None:
context.docker_compose_func = docker_compose
@click.pass_context
def local(context: click.Context) -> None:
context.obj = LocalContext(context.obj.root)
@click.command(help="Configure and run Open edX from scratch")

14
tutor/jobs.py
View File

@ -10,6 +10,10 @@ echo "Loading settings $DJANGO_SETTINGS_MODULE"
class BaseJobRunner:
"""
A job runner is responsible for getting a certain task to complete.
"""
def __init__(self, root: str, config: Config):
self.root = root
self.config = config
@ -25,6 +29,11 @@ class BaseJobRunner:
return rendered
def run_job(self, service: str, command: str) -> int:
"""
Given a (potentially large) string command, run it with the
corresponding service. Implementations will differ depending on the
deployment strategy.
"""
raise NotImplementedError
def iter_plugin_hooks(
@ -33,6 +42,11 @@ class BaseJobRunner:
yield from plugins.iter_hooks(self.config, hook)
class BaseComposeJobRunner(BaseJobRunner):
def docker_compose(self, *command: str) -> int:
raise NotImplementedError
def initialise(runner: BaseJobRunner, limit_to: Optional[str] = None) -> None:
fmt.echo_info("Initialising all services...")
if limit_to is None or limit_to == "mysql":

3
tutor/templates/dev/docker-compose.jobs.yml Normal file
View File

@ -0,0 +1,3 @@
version: "3.7"
services: {% if not patch("dev-docker-compose-jobs-services") %}{}{% endif %}
{{ patch("dev-docker-compose-jobs-services")|indent(4) }}