2020-01-08 18:38:13 +00:00
|
|
|
import click
|
|
|
|
|
|
|
|
from .. import config as tutor_config
|
Improve job running in local and k8s
Running jobs was previously done with "exec". This was because it
allowed us to avoid copying too much container specification information
from the docker-compose/deployments files to the jobs files. However,
this was limiting:
- In order to run a job, the corresponding container had to be running.
This was particularly painful in Kubernetes, where containers are
crashing as long as migrations are not correctly run.
- Containers in which we need to run jobs needed to be present in the
docker-compose/deployments files. This is unnecessary, for example when
mysql is disabled, or in the case of the certbot container.
Now, we create dedicated jobs files, both for local and k8s deployment.
This introduces a little redundancy, but not too much. Note that
dependent containers are not listed in the docker-compose.jobs.yml file,
so an actual platform is still supposed to be running when we launch the
jobs.
This also introduces a subtle change: now, jobs go through the container
entrypoint prior to running. This is probably a good thing, as it will
avoid forgetting about incorrect environment variables.
In k8s, we find ourselves interacting way too much with the kubectl
utility. Parsing output from the CLI is a pain. So we need to switch to
the native kubernetes client library.
2020-03-25 17:47:36 +00:00
|
|
|
from .. import env as tutor_env
|
2020-01-08 18:38:13 +00:00
|
|
|
from .. import fmt
|
|
|
|
from .. import scripts
|
Improve job running in local and k8s
Running jobs was previously done with "exec". This was because it
allowed us to avoid copying too much container specification information
from the docker-compose/deployments files to the jobs files. However,
this was limiting:
- In order to run a job, the corresponding container had to be running.
This was particularly painful in Kubernetes, where containers are
crashing as long as migrations are not correctly run.
- Containers in which we need to run jobs needed to be present in the
docker-compose/deployments files. This is unnecessary, for example when
mysql is disabled, or in the case of the certbot container.
Now, we create dedicated jobs files, both for local and k8s deployment.
This introduces a little redundancy, but not too much. Note that
dependent containers are not listed in the docker-compose.jobs.yml file,
so an actual platform is still supposed to be running when we launch the
jobs.
This also introduces a subtle change: now, jobs go through the container
entrypoint prior to running. This is probably a good thing, as it will
avoid forgetting about incorrect environment variables.
In k8s, we find ourselves interacting way too much with the kubectl
utility. Parsing output from the CLI is a pain. So we need to switch to
the native kubernetes client library.
2020-03-25 17:47:36 +00:00
|
|
|
from .. import serialize
|
2020-03-27 08:59:11 +00:00
|
|
|
from .. import utils
|
2020-01-08 18:38:13 +00:00
|
|
|
|
|
|
|
|
|
|
|
class ScriptRunner(scripts.BaseRunner):
|
|
|
|
def __init__(self, root, config, docker_compose_func):
|
|
|
|
super().__init__(root, config)
|
|
|
|
self.docker_compose_func = docker_compose_func
|
|
|
|
|
Improve job running in local and k8s
Running jobs was previously done with "exec". This was because it
allowed us to avoid copying too much container specification information
from the docker-compose/deployments files to the jobs files. However,
this was limiting:
- In order to run a job, the corresponding container had to be running.
This was particularly painful in Kubernetes, where containers are
crashing as long as migrations are not correctly run.
- Containers in which we need to run jobs needed to be present in the
docker-compose/deployments files. This is unnecessary, for example when
mysql is disabled, or in the case of the certbot container.
Now, we create dedicated jobs files, both for local and k8s deployment.
This introduces a little redundancy, but not too much. Note that
dependent containers are not listed in the docker-compose.jobs.yml file,
so an actual platform is still supposed to be running when we launch the
jobs.
This also introduces a subtle change: now, jobs go through the container
entrypoint prior to running. This is probably a good thing, as it will
avoid forgetting about incorrect environment variables.
In k8s, we find ourselves interacting way too much with the kubectl
utility. Parsing output from the CLI is a pain. So we need to switch to
the native kubernetes client library.
2020-03-25 17:47:36 +00:00
|
|
|
def run_job(self, service, command):
|
|
|
|
"""
|
|
|
|
Run the "{{ service }}-job" service from local/docker-compose.jobs.yml with the
|
|
|
|
specified command. For backward-compatibility reasons, if the corresponding
|
|
|
|
service does not exist, run the service from good old regular
|
|
|
|
docker-compose.yml.
|
|
|
|
"""
|
|
|
|
jobs_path = tutor_env.pathjoin(self.root, "local", "docker-compose.jobs.yml")
|
|
|
|
job_service_name = "{}-job".format(service)
|
2020-04-02 10:36:43 +00:00
|
|
|
opts = [] if utils.is_a_tty() else ["-T"]
|
Improve job running in local and k8s
Running jobs was previously done with "exec". This was because it
allowed us to avoid copying too much container specification information
from the docker-compose/deployments files to the jobs files. However,
this was limiting:
- In order to run a job, the corresponding container had to be running.
This was particularly painful in Kubernetes, where containers are
crashing as long as migrations are not correctly run.
- Containers in which we need to run jobs needed to be present in the
docker-compose/deployments files. This is unnecessary, for example when
mysql is disabled, or in the case of the certbot container.
Now, we create dedicated jobs files, both for local and k8s deployment.
This introduces a little redundancy, but not too much. Note that
dependent containers are not listed in the docker-compose.jobs.yml file,
so an actual platform is still supposed to be running when we launch the
jobs.
This also introduces a subtle change: now, jobs go through the container
entrypoint prior to running. This is probably a good thing, as it will
avoid forgetting about incorrect environment variables.
In k8s, we find ourselves interacting way too much with the kubectl
utility. Parsing output from the CLI is a pain. So we need to switch to
the native kubernetes client library.
2020-03-25 17:47:36 +00:00
|
|
|
if job_service_name in serialize.load(open(jobs_path).read())["services"]:
|
|
|
|
self.docker_compose_func(
|
|
|
|
self.root,
|
|
|
|
self.config,
|
|
|
|
"-f",
|
|
|
|
jobs_path,
|
|
|
|
"run",
|
|
|
|
*opts,
|
|
|
|
"--rm",
|
|
|
|
job_service_name,
|
|
|
|
"sh",
|
|
|
|
"-e",
|
|
|
|
"-c",
|
|
|
|
command,
|
|
|
|
)
|
|
|
|
else:
|
|
|
|
fmt.echo_alert(
|
|
|
|
(
|
|
|
|
"The '{job_service_name}' service does not exist in {jobs_path}. "
|
|
|
|
"This might be caused by an older plugin. Tutor switched to a job "
|
|
|
|
"runner model for running one-time commands, such as database"
|
|
|
|
" initialisation. For the record, this is the command that we are "
|
|
|
|
"running:\n"
|
|
|
|
"\n"
|
|
|
|
" {command}\n"
|
|
|
|
"\n"
|
|
|
|
"Old-style job running will be deprecated soon. Please inform "
|
|
|
|
"your plugin maintainer!"
|
|
|
|
).format(
|
|
|
|
job_service_name=job_service_name,
|
|
|
|
jobs_path=jobs_path,
|
|
|
|
command=command.replace("\n", "\n "),
|
|
|
|
)
|
|
|
|
)
|
|
|
|
self.docker_compose_func(
|
|
|
|
self.root,
|
|
|
|
self.config,
|
|
|
|
"run",
|
|
|
|
*opts,
|
|
|
|
"--rm",
|
|
|
|
service,
|
|
|
|
"sh",
|
|
|
|
"-e",
|
|
|
|
"-c",
|
|
|
|
command,
|
|
|
|
)
|
2020-01-08 18:38:13 +00:00
|
|
|
|
|
|
|
|
|
|
|
@click.command(help="Run all or a selection of configured Open edX services")
|
|
|
|
@click.option("-d", "--detach", is_flag=True, help="Start in daemon mode")
|
|
|
|
@click.argument("services", metavar="service", nargs=-1)
|
|
|
|
@click.pass_obj
|
|
|
|
def start(context, detach, services):
|
|
|
|
command = ["up", "--remove-orphans"]
|
|
|
|
if detach:
|
|
|
|
command.append("-d")
|
|
|
|
|
|
|
|
config = tutor_config.load(context.root)
|
|
|
|
context.docker_compose(context.root, config, *command, *services)
|
|
|
|
|
|
|
|
|
|
|
|
@click.command(help="Stop a running platform")
|
|
|
|
@click.argument("services", metavar="service", nargs=-1)
|
|
|
|
@click.pass_obj
|
|
|
|
def stop(context, services):
|
|
|
|
config = tutor_config.load(context.root)
|
2020-09-04 10:07:44 +00:00
|
|
|
context.docker_compose(context.root, config, "stop", *services)
|
2020-01-08 18:38:13 +00:00
|
|
|
|
|
|
|
|
|
|
|
@click.command(
|
|
|
|
short_help="Reboot an existing platform",
|
|
|
|
help="This is more than just a restart: with reboot, the platform is fully stopped before being restarted again",
|
|
|
|
)
|
|
|
|
@click.option("-d", "--detach", is_flag=True, help="Start in daemon mode")
|
|
|
|
@click.argument("services", metavar="service", nargs=-1)
|
|
|
|
def reboot(detach, services):
|
|
|
|
stop.callback(services)
|
|
|
|
start.callback(detach, services)
|
|
|
|
|
|
|
|
|
|
|
|
@click.command(
|
|
|
|
short_help="Restart some components from a running platform.",
|
|
|
|
help="""Specify 'openedx' to restart the lms, cms and workers, or 'all' to
|
|
|
|
restart all services. Note that this performs a 'docker-compose restart', so new images
|
|
|
|
may not be taken into account. It is useful for reloading settings, for instance. To
|
|
|
|
fully stop the platform, use the 'reboot' command.""",
|
|
|
|
)
|
2020-03-12 08:52:22 +00:00
|
|
|
@click.argument("services", metavar="service", nargs=-1)
|
2020-01-08 18:38:13 +00:00
|
|
|
@click.pass_obj
|
2020-03-12 08:52:22 +00:00
|
|
|
def restart(context, services):
|
2020-01-08 18:38:13 +00:00
|
|
|
config = tutor_config.load(context.root)
|
|
|
|
command = ["restart"]
|
2020-03-12 08:52:22 +00:00
|
|
|
if "all" in services:
|
|
|
|
pass
|
|
|
|
else:
|
|
|
|
for service in services:
|
Improve job running in local and k8s
Running jobs was previously done with "exec". This was because it
allowed us to avoid copying too much container specification information
from the docker-compose/deployments files to the jobs files. However,
this was limiting:
- In order to run a job, the corresponding container had to be running.
This was particularly painful in Kubernetes, where containers are
crashing as long as migrations are not correctly run.
- Containers in which we need to run jobs needed to be present in the
docker-compose/deployments files. This is unnecessary, for example when
mysql is disabled, or in the case of the certbot container.
Now, we create dedicated jobs files, both for local and k8s deployment.
This introduces a little redundancy, but not too much. Note that
dependent containers are not listed in the docker-compose.jobs.yml file,
so an actual platform is still supposed to be running when we launch the
jobs.
This also introduces a subtle change: now, jobs go through the container
entrypoint prior to running. This is probably a good thing, as it will
avoid forgetting about incorrect environment variables.
In k8s, we find ourselves interacting way too much with the kubectl
utility. Parsing output from the CLI is a pain. So we need to switch to
the native kubernetes client library.
2020-03-25 17:47:36 +00:00
|
|
|
if service == "openedx":
|
2020-03-12 08:52:22 +00:00
|
|
|
if config["ACTIVATE_LMS"]:
|
|
|
|
command += ["lms", "lms-worker"]
|
|
|
|
if config["ACTIVATE_CMS"]:
|
|
|
|
command += ["cms", "cms-worker"]
|
|
|
|
else:
|
|
|
|
command.append(service)
|
2020-01-08 18:38:13 +00:00
|
|
|
context.docker_compose(context.root, config, *command)
|
|
|
|
|
|
|
|
|
|
|
|
@click.command(help="Initialise all applications")
|
2020-06-01 20:38:04 +00:00
|
|
|
@click.option("-l", "--limit", help="Limit initialisation to this service or plugin")
|
2020-01-08 18:38:13 +00:00
|
|
|
@click.pass_obj
|
2020-06-01 20:38:04 +00:00
|
|
|
def init(context, limit):
|
2020-01-08 18:38:13 +00:00
|
|
|
config = tutor_config.load(context.root)
|
|
|
|
runner = ScriptRunner(context.root, config, context.docker_compose)
|
2020-06-01 20:38:04 +00:00
|
|
|
scripts.initialise(runner, limit_to=limit)
|
2020-01-08 18:38:13 +00:00
|
|
|
|
|
|
|
|
|
|
|
@click.command(help="Create an Open edX user and interactively set their password")
|
|
|
|
@click.option("--superuser", is_flag=True, help="Make superuser")
|
|
|
|
@click.option("--staff", is_flag=True, help="Make staff user")
|
|
|
|
@click.option(
|
|
|
|
"-p",
|
|
|
|
"--password",
|
|
|
|
help="Specify password from the command line. If undefined, you will be prompted to input a password",
|
|
|
|
)
|
|
|
|
@click.argument("name")
|
|
|
|
@click.argument("email")
|
|
|
|
@click.pass_obj
|
|
|
|
def createuser(context, superuser, staff, password, name, email):
|
|
|
|
config = tutor_config.load(context.root)
|
|
|
|
runner = ScriptRunner(context.root, config, context.docker_compose)
|
|
|
|
command = scripts.create_user_command(
|
|
|
|
superuser, staff, name, email, password=password
|
|
|
|
)
|
Improve job running in local and k8s
Running jobs was previously done with "exec". This was because it
allowed us to avoid copying too much container specification information
from the docker-compose/deployments files to the jobs files. However,
this was limiting:
- In order to run a job, the corresponding container had to be running.
This was particularly painful in Kubernetes, where containers are
crashing as long as migrations are not correctly run.
- Containers in which we need to run jobs needed to be present in the
docker-compose/deployments files. This is unnecessary, for example when
mysql is disabled, or in the case of the certbot container.
Now, we create dedicated jobs files, both for local and k8s deployment.
This introduces a little redundancy, but not too much. Note that
dependent containers are not listed in the docker-compose.jobs.yml file,
so an actual platform is still supposed to be running when we launch the
jobs.
This also introduces a subtle change: now, jobs go through the container
entrypoint prior to running. This is probably a good thing, as it will
avoid forgetting about incorrect environment variables.
In k8s, we find ourselves interacting way too much with the kubectl
utility. Parsing output from the CLI is a pain. So we need to switch to
the native kubernetes client library.
2020-03-25 17:47:36 +00:00
|
|
|
runner.run_job("lms", command)
|
2020-01-08 18:38:13 +00:00
|
|
|
|
|
|
|
|
2020-03-13 11:09:48 +00:00
|
|
|
@click.command(
|
|
|
|
help="Set a theme for a given domain name. To reset to the default theme , use 'default' as the theme name."
|
|
|
|
)
|
|
|
|
@click.argument("theme_name")
|
|
|
|
@click.argument("domain_names", metavar="domain_name", nargs=-1)
|
|
|
|
@click.pass_obj
|
|
|
|
def settheme(context, theme_name, domain_names):
|
|
|
|
config = tutor_config.load(context.root)
|
|
|
|
runner = ScriptRunner(context.root, config, context.docker_compose)
|
|
|
|
for domain_name in domain_names:
|
|
|
|
scripts.set_theme(theme_name, domain_name, runner)
|
|
|
|
|
|
|
|
|
2020-01-08 18:38:13 +00:00
|
|
|
@click.command(help="Import the demo course")
|
|
|
|
@click.pass_obj
|
|
|
|
def importdemocourse(context):
|
|
|
|
config = tutor_config.load(context.root)
|
|
|
|
runner = ScriptRunner(context.root, config, context.docker_compose)
|
|
|
|
fmt.echo_info("Importing demo course")
|
|
|
|
scripts.import_demo_course(runner)
|
|
|
|
|
|
|
|
|
2020-11-16 11:46:01 +00:00
|
|
|
@click.command(
|
|
|
|
short_help="Direct interface to docker-compose.",
|
|
|
|
help=(
|
|
|
|
"Direct interface to docker-compose. This is a wrapper around `docker-compose`. All commands, options and"
|
|
|
|
" arguments passed to this command will be forwarded to docker-compose."
|
|
|
|
),
|
|
|
|
context_settings={"ignore_unknown_options": True},
|
|
|
|
name="dc",
|
|
|
|
)
|
|
|
|
@click.argument("args", nargs=-1, required=True)
|
|
|
|
@click.pass_obj
|
|
|
|
def dc_command(context, args):
|
|
|
|
config = tutor_config.load(context.root)
|
|
|
|
context.docker_compose(context.root, config, *args)
|
|
|
|
|
|
|
|
|
|
|
|
@click.command(
|
|
|
|
short_help="Run a command in a new container",
|
|
|
|
help=(
|
|
|
|
"Run a command in a new container. This is a wrapper around `docker-compose run`. Any option or argument passed"
|
|
|
|
" to this command will be forwarded to docker-compose. Thus, you may use `-v` or `-p` to mount volumes and"
|
|
|
|
" expose ports."
|
|
|
|
),
|
|
|
|
context_settings={"ignore_unknown_options": True},
|
|
|
|
)
|
|
|
|
@click.argument("args", nargs=-1, required=True)
|
|
|
|
def run(args):
|
|
|
|
command = ["run", "--rm"]
|
|
|
|
if not utils.is_a_tty():
|
|
|
|
command.append("-T")
|
2020-11-16 14:36:23 +00:00
|
|
|
dc_command.callback([*command, *args])
|
2020-11-16 11:46:01 +00:00
|
|
|
|
|
|
|
|
|
|
|
@click.command(
|
|
|
|
short_help="Run a command in a running container",
|
|
|
|
help=(
|
|
|
|
"Run a command in a running container. This is a wrapper around `docker-compose exec`. Any option or argument"
|
|
|
|
" passed to this command will be forwarded to docker-compose. Thus, you may use `-e` to manually define"
|
|
|
|
" environment variables."
|
|
|
|
),
|
|
|
|
context_settings={"ignore_unknown_options": True},
|
|
|
|
name="exec",
|
|
|
|
)
|
|
|
|
@click.argument("args", nargs=-1, required=True)
|
|
|
|
def execute(args):
|
2020-11-16 14:36:23 +00:00
|
|
|
dc_command.callback(["exec", *args])
|
2020-11-16 11:46:01 +00:00
|
|
|
|
|
|
|
|
|
|
|
@click.command(
|
|
|
|
short_help="View output from containers",
|
|
|
|
help="View output from containers. This is a wrapper around `docker-compose logs`.",
|
|
|
|
)
|
|
|
|
@click.option("-f", "--follow", is_flag=True, help="Follow log output")
|
|
|
|
@click.option("--tail", type=int, help="Number of lines to show from each container")
|
|
|
|
@click.argument("service", nargs=-1)
|
|
|
|
def logs(follow, tail, service):
|
|
|
|
command = ["logs"]
|
|
|
|
if follow:
|
|
|
|
command += ["--follow"]
|
|
|
|
if tail is not None:
|
|
|
|
command += ["--tail", str(tail)]
|
|
|
|
command += service
|
|
|
|
dc_command.callback(command)
|
|
|
|
|
|
|
|
|
2020-01-08 18:38:13 +00:00
|
|
|
def add_commands(command_group):
|
|
|
|
command_group.add_command(start)
|
|
|
|
command_group.add_command(stop)
|
|
|
|
command_group.add_command(restart)
|
|
|
|
command_group.add_command(reboot)
|
|
|
|
command_group.add_command(init)
|
|
|
|
command_group.add_command(createuser)
|
|
|
|
command_group.add_command(importdemocourse)
|
2020-03-13 11:09:48 +00:00
|
|
|
command_group.add_command(settheme)
|
2020-11-16 11:46:01 +00:00
|
|
|
command_group.add_command(dc_command)
|
|
|
|
command_group.add_command(run)
|
|
|
|
command_group.add_command(execute)
|
|
|
|
command_group.add_command(logs)
|