refactor: annotation with __future__.annotations
Adds `from __future__ import annotations` to the top of every module,
right below the module's docstring. Replaces any usages of t.List,
t.Dict, t.Set, t.Tuple, and t.Type with their built-in equivalents:
list, dict, set, tuple, and type. Ensures that make test still passes
under Python 3.7, 3.8 and 3.9.
2023-01-17 18:57:23 +00:00
from __future__ import annotations
2020-12-25 21:56:42 +00:00
import os
2022-04-15 08:51:19 +00:00
import re
import typing as t
2022-07-25 17:19:28 +00:00
from copy import deepcopy
2020-12-25 21:56:42 +00:00
2020-01-08 18:38:13 +00:00
import click
2022-11-08 09:33:52 +00:00
from click . shell_completion import CompletionItem
feat: strongly typed hooks
Now that the mypy bugs have been resolved, we are able to define more precisely
and cleanly the types of Actions and Filters.
Moreover, can now strongly type named actions and hooks (in consts.py). With
such a strong typing, we get early alerts of hooks called with incorrect
arguments, which is nothing short of awesome :)
This change breaks the hooks API by removing the `context=...` argument. The
reason for that is that we cannot insert arbitrary arguments between `P.args,
P.kwargs`: https://peps.python.org/pep-0612/#the-components-of-a-paramspec
> A function declared as def inner(a: A, b: B, *args: P.args, **kwargs:
> P.kwargs) -> R has type Callable[Concatenate[A, B, P], R]. Placing
> keyword-only parameters between the *args and **kwargs is forbidden.
Getting the documentation to build in nitpicky mode is quite difficult... We
need to add `nitpick_ignore` to the docs conf.py, otherwise sphinx complains
about many missing class references. This, despite upgrading almost all doc
requirements (except docutils).
2022-10-06 10:05:01 +00:00
from typing_extensions import TypeAlias
2020-01-08 18:38:13 +00:00
2022-02-07 17:11:43 +00:00
from tutor import config as tutor_config
from tutor import env as tutor_env
2022-10-18 14:57:07 +00:00
from tutor import fmt , hooks , serialize , utils
from tutor . commands import jobs
2022-10-19 15:46:31 +00:00
from tutor . commands . context import BaseTaskContext
2022-02-07 17:11:43 +00:00
from tutor . exceptions import TutorError
2022-10-19 15:46:31 +00:00
from tutor . tasks import BaseComposeTaskRunner
2022-02-07 17:11:43 +00:00
from tutor . types import Config
2020-01-08 18:38:13 +00:00
refactor: annotation with __future__.annotations
Adds `from __future__ import annotations` to the top of every module,
right below the module's docstring. Replaces any usages of t.List,
t.Dict, t.Set, t.Tuple, and t.Type with their built-in equivalents:
list, dict, set, tuple, and type. Ensures that make test still passes
under Python 3.7, 3.8 and 3.9.
2023-01-17 18:57:23 +00:00
COMPOSE_FILTER_TYPE : TypeAlias = " hooks.filters.Filter[dict[str, t.Any], []] "
feat: strongly typed hooks
Now that the mypy bugs have been resolved, we are able to define more precisely
and cleanly the types of Actions and Filters.
Moreover, can now strongly type named actions and hooks (in consts.py). With
such a strong typing, we get early alerts of hooks called with incorrect
arguments, which is nothing short of awesome :)
This change breaks the hooks API by removing the `context=...` argument. The
reason for that is that we cannot insert arbitrary arguments between `P.args,
P.kwargs`: https://peps.python.org/pep-0612/#the-components-of-a-paramspec
> A function declared as def inner(a: A, b: B, *args: P.args, **kwargs:
> P.kwargs) -> R has type Callable[Concatenate[A, B, P], R]. Placing
> keyword-only parameters between the *args and **kwargs is forbidden.
Getting the documentation to build in nitpicky mode is quite difficult... We
need to add `nitpick_ignore` to the docs conf.py, otherwise sphinx complains
about many missing class references. This, despite upgrading almost all doc
requirements (except docutils).
2022-10-06 10:05:01 +00:00
2020-01-08 18:38:13 +00:00
2022-10-19 15:46:31 +00:00
class ComposeTaskRunner ( BaseComposeTaskRunner ) :
2021-09-27 10:32:28 +00:00
def __init__ ( self , root : str , config : Config ) :
super ( ) . __init__ ( root , config )
self . project_name = " "
refactor: annotation with __future__.annotations
Adds `from __future__ import annotations` to the top of every module,
right below the module's docstring. Replaces any usages of t.List,
t.Dict, t.Set, t.Tuple, and t.Type with their built-in equivalents:
list, dict, set, tuple, and type. Ensures that make test still passes
under Python 3.7, 3.8 and 3.9.
2023-01-17 18:57:23 +00:00
self . docker_compose_files : list [ str ] = [ ]
self . docker_compose_job_files : list [ str ] = [ ]
2021-09-27 10:32:28 +00:00
def docker_compose ( self , * command : str ) - > int :
"""
Run docker - compose with the right yml files .
"""
2022-04-21 14:24:34 +00:00
if " start " in command or " up " in command or " restart " in command :
# Note that we don't trigger the action on "run". That's because we
# don't want to trigger the action for every initialization script.
hooks . Actions . COMPOSE_PROJECT_STARTED . do (
self . root , self . config , self . project_name
)
2021-09-27 10:32:28 +00:00
args = [ ]
for docker_compose_path in self . docker_compose_files :
if os . path . exists ( docker_compose_path ) :
args + = [ " -f " , docker_compose_path ]
return utils . docker_compose (
* args , " --project-name " , self . project_name , * command
)
2022-07-25 17:19:28 +00:00
def update_docker_compose_tmp (
self ,
feat: strongly typed hooks
Now that the mypy bugs have been resolved, we are able to define more precisely
and cleanly the types of Actions and Filters.
Moreover, can now strongly type named actions and hooks (in consts.py). With
such a strong typing, we get early alerts of hooks called with incorrect
arguments, which is nothing short of awesome :)
This change breaks the hooks API by removing the `context=...` argument. The
reason for that is that we cannot insert arbitrary arguments between `P.args,
P.kwargs`: https://peps.python.org/pep-0612/#the-components-of-a-paramspec
> A function declared as def inner(a: A, b: B, *args: P.args, **kwargs:
> P.kwargs) -> R has type Callable[Concatenate[A, B, P], R]. Placing
> keyword-only parameters between the *args and **kwargs is forbidden.
Getting the documentation to build in nitpicky mode is quite difficult... We
need to add `nitpick_ignore` to the docs conf.py, otherwise sphinx complains
about many missing class references. This, despite upgrading almost all doc
requirements (except docutils).
2022-10-06 10:05:01 +00:00
compose_tmp_filter : COMPOSE_FILTER_TYPE ,
compose_jobs_tmp_filter : COMPOSE_FILTER_TYPE ,
2022-07-25 17:19:28 +00:00
docker_compose_tmp_path : str ,
docker_compose_jobs_tmp_path : str ,
) - > None :
2022-04-15 08:51:19 +00:00
"""
2022-07-25 17:19:28 +00:00
Update the contents of the docker - compose . tmp . yml and
docker - compose . jobs . tmp . yml files , which are generated at runtime .
2022-04-15 08:51:19 +00:00
"""
refactor: annotation with __future__.annotations
Adds `from __future__ import annotations` to the top of every module,
right below the module's docstring. Replaces any usages of t.List,
t.Dict, t.Set, t.Tuple, and t.Type with their built-in equivalents:
list, dict, set, tuple, and type. Ensures that make test still passes
under Python 3.7, 3.8 and 3.9.
2023-01-17 18:57:23 +00:00
compose_base : dict [ str , t . Any ] = {
2022-04-15 08:51:19 +00:00
" version " : " {{ DOCKER_COMPOSE_VERSION }} " ,
" services " : { } ,
}
2022-07-25 17:19:28 +00:00
# 1. Apply compose_tmp filter
# 2. Render the resulting dict
# 3. Serialize to yaml
# 4. Save to disk
docker_compose_tmp : str = serialize . dumps (
tutor_env . render_unknown (
self . config , compose_tmp_filter . apply ( deepcopy ( compose_base ) )
)
2022-04-15 08:51:19 +00:00
)
tutor_env . write_to (
2022-07-25 17:19:28 +00:00
docker_compose_tmp ,
docker_compose_tmp_path ,
)
# Same thing but with tmp jobs
docker_compose_jobs_tmp : str = serialize . dumps (
tutor_env . render_unknown (
self . config , compose_jobs_tmp_filter . apply ( deepcopy ( compose_base ) )
)
2022-04-15 08:51:19 +00:00
)
tutor_env . write_to (
2022-07-25 17:19:28 +00:00
docker_compose_jobs_tmp ,
docker_compose_jobs_tmp_path ,
2022-04-15 08:51:19 +00:00
)
2022-10-19 15:46:31 +00:00
def run_task ( self , service : str , command : str ) - > int :
Improve job running in local and k8s
Running jobs was previously done with "exec". This was because it
allowed us to avoid copying too much container specification information
from the docker-compose/deployments files to the jobs files. However,
this was limiting:
- In order to run a job, the corresponding container had to be running.
This was particularly painful in Kubernetes, where containers are
crashing as long as migrations are not correctly run.
- Containers in which we need to run jobs needed to be present in the
docker-compose/deployments files. This is unnecessary, for example when
mysql is disabled, or in the case of the certbot container.
Now, we create dedicated jobs files, both for local and k8s deployment.
This introduces a little redundancy, but not too much. Note that
dependent containers are not listed in the docker-compose.jobs.yml file,
so an actual platform is still supposed to be running when we launch the
jobs.
This also introduces a subtle change: now, jobs go through the container
entrypoint prior to running. This is probably a good thing, as it will
avoid forgetting about incorrect environment variables.
In k8s, we find ourselves interacting way too much with the kubectl
utility. Parsing output from the CLI is a pain. So we need to switch to
the native kubernetes client library.
2020-03-25 17:47:36 +00:00
"""
Run the " {{ service }}-job " service from local / docker - compose . jobs . yml with the
2021-11-02 17:24:38 +00:00
specified command .
Improve job running in local and k8s
Running jobs was previously done with "exec". This was because it
allowed us to avoid copying too much container specification information
from the docker-compose/deployments files to the jobs files. However,
this was limiting:
- In order to run a job, the corresponding container had to be running.
This was particularly painful in Kubernetes, where containers are
crashing as long as migrations are not correctly run.
- Containers in which we need to run jobs needed to be present in the
docker-compose/deployments files. This is unnecessary, for example when
mysql is disabled, or in the case of the certbot container.
Now, we create dedicated jobs files, both for local and k8s deployment.
This introduces a little redundancy, but not too much. Note that
dependent containers are not listed in the docker-compose.jobs.yml file,
so an actual platform is still supposed to be running when we launch the
jobs.
This also introduces a subtle change: now, jobs go through the container
entrypoint prior to running. This is probably a good thing, as it will
avoid forgetting about incorrect environment variables.
In k8s, we find ourselves interacting way too much with the kubectl
utility. Parsing output from the CLI is a pain. So we need to switch to
the native kubernetes client library.
2020-03-25 17:47:36 +00:00
"""
2021-09-27 10:32:28 +00:00
run_command = [ ]
for docker_compose_path in self . docker_compose_job_files :
path = tutor_env . pathjoin ( self . root , docker_compose_path )
if os . path . exists ( path ) :
run_command + = [ " -f " , path ]
2021-09-06 14:20:36 +00:00
run_command + = [ " run " , " --rm " ]
if not utils . is_a_tty ( ) :
run_command + = [ " -T " ]
2022-02-21 10:52:46 +00:00
job_service_name = f " { service } -job "
2021-09-27 09:48:17 +00:00
return self . docker_compose (
2021-09-06 14:20:36 +00:00
* run_command ,
job_service_name ,
2021-02-25 08:09:14 +00:00
" sh " ,
" -e " ,
" -c " ,
command ,
)
2020-01-08 18:38:13 +00:00
2022-10-19 15:46:31 +00:00
class BaseComposeContext ( BaseTaskContext ) :
feat: strongly typed hooks
Now that the mypy bugs have been resolved, we are able to define more precisely
and cleanly the types of Actions and Filters.
Moreover, can now strongly type named actions and hooks (in consts.py). With
such a strong typing, we get early alerts of hooks called with incorrect
arguments, which is nothing short of awesome :)
This change breaks the hooks API by removing the `context=...` argument. The
reason for that is that we cannot insert arbitrary arguments between `P.args,
P.kwargs`: https://peps.python.org/pep-0612/#the-components-of-a-paramspec
> A function declared as def inner(a: A, b: B, *args: P.args, **kwargs:
> P.kwargs) -> R has type Callable[Concatenate[A, B, P], R]. Placing
> keyword-only parameters between the *args and **kwargs is forbidden.
Getting the documentation to build in nitpicky mode is quite difficult... We
need to add `nitpick_ignore` to the docs conf.py, otherwise sphinx complains
about many missing class references. This, despite upgrading almost all doc
requirements (except docutils).
2022-10-06 10:05:01 +00:00
COMPOSE_TMP_FILTER : COMPOSE_FILTER_TYPE = NotImplemented
COMPOSE_JOBS_TMP_FILTER : COMPOSE_FILTER_TYPE = NotImplemented
2022-07-25 17:19:28 +00:00
2022-10-19 15:46:31 +00:00
def job_runner ( self , config : Config ) - > ComposeTaskRunner :
2021-09-27 09:48:17 +00:00
raise NotImplementedError
2022-04-15 08:51:19 +00:00
class MountParam ( click . ParamType ) :
"""
Parser for - - mount arguments of the form " service1[,service2,...]:/host/path:/container/path " .
"""
name = " mount "
MountType = t . Tuple [ str , str , str ]
# Note that this syntax does not allow us to include colon ':' characters in paths
PARAM_REGEXP = (
2022-04-21 15:43:44 +00:00
r " (?P<services>[a-zA-Z0-9-_, ]+):(?P<host_path>[^:]+):(?P<container_path>[^:]+) "
2022-04-15 08:51:19 +00:00
)
def convert (
self ,
value : str ,
param : t . Optional [ " click.Parameter " ] ,
ctx : t . Optional [ click . Context ] ,
refactor: annotation with __future__.annotations
Adds `from __future__ import annotations` to the top of every module,
right below the module's docstring. Replaces any usages of t.List,
t.Dict, t.Set, t.Tuple, and t.Type with their built-in equivalents:
list, dict, set, tuple, and type. Ensures that make test still passes
under Python 3.7, 3.8 and 3.9.
2023-01-17 18:57:23 +00:00
) - > list [ " MountType " ] :
2022-07-25 17:19:28 +00:00
mounts = self . convert_explicit_form ( value ) or self . convert_implicit_form ( value )
return mounts
refactor: annotation with __future__.annotations
Adds `from __future__ import annotations` to the top of every module,
right below the module's docstring. Replaces any usages of t.List,
t.Dict, t.Set, t.Tuple, and t.Type with their built-in equivalents:
list, dict, set, tuple, and type. Ensures that make test still passes
under Python 3.7, 3.8 and 3.9.
2023-01-17 18:57:23 +00:00
def convert_explicit_form ( self , value : str ) - > list [ " MountParam.MountType " ] :
2022-07-25 17:19:28 +00:00
"""
Argument is of the form " containers:/host/path:/container/path " .
"""
2022-04-15 08:51:19 +00:00
match = re . match ( self . PARAM_REGEXP , value )
2022-07-25 17:19:28 +00:00
if not match :
return [ ]
refactor: annotation with __future__.annotations
Adds `from __future__ import annotations` to the top of every module,
right below the module's docstring. Replaces any usages of t.List,
t.Dict, t.Set, t.Tuple, and t.Type with their built-in equivalents:
list, dict, set, tuple, and type. Ensures that make test still passes
under Python 3.7, 3.8 and 3.9.
2023-01-17 18:57:23 +00:00
mounts : list [ " MountParam.MountType " ] = [ ]
services : list [ str ] = [
2022-07-25 17:19:28 +00:00
service . strip ( ) for service in match [ " services " ] . split ( " , " )
]
host_path = os . path . abspath ( os . path . expanduser ( match [ " host_path " ] ) )
host_path = host_path . replace ( os . path . sep , " / " )
container_path = match [ " container_path " ]
for service in services :
if not service :
self . fail ( f " incorrect services syntax: ' { match [ ' services ' ] } ' " )
mounts . append ( ( service , host_path , container_path ) )
return mounts
refactor: annotation with __future__.annotations
Adds `from __future__ import annotations` to the top of every module,
right below the module's docstring. Replaces any usages of t.List,
t.Dict, t.Set, t.Tuple, and t.Type with their built-in equivalents:
list, dict, set, tuple, and type. Ensures that make test still passes
under Python 3.7, 3.8 and 3.9.
2023-01-17 18:57:23 +00:00
def convert_implicit_form ( self , value : str ) - > list [ " MountParam.MountType " ] :
2022-07-25 17:19:28 +00:00
"""
Argument is of the form " /host/path "
"""
refactor: annotation with __future__.annotations
Adds `from __future__ import annotations` to the top of every module,
right below the module's docstring. Replaces any usages of t.List,
t.Dict, t.Set, t.Tuple, and t.Type with their built-in equivalents:
list, dict, set, tuple, and type. Ensures that make test still passes
under Python 3.7, 3.8 and 3.9.
2023-01-17 18:57:23 +00:00
mounts : list [ " MountParam.MountType " ] = [ ]
2022-07-25 17:19:28 +00:00
host_path = os . path . abspath ( os . path . expanduser ( value ) )
feat: strongly typed hooks
Now that the mypy bugs have been resolved, we are able to define more precisely
and cleanly the types of Actions and Filters.
Moreover, can now strongly type named actions and hooks (in consts.py). With
such a strong typing, we get early alerts of hooks called with incorrect
arguments, which is nothing short of awesome :)
This change breaks the hooks API by removing the `context=...` argument. The
reason for that is that we cannot insert arbitrary arguments between `P.args,
P.kwargs`: https://peps.python.org/pep-0612/#the-components-of-a-paramspec
> A function declared as def inner(a: A, b: B, *args: P.args, **kwargs:
> P.kwargs) -> R has type Callable[Concatenate[A, B, P], R]. Placing
> keyword-only parameters between the *args and **kwargs is forbidden.
Getting the documentation to build in nitpicky mode is quite difficult... We
need to add `nitpick_ignore` to the docs conf.py, otherwise sphinx complains
about many missing class references. This, despite upgrading almost all doc
requirements (except docutils).
2022-10-06 10:05:01 +00:00
for service , container_path in hooks . Filters . COMPOSE_MOUNTS . iterate (
2022-07-25 17:19:28 +00:00
os . path . basename ( host_path )
feat: strongly typed hooks
Now that the mypy bugs have been resolved, we are able to define more precisely
and cleanly the types of Actions and Filters.
Moreover, can now strongly type named actions and hooks (in consts.py). With
such a strong typing, we get early alerts of hooks called with incorrect
arguments, which is nothing short of awesome :)
This change breaks the hooks API by removing the `context=...` argument. The
reason for that is that we cannot insert arbitrary arguments between `P.args,
P.kwargs`: https://peps.python.org/pep-0612/#the-components-of-a-paramspec
> A function declared as def inner(a: A, b: B, *args: P.args, **kwargs:
> P.kwargs) -> R has type Callable[Concatenate[A, B, P], R]. Placing
> keyword-only parameters between the *args and **kwargs is forbidden.
Getting the documentation to build in nitpicky mode is quite difficult... We
need to add `nitpick_ignore` to the docs conf.py, otherwise sphinx complains
about many missing class references. This, despite upgrading almost all doc
requirements (except docutils).
2022-10-06 10:05:01 +00:00
) :
2022-07-25 17:19:28 +00:00
mounts . append ( ( service , host_path , container_path ) )
2022-04-15 08:51:19 +00:00
if not mounts :
2022-07-25 17:19:28 +00:00
raise self . fail ( f " no mount found for { value } " )
2022-04-15 08:51:19 +00:00
return mounts
2022-11-08 09:33:52 +00:00
def shell_complete (
self , ctx : click . Context , param : click . Parameter , incomplete : str
refactor: annotation with __future__.annotations
Adds `from __future__ import annotations` to the top of every module,
right below the module's docstring. Replaces any usages of t.List,
t.Dict, t.Set, t.Tuple, and t.Type with their built-in equivalents:
list, dict, set, tuple, and type. Ensures that make test still passes
under Python 3.7, 3.8 and 3.9.
2023-01-17 18:57:23 +00:00
) - > list [ CompletionItem ] :
2022-11-08 09:33:52 +00:00
"""
Mount argument completion works only for the single path ( implicit ) form . The
reason is that colons break words in bash completion :
http : / / tiswww . case . edu / php / chet / bash / FAQ ( E13 )
Thus , we do not even attempt to auto - complete mount arguments that include
colons : such arguments will not even reach this method .
"""
return [ CompletionItem ( incomplete , type = " file " ) ]
2022-04-15 08:51:19 +00:00
mount_option = click . option (
" -m " ,
" --mount " ,
" mounts " ,
help = """ Bind-mount a folder from the host in the right containers. This option can take two different forms. The first one is explicit: ' service1[,service2...]:/host/path:/container/path ' . The other is implicit: ' /host/path ' . Arguments passed in the implicit form will be parsed by plugins to define the right folders to bind-mount from the host. """ ,
type = MountParam ( ) ,
multiple = True ,
)
2022-07-25 17:19:28 +00:00
def mount_tmp_volumes (
refactor: annotation with __future__.annotations
Adds `from __future__ import annotations` to the top of every module,
right below the module's docstring. Replaces any usages of t.List,
t.Dict, t.Set, t.Tuple, and t.Type with their built-in equivalents:
list, dict, set, tuple, and type. Ensures that make test still passes
under Python 3.7, 3.8 and 3.9.
2023-01-17 18:57:23 +00:00
all_mounts : tuple [ list [ MountParam . MountType ] , . . . ] ,
2022-07-25 17:19:28 +00:00
context : BaseComposeContext ,
) - > None :
for mounts in all_mounts :
for service , host_path , container_path in mounts :
mount_tmp_volume ( service , host_path , container_path , context )
def mount_tmp_volume (
service : str ,
host_path : str ,
container_path : str ,
context : BaseComposeContext ,
) - > None :
"""
Append user - defined bind - mounted volumes to the docker - compose . tmp file ( s ) .
The service / host path / container path values are appended to the docker - compose
files by mean of two filters . Each dev / local environment is then responsible for
generating the files based on the output of these filters .
Bind - mounts that are associated to " *-job " services will be added to the
docker - compose jobs file .
"""
fmt . echo_info ( f " Bind-mount: { host_path } -> { container_path } in { service } " )
feat: strongly typed hooks
Now that the mypy bugs have been resolved, we are able to define more precisely
and cleanly the types of Actions and Filters.
Moreover, can now strongly type named actions and hooks (in consts.py). With
such a strong typing, we get early alerts of hooks called with incorrect
arguments, which is nothing short of awesome :)
This change breaks the hooks API by removing the `context=...` argument. The
reason for that is that we cannot insert arbitrary arguments between `P.args,
P.kwargs`: https://peps.python.org/pep-0612/#the-components-of-a-paramspec
> A function declared as def inner(a: A, b: B, *args: P.args, **kwargs:
> P.kwargs) -> R has type Callable[Concatenate[A, B, P], R]. Placing
> keyword-only parameters between the *args and **kwargs is forbidden.
Getting the documentation to build in nitpicky mode is quite difficult... We
need to add `nitpick_ignore` to the docs conf.py, otherwise sphinx complains
about many missing class references. This, despite upgrading almost all doc
requirements (except docutils).
2022-10-06 10:05:01 +00:00
compose_tmp_filter : COMPOSE_FILTER_TYPE = (
2022-07-25 17:19:28 +00:00
context . COMPOSE_JOBS_TMP_FILTER
if service . endswith ( " -job " )
else context . COMPOSE_TMP_FILTER
)
@compose_tmp_filter.add ( )
def _add_mounts_to_docker_compose_tmp (
refactor: annotation with __future__.annotations
Adds `from __future__ import annotations` to the top of every module,
right below the module's docstring. Replaces any usages of t.List,
t.Dict, t.Set, t.Tuple, and t.Type with their built-in equivalents:
list, dict, set, tuple, and type. Ensures that make test still passes
under Python 3.7, 3.8 and 3.9.
2023-01-17 18:57:23 +00:00
docker_compose : dict [ str , t . Any ] ,
) - > dict [ str , t . Any ] :
2022-07-25 17:19:28 +00:00
services = docker_compose . setdefault ( " services " , { } )
services . setdefault ( service , { " volumes " : [ ] } )
services [ service ] [ " volumes " ] . append ( f " { host_path } : { container_path } " )
return docker_compose
2021-04-13 20:14:43 +00:00
@click.command (
short_help = " Run all or a selection of services. " ,
help = " Run all or a selection of services. Docker images will be rebuilt where necessary. " ,
)
2021-10-18 09:43:40 +00:00
@click.option ( " --skip-build " , is_flag = True , help = " Skip image building " )
2020-01-08 18:38:13 +00:00
@click.option ( " -d " , " --detach " , is_flag = True , help = " Start in daemon mode " )
2022-04-15 08:51:19 +00:00
@mount_option
2020-01-08 18:38:13 +00:00
@click.argument ( " services " , metavar = " service " , nargs = - 1 )
@click.pass_obj
2021-10-18 09:43:40 +00:00
def start (
2022-04-15 08:51:19 +00:00
context : BaseComposeContext ,
skip_build : bool ,
detach : bool ,
refactor: annotation with __future__.annotations
Adds `from __future__ import annotations` to the top of every module,
right below the module's docstring. Replaces any usages of t.List,
t.Dict, t.Set, t.Tuple, and t.Type with their built-in equivalents:
list, dict, set, tuple, and type. Ensures that make test still passes
under Python 3.7, 3.8 and 3.9.
2023-01-17 18:57:23 +00:00
mounts : tuple [ list [ MountParam . MountType ] ] ,
services : list [ str ] ,
2021-10-18 09:43:40 +00:00
) - > None :
command = [ " up " , " --remove-orphans " ]
if not skip_build :
command . append ( " --build " )
2020-01-08 18:38:13 +00:00
if detach :
command . append ( " -d " )
2021-04-13 20:14:43 +00:00
# Start services
2022-07-25 17:19:28 +00:00
mount_tmp_volumes ( mounts , context )
2021-09-27 09:48:17 +00:00
config = tutor_config . load ( context . root )
context . job_runner ( config ) . docker_compose ( * command , * services )
2020-01-08 18:38:13 +00:00
@click.command ( help = " Stop a running platform " )
@click.argument ( " services " , metavar = " service " , nargs = - 1 )
@click.pass_obj
refactor: annotation with __future__.annotations
Adds `from __future__ import annotations` to the top of every module,
right below the module's docstring. Replaces any usages of t.List,
t.Dict, t.Set, t.Tuple, and t.Type with their built-in equivalents:
list, dict, set, tuple, and type. Ensures that make test still passes
under Python 3.7, 3.8 and 3.9.
2023-01-17 18:57:23 +00:00
def stop ( context : BaseComposeContext , services : list [ str ] ) - > None :
2020-01-08 18:38:13 +00:00
config = tutor_config . load ( context . root )
2021-09-27 09:48:17 +00:00
context . job_runner ( config ) . docker_compose ( " stop " , * services )
2020-01-08 18:38:13 +00:00
@click.command (
short_help = " Reboot an existing platform " ,
help = " This is more than just a restart: with reboot, the platform is fully stopped before being restarted again " ,
)
@click.option ( " -d " , " --detach " , is_flag = True , help = " Start in daemon mode " )
@click.argument ( " services " , metavar = " service " , nargs = - 1 )
2021-02-25 08:09:14 +00:00
@click.pass_context
refactor: annotation with __future__.annotations
Adds `from __future__ import annotations` to the top of every module,
right below the module's docstring. Replaces any usages of t.List,
t.Dict, t.Set, t.Tuple, and t.Type with their built-in equivalents:
list, dict, set, tuple, and type. Ensures that make test still passes
under Python 3.7, 3.8 and 3.9.
2023-01-17 18:57:23 +00:00
def reboot ( context : click . Context , detach : bool , services : list [ str ] ) - > None :
2021-02-25 08:09:14 +00:00
context . invoke ( stop , services = services )
context . invoke ( start , detach = detach , services = services )
2020-01-08 18:38:13 +00:00
@click.command (
short_help = " Restart some components from a running platform. " ,
help = """ Specify ' openedx ' to restart the lms, cms and workers, or ' all ' to
restart all services . Note that this performs a ' docker-compose restart ' , so new images
may not be taken into account . It is useful for reloading settings , for instance . To
fully stop the platform , use the ' reboot ' command . """ ,
)
2020-03-12 08:52:22 +00:00
@click.argument ( " services " , metavar = " service " , nargs = - 1 )
2020-01-08 18:38:13 +00:00
@click.pass_obj
refactor: annotation with __future__.annotations
Adds `from __future__ import annotations` to the top of every module,
right below the module's docstring. Replaces any usages of t.List,
t.Dict, t.Set, t.Tuple, and t.Type with their built-in equivalents:
list, dict, set, tuple, and type. Ensures that make test still passes
under Python 3.7, 3.8 and 3.9.
2023-01-17 18:57:23 +00:00
def restart ( context : BaseComposeContext , services : list [ str ] ) - > None :
2020-01-08 18:38:13 +00:00
config = tutor_config . load ( context . root )
command = [ " restart " ]
2020-03-12 08:52:22 +00:00
if " all " in services :
pass
else :
for service in services :
Improve job running in local and k8s
Running jobs was previously done with "exec". This was because it
allowed us to avoid copying too much container specification information
from the docker-compose/deployments files to the jobs files. However,
this was limiting:
- In order to run a job, the corresponding container had to be running.
This was particularly painful in Kubernetes, where containers are
crashing as long as migrations are not correctly run.
- Containers in which we need to run jobs needed to be present in the
docker-compose/deployments files. This is unnecessary, for example when
mysql is disabled, or in the case of the certbot container.
Now, we create dedicated jobs files, both for local and k8s deployment.
This introduces a little redundancy, but not too much. Note that
dependent containers are not listed in the docker-compose.jobs.yml file,
so an actual platform is still supposed to be running when we launch the
jobs.
This also introduces a subtle change: now, jobs go through the container
entrypoint prior to running. This is probably a good thing, as it will
avoid forgetting about incorrect environment variables.
In k8s, we find ourselves interacting way too much with the kubectl
utility. Parsing output from the CLI is a pain. So we need to switch to
the native kubernetes client library.
2020-03-25 17:47:36 +00:00
if service == " openedx " :
2020-09-17 10:53:14 +00:00
if config [ " RUN_LMS " ] :
2020-03-12 08:52:22 +00:00
command + = [ " lms " , " lms-worker " ]
2020-09-17 10:53:14 +00:00
if config [ " RUN_CMS " ] :
2020-03-12 08:52:22 +00:00
command + = [ " cms " , " cms-worker " ]
else :
command . append ( service )
2021-09-27 09:48:17 +00:00
context . job_runner ( config ) . docker_compose ( * command )
2020-01-08 18:38:13 +00:00
2022-10-19 15:46:31 +00:00
@jobs.do_group
2022-04-15 08:51:19 +00:00
@mount_option
2020-01-08 18:38:13 +00:00
@click.pass_obj
refactor: annotation with __future__.annotations
Adds `from __future__ import annotations` to the top of every module,
right below the module's docstring. Replaces any usages of t.List,
t.Dict, t.Set, t.Tuple, and t.Type with their built-in equivalents:
list, dict, set, tuple, and type. Ensures that make test still passes
under Python 3.7, 3.8 and 3.9.
2023-01-17 18:57:23 +00:00
def do ( context : BaseComposeContext , mounts : tuple [ list [ MountParam . MountType ] ] ) - > None :
2022-10-19 15:46:31 +00:00
"""
Run a custom job in the right container ( s ) .
"""
2022-11-11 13:13:36 +00:00
2022-10-19 15:46:31 +00:00
@hooks.Actions.DO_JOB.add ( )
def _mount_tmp_volumes ( _job_name : str , * _args : t . Any , * * _kwargs : t . Any ) - > None :
"""
We add this logic to an action callback because we do not want to trigger it
whenever we run ` tutor local do < job > - - help ` .
"""
mount_tmp_volumes ( mounts , context )
2020-01-08 18:38:13 +00:00
2020-11-16 11:46:01 +00:00
@click.command (
short_help = " Run a command in a new container " ,
help = (
" Run a command in a new container. This is a wrapper around `docker-compose run`. Any option or argument passed "
" to this command will be forwarded to docker-compose. Thus, you may use `-v` or `-p` to mount volumes and "
" expose ports. "
) ,
context_settings = { " ignore_unknown_options " : True } ,
)
2022-04-15 08:51:19 +00:00
@mount_option
2020-11-16 11:46:01 +00:00
@click.argument ( " args " , nargs = - 1 , required = True )
2021-02-25 08:09:14 +00:00
@click.pass_context
2022-04-15 08:51:19 +00:00
def run (
context : click . Context ,
refactor: annotation with __future__.annotations
Adds `from __future__ import annotations` to the top of every module,
right below the module's docstring. Replaces any usages of t.List,
t.Dict, t.Set, t.Tuple, and t.Type with their built-in equivalents:
list, dict, set, tuple, and type. Ensures that make test still passes
under Python 3.7, 3.8 and 3.9.
2023-01-17 18:57:23 +00:00
mounts : tuple [ list [ MountParam . MountType ] ] ,
args : list [ str ] ,
2022-04-15 08:51:19 +00:00
) - > None :
2020-12-25 21:56:42 +00:00
extra_args = [ " --rm " ]
2020-11-16 11:46:01 +00:00
if not utils . is_a_tty ( ) :
2020-12-25 21:56:42 +00:00
extra_args . append ( " -T " )
2022-07-25 17:19:28 +00:00
context . invoke ( dc_command , mounts = mounts , command = " run " , args = [ * extra_args , * args ] )
2020-12-25 21:56:42 +00:00
2022-04-20 20:24:17 +00:00
@click.command (
name = " copyfrom " ,
help = " Copy files/folders from a container directory to the local filesystem. " ,
)
@click.argument ( " service " )
@click.argument ( " container_path " )
@click.argument (
" host_path " ,
type = click . Path ( dir_okay = True , file_okay = False , resolve_path = True ) ,
)
@click.pass_obj
def copyfrom (
context : BaseComposeContext , service : str , container_path : str , host_path : str
) - > None :
# Path management
container_root_path = " /tmp/mount "
container_dst_path = container_root_path
if not os . path . exists ( host_path ) :
# Emulate cp semantics, where if the destination path does not exist
# then we copy to its parent and rename to the destination folder
container_dst_path + = " / " + os . path . basename ( host_path )
host_path = os . path . dirname ( host_path )
if not os . path . exists ( host_path ) :
raise TutorError (
f " Cannot create directory { host_path } . No such file or directory. "
)
# cp/mv commands
command = f " cp --recursive --preserve { container_path } { container_dst_path } "
config = tutor_config . load ( context . root )
runner = context . job_runner ( config )
runner . docker_compose (
" run " ,
" --rm " ,
" --no-deps " ,
" --user=0 " ,
f " --volume= { host_path } : { container_root_path } " ,
service ,
" sh " ,
" -e " ,
" -c " ,
command ,
)
2020-11-16 11:46:01 +00:00
@click.command (
short_help = " Run a command in a running container " ,
help = (
" Run a command in a running container. This is a wrapper around `docker-compose exec`. Any option or argument "
" passed to this command will be forwarded to docker-compose. Thus, you may use `-e` to manually define "
" environment variables. "
) ,
context_settings = { " ignore_unknown_options " : True } ,
name = " exec " ,
)
@click.argument ( " args " , nargs = - 1 , required = True )
2021-02-25 08:09:14 +00:00
@click.pass_context
refactor: annotation with __future__.annotations
Adds `from __future__ import annotations` to the top of every module,
right below the module's docstring. Replaces any usages of t.List,
t.Dict, t.Set, t.Tuple, and t.Type with their built-in equivalents:
list, dict, set, tuple, and type. Ensures that make test still passes
under Python 3.7, 3.8 and 3.9.
2023-01-17 18:57:23 +00:00
def execute ( context : click . Context , args : list [ str ] ) - > None :
2021-02-25 08:09:14 +00:00
context . invoke ( dc_command , command = " exec " , args = args )
2020-11-16 11:46:01 +00:00
@click.command (
short_help = " View output from containers " ,
help = " View output from containers. This is a wrapper around `docker-compose logs`. " ,
)
@click.option ( " -f " , " --follow " , is_flag = True , help = " Follow log output " )
@click.option ( " --tail " , type = int , help = " Number of lines to show from each container " )
@click.argument ( " service " , nargs = - 1 )
2021-02-25 08:09:14 +00:00
@click.pass_context
def logs ( context : click . Context , follow : bool , tail : bool , service : str ) - > None :
2020-12-25 21:56:42 +00:00
args = [ ]
2020-11-16 11:46:01 +00:00
if follow :
2020-12-25 21:56:42 +00:00
args . append ( " --follow " )
2020-11-16 11:46:01 +00:00
if tail is not None :
2020-12-25 21:56:42 +00:00
args + = [ " --tail " , str ( tail ) ]
args + = service
2021-02-25 08:09:14 +00:00
context . invoke ( dc_command , command = " logs " , args = args )
2020-12-25 21:56:42 +00:00
2022-04-08 16:02:21 +00:00
@click.command ( help = " Print status information for containers " )
@click.pass_context
def status ( context : click . Context ) - > None :
context . invoke ( dc_command , command = " ps " )
2020-12-25 21:56:42 +00:00
@click.command (
short_help = " Direct interface to docker-compose. " ,
help = (
" Direct interface to docker-compose. This is a wrapper around `docker-compose`. Most commands, options and "
" arguments passed to this command will be forwarded as-is to docker-compose. "
) ,
context_settings = { " ignore_unknown_options " : True } ,
name = " dc " ,
)
2022-07-25 17:19:28 +00:00
@mount_option
2020-12-25 21:56:42 +00:00
@click.argument ( " command " )
2021-05-04 15:57:47 +00:00
@click.argument ( " args " , nargs = - 1 )
2020-12-25 21:56:42 +00:00
@click.pass_obj
2022-07-25 17:19:28 +00:00
def dc_command (
context : BaseComposeContext ,
refactor: annotation with __future__.annotations
Adds `from __future__ import annotations` to the top of every module,
right below the module's docstring. Replaces any usages of t.List,
t.Dict, t.Set, t.Tuple, and t.Type with their built-in equivalents:
list, dict, set, tuple, and type. Ensures that make test still passes
under Python 3.7, 3.8 and 3.9.
2023-01-17 18:57:23 +00:00
mounts : tuple [ list [ MountParam . MountType ] ] ,
2022-07-25 17:19:28 +00:00
command : str ,
refactor: annotation with __future__.annotations
Adds `from __future__ import annotations` to the top of every module,
right below the module's docstring. Replaces any usages of t.List,
t.Dict, t.Set, t.Tuple, and t.Type with their built-in equivalents:
list, dict, set, tuple, and type. Ensures that make test still passes
under Python 3.7, 3.8 and 3.9.
2023-01-17 18:57:23 +00:00
args : list [ str ] ,
2022-07-25 17:19:28 +00:00
) - > None :
mount_tmp_volumes ( mounts , context )
2020-12-25 21:56:42 +00:00
config = tutor_config . load ( context . root )
2022-10-18 12:40:32 +00:00
context . job_runner ( config ) . docker_compose ( command , * args )
2020-11-16 11:46:01 +00:00
2022-04-15 08:51:19 +00:00
@hooks.Filters.COMPOSE_MOUNTS.add ( )
def _mount_edx_platform (
refactor: annotation with __future__.annotations
Adds `from __future__ import annotations` to the top of every module,
right below the module's docstring. Replaces any usages of t.List,
t.Dict, t.Set, t.Tuple, and t.Type with their built-in equivalents:
list, dict, set, tuple, and type. Ensures that make test still passes
under Python 3.7, 3.8 and 3.9.
2023-01-17 18:57:23 +00:00
volumes : list [ tuple [ str , str ] ] , name : str
) - > list [ tuple [ str , str ] ] :
2022-04-15 08:51:19 +00:00
"""
When mounting edx - platform with ` - - mount = / path / to / edx - platform ` , bind - mount the host
repo in the lms / cms containers .
"""
if name == " edx-platform " :
path = " /openedx/edx-platform "
volumes + = [
( " lms " , path ) ,
( " cms " , path ) ,
( " lms-worker " , path ) ,
( " cms-worker " , path ) ,
( " lms-job " , path ) ,
( " cms-job " , path ) ,
]
return volumes
2021-02-25 08:09:14 +00:00
def add_commands ( command_group : click . Group ) - > None :
2020-01-08 18:38:13 +00:00
command_group . add_command ( start )
command_group . add_command ( stop )
command_group . add_command ( restart )
command_group . add_command ( reboot )
2020-11-16 11:46:01 +00:00
command_group . add_command ( dc_command )
command_group . add_command ( run )
2022-04-20 20:24:17 +00:00
command_group . add_command ( copyfrom )
2020-11-16 11:46:01 +00:00
command_group . add_command ( execute )
command_group . add_command ( logs )
2022-04-08 16:02:21 +00:00
command_group . add_command ( status )
2022-10-19 15:46:31 +00:00
@hooks.Actions.PLUGINS_LOADED.add ( )
def _add_do_commands ( ) - > None :
jobs . add_job_commands ( do )
command_group . add_command ( do )