2022-10-18 14:57:07 +00:00
"""
Common jobs that must be added both to local , dev and k8s commands .
"""
refactor: annotation with __future__.annotations
Adds `from __future__ import annotations` to the top of every module,
right below the module's docstring. Replaces any usages of t.List,
t.Dict, t.Set, t.Tuple, and t.Type with their built-in equivalents:
list, dict, set, tuple, and type. Ensures that make test still passes
under Python 3.7, 3.8 and 3.9.
2023-01-17 18:57:23 +00:00
from __future__ import annotations
2023-01-06 18:02:17 +00:00
2022-10-19 15:46:31 +00:00
import functools
2022-10-18 14:57:07 +00:00
import typing as t
import click
2022-10-19 15:46:31 +00:00
from typing_extensions import ParamSpec
2022-10-18 14:57:07 +00:00
from tutor import config as tutor_config
2022-10-19 15:46:31 +00:00
from tutor import env , fmt , hooks
2022-10-18 14:57:07 +00:00
2022-10-19 15:46:31 +00:00
class DoGroup ( click . Group ) :
"""
A Click group that prints subcommands under ' Jobs ' instead of ' Commands ' when we run
` . . do - - help ` . Hackish but it works .
"""
def get_help ( self , ctx : click . Context ) - > str :
return super ( ) . get_help ( ctx ) . replace ( " Commands: \n " , " Jobs: \n " )
# A convenient easy-to-use decorator for creating `do` commands.
do_group = click . group ( cls = DoGroup , subcommand_metavar = " JOB [ARGS]... " )
2022-10-18 14:57:07 +00:00
@hooks.Actions.CORE_READY.add ( )
def _add_core_init_tasks ( ) - > None :
"""
Declare core init scripts at runtime .
The context is important , because it allows us to select the init scripts based on
the - - limit argument .
"""
with hooks . Contexts . APP ( " mysql " ) . enter ( ) :
2022-10-19 15:46:31 +00:00
hooks . Filters . CLI_DO_INIT_TASKS . add_item (
2022-11-21 08:56:59 +00:00
( " mysql " , env . read_core_template_file ( " jobs " , " init " , " mysql.sh " ) )
2022-10-19 15:46:31 +00:00
)
2022-10-18 14:57:07 +00:00
with hooks . Contexts . APP ( " lms " ) . enter ( ) :
2022-10-19 15:46:31 +00:00
hooks . Filters . CLI_DO_INIT_TASKS . add_item (
2022-11-21 08:56:59 +00:00
( " lms " , env . read_core_template_file ( " jobs " , " init " , " lms.sh " ) )
2022-10-19 15:46:31 +00:00
)
2022-10-18 14:57:07 +00:00
with hooks . Contexts . APP ( " cms " ) . enter ( ) :
2022-10-19 15:46:31 +00:00
hooks . Filters . CLI_DO_INIT_TASKS . add_item (
2022-11-21 08:56:59 +00:00
( " cms " , env . read_core_template_file ( " jobs " , " init " , " cms.sh " ) )
2022-10-19 15:46:31 +00:00
)
2022-10-18 14:57:07 +00:00
2022-10-19 15:46:31 +00:00
@click.command ( " init " , help = " Initialise all applications " )
@click.option ( " -l " , " --limit " , help = " Limit initialisation to this service or plugin " )
refactor: annotation with __future__.annotations
Adds `from __future__ import annotations` to the top of every module,
right below the module's docstring. Replaces any usages of t.List,
t.Dict, t.Set, t.Tuple, and t.Type with their built-in equivalents:
list, dict, set, tuple, and type. Ensures that make test still passes
under Python 3.7, 3.8 and 3.9.
2023-01-17 18:57:23 +00:00
def initialise ( limit : t . Optional [ str ] ) - > t . Iterator [ tuple [ str , str ] ] :
2022-10-18 14:57:07 +00:00
fmt . echo_info ( " Initialising all services... " )
2022-10-19 15:46:31 +00:00
filter_context = hooks . Contexts . APP ( limit ) . name if limit else None
2022-10-18 14:57:07 +00:00
2022-10-19 15:46:31 +00:00
# Deprecated pre-init tasks
2022-11-15 15:59:19 +00:00
for service , path in hooks . Filters . COMMANDS_PRE_INIT . iterate_from_context (
filter_context
) :
2022-10-19 15:46:31 +00:00
fmt . echo_alert (
f " Running deprecated pre-init task: { ' / ' . join ( path ) } . Init tasks should no longer be added to the COMMANDS_PRE_INIT filter. Plugin developers should use the CLI_DO_INIT_TASKS filter instead, with a high priority. "
)
yield service , env . read_template_file ( * path )
2022-10-18 14:57:07 +00:00
# Init tasks
2022-11-15 15:59:19 +00:00
for service , task in hooks . Filters . CLI_DO_INIT_TASKS . iterate_from_context (
filter_context
) :
2022-10-19 15:46:31 +00:00
fmt . echo_info ( f " Running init task in { service } " )
yield service , task
# Deprecated init tasks
2022-11-15 15:59:19 +00:00
for service , path in hooks . Filters . COMMANDS_INIT . iterate_from_context (
filter_context
) :
2022-10-19 15:46:31 +00:00
fmt . echo_alert (
f " Running deprecated init task: { ' / ' . join ( path ) } . Init tasks should no longer be added to the COMMANDS_INIT filter. Plugin developers should use the CLI_DO_INIT_TASKS filter instead. "
)
yield service , env . read_template_file ( * path )
2022-10-18 14:57:07 +00:00
fmt . echo_info ( " All services initialised. " )
@click.command ( help = " Create an Open edX user and interactively set their password " )
@click.option ( " --superuser " , is_flag = True , help = " Make superuser " )
@click.option ( " --staff " , is_flag = True , help = " Make staff user " )
@click.option (
" -p " ,
" --password " ,
help = " Specify password from the command line. If undefined, you will be prompted to input a password " ,
prompt = True ,
hide_input = True ,
)
@click.argument ( " name " )
@click.argument ( " email " )
def createuser (
superuser : str ,
staff : bool ,
password : str ,
name : str ,
email : str ,
refactor: annotation with __future__.annotations
Adds `from __future__ import annotations` to the top of every module,
right below the module's docstring. Replaces any usages of t.List,
t.Dict, t.Set, t.Tuple, and t.Type with their built-in equivalents:
list, dict, set, tuple, and type. Ensures that make test still passes
under Python 3.7, 3.8 and 3.9.
2023-01-17 18:57:23 +00:00
) - > t . Iterable [ tuple [ str , str ] ] :
2022-10-19 15:46:31 +00:00
"""
Create an Open edX user
2022-10-18 14:57:07 +00:00
2022-10-19 15:46:31 +00:00
Password can be passed as an option or will be set interactively .
"""
yield ( " lms " , create_user_template ( superuser , staff , name , email , password ) )
2022-10-18 14:57:07 +00:00
def create_user_template (
superuser : str , staff : bool , username : str , email : str , password : str
) - > str :
opts = " "
if superuser :
opts + = " --superuser "
if staff :
opts + = " --staff "
2022-10-19 15:46:31 +00:00
return f """
2022-10-18 14:57:07 +00:00
. / manage . py lms manage_user { opts } { username } { email }
. / manage . py lms shell - c "
from django . contrib . auth import get_user_model
u = get_user_model ( ) . objects . get ( username = ' {username} ' )
u . set_password ( ' {password} ' )
u . save ( ) "
"""
2022-10-19 15:46:31 +00:00
@click.command ( help = " Import the demo course " )
refactor: annotation with __future__.annotations
Adds `from __future__ import annotations` to the top of every module,
right below the module's docstring. Replaces any usages of t.List,
t.Dict, t.Set, t.Tuple, and t.Type with their built-in equivalents:
list, dict, set, tuple, and type. Ensures that make test still passes
under Python 3.7, 3.8 and 3.9.
2023-01-17 18:57:23 +00:00
def importdemocourse ( ) - > t . Iterable [ tuple [ str , str ] ] :
2022-10-19 15:46:31 +00:00
template = """
2022-10-18 14:57:07 +00:00
# Import demo course
git clone https : / / github . com / openedx / edx - demo - course - - branch { { OPENEDX_COMMON_VERSION } } - - depth 1 . . / edx - demo - course
python . / manage . py cms import . . / data . . / edx - demo - course
# Re-index courses
. / manage . py cms reindex_course - - all - - setup """
2022-10-19 15:46:31 +00:00
yield ( " cms " , template )
@click.command ( )
@click.option (
" -d " ,
" --domain " ,
" domains " ,
multiple = True ,
help = (
" Limit the theme to these domain names. By default, the theme is "
" applied to the LMS and the CMS, both in development and production mode "
) ,
)
@click.argument ( " theme_name " )
refactor: annotation with __future__.annotations
Adds `from __future__ import annotations` to the top of every module,
right below the module's docstring. Replaces any usages of t.List,
t.Dict, t.Set, t.Tuple, and t.Type with their built-in equivalents:
list, dict, set, tuple, and type. Ensures that make test still passes
under Python 3.7, 3.8 and 3.9.
2023-01-17 18:57:23 +00:00
def settheme ( domains : list [ str ] , theme_name : str ) - > t . Iterable [ tuple [ str , str ] ] :
2022-10-19 15:46:31 +00:00
"""
Assign a theme to the LMS and the CMS .
To reset to the default theme , use ' default ' as the theme name .
"""
yield ( " lms " , set_theme_template ( theme_name , domains ) )
2022-10-18 14:57:07 +00:00
refactor: annotation with __future__.annotations
Adds `from __future__ import annotations` to the top of every module,
right below the module's docstring. Replaces any usages of t.List,
t.Dict, t.Set, t.Tuple, and t.Type with their built-in equivalents:
list, dict, set, tuple, and type. Ensures that make test still passes
under Python 3.7, 3.8 and 3.9.
2023-01-17 18:57:23 +00:00
def set_theme_template ( theme_name : str , domain_names : list [ str ] ) - > str :
2022-10-18 14:57:07 +00:00
"""
For each domain , get or create a Site object and assign the selected theme .
"""
# Note that there are no double quotes " in this piece of code
python_command = """
import sys
from django . contrib . sites . models import Site
def assign_theme ( name , domain ) :
print ( ' Assigning theme ' , name , ' to ' , domain )
if len ( domain ) > 50 :
sys . stderr . write (
' Assigning a theme to a site with a long (> 50 characters) domain name. '
' The displayed site name will be truncated to 50 characters. \\ n '
)
site , _ = Site . objects . get_or_create ( domain = domain )
if not site . name :
name_max_length = Site . _meta . get_field ( ' name ' ) . max_length
site . name = domain [ : name_max_length ]
site . save ( )
site . themes . all ( ) . delete ( )
site . themes . create ( theme_dir_name = name )
"""
domain_names = domain_names or [
" {{ LMS_HOST }} " ,
" {{ LMS_HOST }}:8000 " ,
" {{ CMS_HOST }} " ,
" {{ CMS_HOST }}:8001 " ,
" {{ PREVIEW_LMS_HOST }} " ,
" {{ PREVIEW_LMS_HOST }}:8000 " ,
]
for domain_name in domain_names :
python_command + = f " assign_theme( ' { theme_name } ' , ' { domain_name } ' ) \n "
2022-10-19 15:46:31 +00:00
return f ' ./manage.py lms shell -c " { python_command } " '
2022-11-24 17:20:23 +00:00
def add_job_commands ( do_command_group : click . Group ) - > None :
2022-10-19 15:46:31 +00:00
"""
2022-11-24 17:20:23 +00:00
This is meant to be called with the ` local / dev / k8s do ` group commands , to add the
different ` do ` subcommands .
2022-10-19 15:46:31 +00:00
"""
2022-11-24 17:20:23 +00:00
for subcommand in hooks . Filters . CLI_DO_COMMANDS . iterate ( ) :
assert isinstance ( subcommand , click . Command )
do_command_group . add_command ( subcommand )
2022-10-19 15:46:31 +00:00
@hooks.Actions.PLUGINS_LOADED.add ( )
def _patch_do_commands_callbacks ( ) - > None :
"""
After plugins have been loaded , patch ` do ` subcommands such that their output is
forwarded to ` do_callback ` .
2022-11-24 17:20:23 +00:00
This function is not called as part of add_job_commands because subcommands must be
patched just once .
2022-10-19 15:46:31 +00:00
"""
2022-11-24 17:20:23 +00:00
for subcommand in hooks . Filters . CLI_DO_COMMANDS . iterate ( ) :
if not isinstance ( subcommand , click . Command ) :
raise ValueError (
f " Command { subcommand } which was added to the CLI_DO_COMMANDS filter must be an instance of click.Command "
)
2022-10-19 15:46:31 +00:00
# Modify the subcommand callback such that job results are processed by do_callback
if subcommand . callback is None :
raise ValueError ( " Cannot patch None callback " )
if subcommand . name is None :
raise ValueError ( " Defined job with None name " )
subcommand . callback = _patch_callback ( subcommand . name , subcommand . callback )
P = ParamSpec ( " P " )
def _patch_callback (
refactor: annotation with __future__.annotations
Adds `from __future__ import annotations` to the top of every module,
right below the module's docstring. Replaces any usages of t.List,
t.Dict, t.Set, t.Tuple, and t.Type with their built-in equivalents:
list, dict, set, tuple, and type. Ensures that make test still passes
under Python 3.7, 3.8 and 3.9.
2023-01-17 18:57:23 +00:00
job_name : str , func : t . Callable [ P , t . Iterable [ tuple [ str , str ] ] ]
2022-10-19 15:46:31 +00:00
) - > t . Callable [ P , None ] :
"""
Modify a subcommand callback function such that its results are processed by ` do_callback ` .
"""
def new_callback ( * args : P . args , * * kwargs : P . kwargs ) - > None :
2022-11-15 15:59:19 +00:00
hooks . Actions . DO_JOB . do ( job_name , * args , * * kwargs )
2022-10-19 15:46:31 +00:00
do_callback ( func ( * args , * * kwargs ) )
2022-10-18 14:57:07 +00:00
2022-10-19 15:46:31 +00:00
# Make the new callback behave like the old one
functools . update_wrapper ( new_callback , func )
2022-10-18 14:57:07 +00:00
2022-10-19 15:46:31 +00:00
return new_callback
2022-11-24 17:20:23 +00:00
refactor: annotation with __future__.annotations
Adds `from __future__ import annotations` to the top of every module,
right below the module's docstring. Replaces any usages of t.List,
t.Dict, t.Set, t.Tuple, and t.Type with their built-in equivalents:
list, dict, set, tuple, and type. Ensures that make test still passes
under Python 3.7, 3.8 and 3.9.
2023-01-17 18:57:23 +00:00
def do_callback ( service_commands : t . Iterable [ tuple [ str , str ] ] ) - > None :
2022-11-24 17:20:23 +00:00
"""
This function must be added as a callback to all ` do ` subcommands .
` do ` subcommands don ' t actually run any task. They just yield tuples of (service
name , unrendered script string ) . This function is responsible for actually running
the scripts . It does the following :
- Prefix the script with a base command
- Render the script string
- Run a job in the right container
This callback is added to the " do " subcommands by the ` add_job_commands ` function .
"""
context = click . get_current_context ( ) . obj
config = tutor_config . load ( context . root )
runner = context . job_runner ( config )
for service , command in service_commands :
2022-11-24 17:26:00 +00:00
runner . run_task_from_str ( service , command )
2022-11-24 17:20:23 +00:00
hooks . Filters . CLI_DO_COMMANDS . add_items (
[
createuser ,
importdemocourse ,
initialise ,
settheme ,
]
)