From b3c5c9685bf742431cb39a011b51bd6ec57a4c5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Sun, 26 Mar 2023 10:24:09 -0400 Subject: [PATCH 1/4] feat: custom importable courses This makes it possible to import courses not just from the demo repo. Close #730 --- .../20230326_102311_regis_moar_tasks.md | 1 + tutor/commands/jobs.py | 23 +++++++++++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) create mode 100644 changelog.d/20230326_102311_regis_moar_tasks.md diff --git a/changelog.d/20230326_102311_regis_moar_tasks.md b/changelog.d/20230326_102311_regis_moar_tasks.md new file mode 100644 index 0000000..cd6910c --- /dev/null +++ b/changelog.d/20230326_102311_regis_moar_tasks.md @@ -0,0 +1 @@ +- [Feature] Make it possible to import the demo course from a different git repository or version. (by @regisb) diff --git a/tutor/commands/jobs.py b/tutor/commands/jobs.py index 4cfa87a..47680c4 100644 --- a/tutor/commands/jobs.py +++ b/tutor/commands/jobs.py @@ -139,11 +139,26 @@ u.save()" @click.command(help="Import the demo course") -def importdemocourse() -> t.Iterable[tuple[str, str]]: - template = """ +@click.option( + "-r", + "--repo", + default="https://github.com/openedx/edx-demo-course", + show_default=True, + help="Git repository that contains the course to be imported", +) +@click.option( + "-v", + "--version", + help="Git branch, tag or sha1 identifier. If unspecified, will default to the value of the OPENEDX_COMMON_VERSION setting.", +) +def importdemocourse( + repo: str, version: t.Optional[str] +) -> t.Iterable[tuple[str, str]]: + version = version or "{{ OPENEDX_COMMON_VERSION }}" + template = f""" # Import demo course -git clone https://github.com/openedx/edx-demo-course --branch {{ OPENEDX_COMMON_VERSION }} --depth 1 ../edx-demo-course -python ./manage.py cms import ../data ../edx-demo-course +git clone {repo} --branch {version} --depth 1 /tmp/course +python ./manage.py cms import ../data /tmp/course # Re-index courses ./manage.py cms reindex_course --all --setup""" From 586045143f643066f461b04cb6cab9dd45d5fa00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Sun, 26 Mar 2023 17:54:00 -0400 Subject: [PATCH 2/4] feat: add a `do print-edx-platform-setting` command The command is pretty straightforward, but quite convenient. --- .../20230326_102311_regis_moar_tasks.md | 1 + tutor/commands/jobs.py | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/changelog.d/20230326_102311_regis_moar_tasks.md b/changelog.d/20230326_102311_regis_moar_tasks.md index cd6910c..16c16f3 100644 --- a/changelog.d/20230326_102311_regis_moar_tasks.md +++ b/changelog.d/20230326_102311_regis_moar_tasks.md @@ -1 +1,2 @@ - [Feature] Make it possible to import the demo course from a different git repository or version. (by @regisb) +- [Feature] Add a convenient `do print-edx-platform-setting` command to print the value of an edx-platform setting. (by @regisb) diff --git a/tutor/commands/jobs.py b/tutor/commands/jobs.py index 47680c4..9edffcc 100644 --- a/tutor/commands/jobs.py +++ b/tutor/commands/jobs.py @@ -222,6 +222,26 @@ def assign_theme(name, domain): return f'./manage.py lms shell -c "{python_command}"' +@click.command( + name="print-edx-platform-setting", + help="Print the value of an edx-platform Django setting.", +) +@click.argument("setting") +@click.option( + "-s", + "--service", + type=click.Choice(["lms", "cms"]), + default="lms", + show_default=True, + help="Service to fetch the setting from", +) +def print_edx_platform_setting( + setting: str, service: str +) -> t.Iterable[tuple[str, str]]: + command = f"./manage.py {service} shell -c 'from django.conf import settings; print(settings.{setting})'" + yield (service, command) + + def add_job_commands(do_command_group: click.Group) -> None: """ This is meant to be called with the `local/dev/k8s do` group commands, to add the @@ -300,6 +320,7 @@ hooks.Filters.CLI_DO_COMMANDS.add_items( createuser, importdemocourse, initialise, + print_edx_platform_setting, settheme, ] ) From 6257c1c7acb8bd81d9dc244e7710a1faac87d563 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Sun, 26 Mar 2023 18:29:18 -0400 Subject: [PATCH 3/4] feat: improve edx-platform logging by silencing a few warnings These warnings were occurring when launching a django shell ("shell" command) or a development server ("runserver"). --- changelog.d/20230326_102311_regis_moar_tasks.md | 1 + tutor/templates/apps/openedx/settings/partials/common_all.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/changelog.d/20230326_102311_regis_moar_tasks.md b/changelog.d/20230326_102311_regis_moar_tasks.md index 16c16f3..f7d4a87 100644 --- a/changelog.d/20230326_102311_regis_moar_tasks.md +++ b/changelog.d/20230326_102311_regis_moar_tasks.md @@ -1,2 +1,3 @@ - [Feature] Make it possible to import the demo course from a different git repository or version. (by @regisb) - [Feature] Add a convenient `do print-edx-platform-setting` command to print the value of an edx-platform setting. (by @regisb) +- [Improvement] Improve edx-platform logging by silencing a couple deprecation warnings. (by @regisb) diff --git a/tutor/templates/apps/openedx/settings/partials/common_all.py b/tutor/templates/apps/openedx/settings/partials/common_all.py index f23c8e1..222fe9e 100644 --- a/tutor/templates/apps/openedx/settings/partials/common_all.py +++ b/tutor/templates/apps/openedx/settings/partials/common_all.py @@ -143,6 +143,11 @@ warnings.filterwarnings("ignore", category=RemovedInDjango40Warning) warnings.filterwarnings("ignore", category=RemovedInDjango41Warning) warnings.filterwarnings("ignore", category=DeprecationWarning, module="lms.djangoapps.course_wiki.plugins.markdownedx.wiki_plugin") warnings.filterwarnings("ignore", category=DeprecationWarning, module="wiki.plugins.links.wiki_plugin") +warnings.filterwarnings("ignore", category=DeprecationWarning, module="boto.plugin") +warnings.filterwarnings("ignore", category=DeprecationWarning, module="botocore.vendored.requests.packages.urllib3._collections") +warnings.filterwarnings("ignore", category=DeprecationWarning, module="storages.backends.s3boto") +warnings.filterwarnings("ignore", category=DeprecationWarning, module="openedx.core.types.admin") +SILENCED_SYSTEM_CHECKS = ["2_0.W001", "fields.W903"] # Email EMAIL_USE_SSL = {{ SMTP_USE_SSL }} From 19016b8ab1eab344188cf642e4a0aa73c53af9de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Tue, 4 Apr 2023 14:19:31 +0200 Subject: [PATCH 4/4] feat: add `do sqlshell` command An optional `--db=openedx` argument can be passed to the job command. This should close https://github.com/openedx/wg-developer-experience/issues/51 --- .../20230326_102311_regis_moar_tasks.md | 1 + tutor/commands/jobs.py | 54 ++++++++++++------- tutor/templates/build/openedx/Dockerfile | 2 +- 3 files changed, 37 insertions(+), 20 deletions(-) diff --git a/changelog.d/20230326_102311_regis_moar_tasks.md b/changelog.d/20230326_102311_regis_moar_tasks.md index f7d4a87..9d85078 100644 --- a/changelog.d/20230326_102311_regis_moar_tasks.md +++ b/changelog.d/20230326_102311_regis_moar_tasks.md @@ -1,3 +1,4 @@ - [Feature] Make it possible to import the demo course from a different git repository or version. (by @regisb) - [Feature] Add a convenient `do print-edx-platform-setting` command to print the value of an edx-platform setting. (by @regisb) - [Improvement] Improve edx-platform logging by silencing a couple deprecation warnings. (by @regisb) +- [Feature] Add a convenient `do sqlshell` command to enter a SQL shell as root. (by @regisb) diff --git a/tutor/commands/jobs.py b/tutor/commands/jobs.py index 9edffcc..22de90c 100644 --- a/tutor/commands/jobs.py +++ b/tutor/commands/jobs.py @@ -10,7 +10,7 @@ import click from typing_extensions import ParamSpec from tutor import config as tutor_config -from tutor import env, fmt, hooks +from tutor import env, fmt, hooks, utils from tutor.hooks import priorities @@ -165,6 +165,26 @@ python ./manage.py cms import ../data /tmp/course yield ("cms", template) +@click.command( + name="print-edx-platform-setting", + help="Print the value of an edx-platform Django setting.", +) +@click.argument("setting") +@click.option( + "-s", + "--service", + type=click.Choice(["lms", "cms"]), + default="lms", + show_default=True, + help="Service to fetch the setting from", +) +def print_edx_platform_setting( + setting: str, service: str +) -> t.Iterable[tuple[str, str]]: + command = f"./manage.py {service} shell -c 'from django.conf import settings; print(settings.{setting})'" + yield (service, command) + + @click.command() @click.option( "-d", @@ -222,24 +242,19 @@ def assign_theme(name, domain): return f'./manage.py lms shell -c "{python_command}"' -@click.command( - name="print-edx-platform-setting", - help="Print the value of an edx-platform Django setting.", -) -@click.argument("setting") -@click.option( - "-s", - "--service", - type=click.Choice(["lms", "cms"]), - default="lms", - show_default=True, - help="Service to fetch the setting from", -) -def print_edx_platform_setting( - setting: str, service: str -) -> t.Iterable[tuple[str, str]]: - command = f"./manage.py {service} shell -c 'from django.conf import settings; print(settings.{setting})'" - yield (service, command) +@click.command(context_settings={"ignore_unknown_options": True}) +@click.argument("args", nargs=-1) +def sqlshell(args: list[str]) -> t.Iterable[tuple[str, str]]: + """ + Open an SQL shell as root + + Extra arguments will be passed to the `mysql` command verbatim. For instance, to + show tables from the "openedx" database, run `do sqlshell openedx -e 'show tables'`. + """ + command = "mysql --user={{ MYSQL_ROOT_USERNAME }} --password={{ MYSQL_ROOT_PASSWORD }} --host={{ MYSQL_HOST }} --port={{ MYSQL_PORT }}" + if args: + command += " " + utils._shlex_join(*args) # pylint: disable=protected-access + yield ("lms", command) def add_job_commands(do_command_group: click.Group) -> None: @@ -322,5 +337,6 @@ hooks.Filters.CLI_DO_COMMANDS.add_items( initialise, print_edx_platform_setting, settheme, + sqlshell, ] ) diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index 31aafd6..698f631 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -113,7 +113,7 @@ FROM minimal as production # Install system requirements RUN apt update && \ - apt install -y gettext gfortran graphviz graphviz-dev libffi-dev libfreetype6-dev libgeos-dev libjpeg8-dev liblapack-dev libmysqlclient-dev libpng-dev libsqlite3-dev libxmlsec1-dev lynx ntp pkg-config rdfind && \ + apt install -y gettext gfortran graphviz graphviz-dev libffi-dev libfreetype6-dev libgeos-dev libjpeg8-dev liblapack-dev libmysqlclient-dev libpng-dev libsqlite3-dev libxmlsec1-dev lynx mysql-client ntp pkg-config rdfind && \ rm -rf /var/lib/apt/lists/* # From then on, run as unprivileged "app" user