From c3c914f22f9b26dd7579305077636dd7f38b1b2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Tue, 7 Sep 2021 13:17:43 +0200 Subject: [PATCH 01/56] feat: upgrade to nightly Get Tutor to work on the master branches of Open edX. The corresponding images will have to be rebuilt manually. Note that the process to contribute to the nightly branch is slightly different from the master branch (see the instructions from the corresponding tutorial). --- tutor/__about__.py | 2 +- tutor/templates/build/openedx/Dockerfile | 6 ++---- tutor/templates/config.yml | 8 ++++---- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/tutor/__about__.py b/tutor/__about__.py index 2da2ef6..e3eecf3 100644 --- a/tutor/__about__.py +++ b/tutor/__about__.py @@ -10,7 +10,7 @@ __version__ = "12.1.4" # the nightly branch. # The suffix is cleanly separated from the __version__ in this module to avoid # conflicts when merging branches. -__version_suffix__ = "" +__version_suffix__ = "nightly" # The app name will be used to define the name of the default tutor root and # plugin directory. To avoid conflicts between multiple locally-installed diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index 881ac46..b314e7c 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -44,8 +44,6 @@ RUN git config --global user.email "tutor@overhang.io" \ {{ patch("openedx-dockerfile-git-patches-default") }} {% else %} # Patch edx-platform -# Security patch: https://github.com/edx/edx-platform/pull/28442 -RUN git fetch https://github.com/edx/edx-platform 8ecc1903ca9170a719c0e63e99fb231822eb26d8 && git cherry-pick 8ecc1903ca9170a719c0e63e99fb231822eb26d8 {% endif %} {# Example: RUN git fetch https://github.com/edx/edx-platform && git cherry-pick #} @@ -75,7 +73,7 @@ COPY --from=code /openedx/edx-platform /openedx/edx-platform WORKDIR /openedx/edx-platform # Install the right version of pip/setuptools -RUN pip install setuptools==44.1.0 pip==20.0.2 wheel==0.34.2 +RUN pip install setuptools==44.1.0 pip==20.3.4 wheel==0.37.0 # Install base requirements RUN pip install -r ./requirements/edx/base.txt @@ -84,7 +82,7 @@ RUN pip install -r ./requirements/edx/base.txt RUN pip install "openedx-scorm-xblock<13.0.0,>=12.0.0" # Install django-redis for using redis as a django cache -RUN pip install django-redis==4.12.1 +RUN pip install django-redis==5.0.0 # Install uwsgi RUN pip install uwsgi==2.0.19.1 diff --git a/tutor/templates/config.yml b/tutor/templates/config.yml index e46af60..4b2bc1c 100644 --- a/tutor/templates/config.yml +++ b/tutor/templates/config.yml @@ -31,10 +31,10 @@ DOCKER_IMAGE_OPENEDX_DEV: "{{ DOCKER_REGISTRY }}overhangio/openedx-dev:{{ TUTOR_ DOCKER_IMAGE_CADDY: "{{ DOCKER_REGISTRY }}caddy:2.3.0" DOCKER_IMAGE_FORUM: "{{ DOCKER_REGISTRY }}overhangio/openedx-forum:{{ TUTOR_VERSION }}" DOCKER_IMAGE_MONGODB: "{{ DOCKER_REGISTRY }}mongo:4.0.25" -DOCKER_IMAGE_MYSQL: "{{ DOCKER_REGISTRY }}mysql:5.7.33" +DOCKER_IMAGE_MYSQL: "{{ DOCKER_REGISTRY }}mysql:5.7.35" DOCKER_IMAGE_ELASTICSEARCH: "{{ DOCKER_REGISTRY }}elasticsearch:7.8.1" -DOCKER_IMAGE_NGINX: "{{ DOCKER_REGISTRY }}nginx:1.19.9" -DOCKER_IMAGE_REDIS: "{{ DOCKER_REGISTRY }}redis:6.2.1" +DOCKER_IMAGE_NGINX: "{{ DOCKER_REGISTRY }}nginx:1.21.1" +DOCKER_IMAGE_REDIS: "{{ DOCKER_REGISTRY }}redis:6.2.5" DOCKER_IMAGE_SMTP: "{{ DOCKER_REGISTRY }}namshi/smtp:latest" LOCAL_PROJECT_NAME: "{{ TUTOR_APP }}_local" ELASTICSEARCH_HOST: "elasticsearch" @@ -61,7 +61,7 @@ OPENEDX_LMS_UWSGI_WORKERS: 2 OPENEDX_MYSQL_DATABASE: "openedx" OPENEDX_CSMH_MYSQL_DATABASE: "{{ OPENEDX_MYSQL_DATABASE }}_csmh" OPENEDX_MYSQL_USERNAME: "openedx" -OPENEDX_COMMON_VERSION: "open-release/lilac.2" +OPENEDX_COMMON_VERSION: "master" MYSQL_HOST: "mysql" MYSQL_PORT: 3306 MYSQL_ROOT_USERNAME: "root" From eed13cdeed7a23b922c4a1f8738f78e92b0117b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Mon, 25 Oct 2021 14:16:10 +0200 Subject: [PATCH 02/56] chore: upgrade elasticsearch/mongodb/redis Open edX master now runs elasticsearch 7.10 and mongodb 4.2. Redis also received a minor upgrade. --- tutor/templates/config.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tutor/templates/config.yml b/tutor/templates/config.yml index 4b2bc1c..d8798ad 100644 --- a/tutor/templates/config.yml +++ b/tutor/templates/config.yml @@ -30,11 +30,11 @@ DOCKER_IMAGE_OPENEDX: "{{ DOCKER_REGISTRY }}overhangio/openedx:{{ TUTOR_VERSION DOCKER_IMAGE_OPENEDX_DEV: "{{ DOCKER_REGISTRY }}overhangio/openedx-dev:{{ TUTOR_VERSION }}" DOCKER_IMAGE_CADDY: "{{ DOCKER_REGISTRY }}caddy:2.3.0" DOCKER_IMAGE_FORUM: "{{ DOCKER_REGISTRY }}overhangio/openedx-forum:{{ TUTOR_VERSION }}" -DOCKER_IMAGE_MONGODB: "{{ DOCKER_REGISTRY }}mongo:4.0.25" +DOCKER_IMAGE_MONGODB: "{{ DOCKER_REGISTRY }}mongo:4.2.17" DOCKER_IMAGE_MYSQL: "{{ DOCKER_REGISTRY }}mysql:5.7.35" -DOCKER_IMAGE_ELASTICSEARCH: "{{ DOCKER_REGISTRY }}elasticsearch:7.8.1" +DOCKER_IMAGE_ELASTICSEARCH: "{{ DOCKER_REGISTRY }}elasticsearch:7.10.1" DOCKER_IMAGE_NGINX: "{{ DOCKER_REGISTRY }}nginx:1.21.1" -DOCKER_IMAGE_REDIS: "{{ DOCKER_REGISTRY }}redis:6.2.5" +DOCKER_IMAGE_REDIS: "{{ DOCKER_REGISTRY }}redis:6.2.6" DOCKER_IMAGE_SMTP: "{{ DOCKER_REGISTRY }}namshi/smtp:latest" LOCAL_PROJECT_NAME: "{{ TUTOR_APP }}_local" ELASTICSEARCH_HOST: "elasticsearch" @@ -68,6 +68,7 @@ MYSQL_ROOT_USERNAME: "root" NGINX_HTTP_PORT: 80 PLATFORM_NAME: "My Open edX" PLUGINS: [] +PREVIEW_LMS_HOST: "preview.{{ LMS_HOST }}" REDIS_HOST: "redis" REDIS_PORT: 6379 REDIS_USERNAME: "" From 8cb74b202a3fccd55a572031ed286e709be5074a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Mon, 25 Oct 2021 14:22:08 +0200 Subject: [PATCH 03/56] fix: running mongodb locally and on k8s --- tutor/templates/k8s/deployments.yml | 2 +- tutor/templates/local/docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tutor/templates/k8s/deployments.yml b/tutor/templates/k8s/deployments.yml index 738ca15..c0d479b 100644 --- a/tutor/templates/k8s/deployments.yml +++ b/tutor/templates/k8s/deployments.yml @@ -306,7 +306,7 @@ spec: containers: - name: mongodb image: {{ DOCKER_IMAGE_MONGODB }} - args: ["mongod", "--smallfiles", "--nojournal", "--storageEngine", "wiredTiger"] + args: ["mongod", "--nojournal", "--storageEngine", "wiredTiger"] ports: - containerPort: 27017 volumeMounts: diff --git a/tutor/templates/local/docker-compose.yml b/tutor/templates/local/docker-compose.yml index bab2413..cef04c2 100644 --- a/tutor/templates/local/docker-compose.yml +++ b/tutor/templates/local/docker-compose.yml @@ -7,7 +7,7 @@ services: mongodb: image: {{ DOCKER_IMAGE_MONGODB }} # Use WiredTiger in all environments, just like at edx.org - command: mongod --smallfiles --nojournal --storageEngine wiredTiger + command: mongod --nojournal --storageEngine wiredTiger restart: unless-stopped volumes: - ../../data/mongodb:/data/db From 4f034f83d9ae6ed8b80bd5a219387fdd1af96892 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Mon, 25 Oct 2021 14:35:19 +0200 Subject: [PATCH 04/56] fix: lms 500 error caused by missing LANGUAGE_COOKIE_NAME setting See also: https://github.com/overhangio/tutor/pull/507 Upstream fix: https://github.com/edx/edx-platform/pull/29096 --- tutor/templates/apps/openedx/settings/partials/common_all.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tutor/templates/apps/openedx/settings/partials/common_all.py b/tutor/templates/apps/openedx/settings/partials/common_all.py index 634093c..82c1190 100644 --- a/tutor/templates/apps/openedx/settings/partials/common_all.py +++ b/tutor/templates/apps/openedx/settings/partials/common_all.py @@ -109,8 +109,10 @@ ACE_CHANNEL_DEFAULT_EMAIL = "django_email" ACE_CHANNEL_TRANSACTIONAL_EMAIL = "django_email" EMAIL_FILE_PATH = "/tmp/openedx/emails" +# Language/locales LOCALE_PATHS.append("/openedx/locale/contrib/locale") LOCALE_PATHS.append("/openedx/locale/user/locale") +LANGUAGE_COOKIE_NAME = "openedx-language-preference" # Allow the platform to include itself in an iframe X_FRAME_OPTIONS = "SAMEORIGIN" From e19f334ebbbb73b7d8955bea23bdbf42ad8a62e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Thu, 14 Oct 2021 12:47:23 +0200 Subject: [PATCH 05/56] feat: get rid of the nginx container and services Nginx and Caddy performed duplicate tasks. It was decided to get rid of the nginx container, for simplification. This is a breaking change for plugin developers. Also, applications that collect nginx logs will have to be modified. See: - Corresponding TEP: https://discuss.overhang.io/t/tep-get-rid-of-the-nginx-container/2024 - the prior discussion: https://discuss.overhang.io/t/why-caddy-nginx/1952 --- CHANGELOG-nightly.md | 5 ++ docs/configuration.rst | 14 ++-- docs/troubleshooting.rst | 6 +- docs/tutorials/multiplatforms.rst | 2 +- docs/tutorials/proxy.rst | 8 +-- tutor/commands/images.py | 1 - tutor/commands/k8s.py | 6 +- tutor/config.py | 6 ++ tutor/env.py | 8 +-- tutor/templates/apps/caddy/Caddyfile | 68 +++++++++++++++++-- tutor/templates/apps/nginx/_tutor.conf | 10 --- tutor/templates/apps/nginx/cms.conf | 28 -------- tutor/templates/apps/nginx/extra.conf | 1 - tutor/templates/apps/nginx/lms.conf | 47 ------------- tutor/templates/config.yml | 7 +- tutor/templates/k8s/deployments.yml | 32 +-------- tutor/templates/k8s/services.yml | 14 +--- tutor/templates/k8s/volumes.yml | 2 +- tutor/templates/kustomization.yml | 3 - tutor/templates/local/docker-compose.prod.yml | 31 +++------ 20 files changed, 107 insertions(+), 192 deletions(-) delete mode 100644 tutor/templates/apps/nginx/_tutor.conf delete mode 100644 tutor/templates/apps/nginx/cms.conf delete mode 100644 tutor/templates/apps/nginx/extra.conf delete mode 100644 tutor/templates/apps/nginx/lms.conf diff --git a/CHANGELOG-nightly.md b/CHANGELOG-nightly.md index 41ccab4..4844fda 100644 --- a/CHANGELOG-nightly.md +++ b/CHANGELOG-nightly.md @@ -1,3 +1,8 @@ # Changelog (nightly branch) Note: Breaking changes between versions are indicated by "💥". + +- 💥[Feature] Get rid of the nginx container and service, which is now replaced by Caddy. this has the following consequences: + - Patches "nginx-cms", "nginx-lms", "nginx-extra", "local-docker-compose-nginx-aliases" are replaced by "caddyfile-cms", "caddyfile-lms", "caddyfile", " local-docker-compose-caddy-aliases". + - Patches "k8s-deployments-nginx-volume-mounts", "k8s-deployments-nginx-volumes" were obsolete and are removed. + - The `NGINX_HTTP_PORT` setting is renamed to `CADDY_HTTP_PORT`. \ No newline at end of file diff --git a/docs/configuration.rst b/docs/configuration.rst index e305901..f5e0a89 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -99,16 +99,10 @@ Vendor services Caddy ***** -- ``RUN_CADDY`` (default: ``true``) +- ``CADDY_HTTP_PORT`` (default: ``80``) +- ``ENABLE_WEB_PROXY`` (default: ``true``) -`Caddy `__ is a web server used in Tutor as a web proxy for the generation of SSL/TLS certificates at runtime. If ``RUN_CADDY`` is set to ``false`` then we assume that SSL termination does not occur in the Caddy container, and thus the ``caddy`` container is not started. - -Nginx -***** - -- ``NGINX_HTTP_PORT`` (default: ``80``) - -Nginx is used to route web traffic to the various applications and to serve static assets. When ``RUN_CADDY`` is false, the ``NGINX_HTTP_PORT`` is exposed on the host. +`Caddy `__ is a web server used in Tutor both as a web proxy and for the generation of SSL/TLS certificates at runtime. Port indicated by ``CADDY_HTTP_PORT`` is exposed on the host, in addition to port 443. If ``ENABLE_WEB_PROXY`` is set to ``false`` then we assume that SSL termination does not occur in the Caddy container and only ``CADDY_HTTP_PORT`` is exposed on the host. MySQL ***** @@ -193,7 +187,7 @@ The following DNS records must exist and point to your server:: Thus, **this feature will (probably) not work in development** because the DNS records will (probably) not point to your development machine. -If you would like to perform SSL/TLS termination with your own custom certificates, you will have to keep ``ENABLE_HTTPS=true`` and turn off the Caddy server with ``RUN_CADDY=false``. See the corresponding :ref:`tutorial ` for more information. +If you would like to perform SSL/TLS termination with your own custom certificates, you will have to keep ``ENABLE_HTTPS=true`` and turn off the Caddy load balancing with ``ENABLE_WEB_PROXY=false``. See the corresponding :ref:`tutorial ` for more information. .. _customise: diff --git a/docs/troubleshooting.rst b/docs/troubleshooting.rst index 0b3671d..7a5b0f7 100644 --- a/docs/troubleshooting.rst +++ b/docs/troubleshooting.rst @@ -33,7 +33,7 @@ To view the logs from all containers use the ``tutor local logs`` command, which To view the logs from just one container, for instance the web server:: - tutor local logs --follow nginx + tutor local logs --follow caddy The last commands produce the logs since the creation of the containers, which can be a lot. Similar to a ``tail -f``, you can run:: @@ -43,10 +43,10 @@ If you'd rather use a graphical user interface for viewing logs, you are encoura .. _webserver: -"Cannot start service nginx: driver failed programming external connectivity" +"Cannot start service caddy: driver failed programming external connectivity" ----------------------------------------------------------------------------- -The containerized Nginx needs to listen to ports 80 and 443 on the host. If there is already a webserver, such as Apache or Nginx, running on the host, the nginx container will not be able to start. To solve this issue, check the section on :ref:`how to setup a web proxy `. +The containerized Caddy needs to listen to ports 80 and 443 on the host. If there is already a webserver, such as Apache, Caddy or Nginx, running on the host, the caddy container will not be able to start. To solve this issue, check the section on :ref:`how to setup a web proxy `. "Couldn't connect to docker daemon" ----------------------------------- diff --git a/docs/tutorials/multiplatforms.rst b/docs/tutorials/multiplatforms.rst index 74272a5..5d8ebf0 100644 --- a/docs/tutorials/multiplatforms.rst +++ b/docs/tutorials/multiplatforms.rst @@ -5,7 +5,7 @@ With Tutor, it is easy to run multiple Open edX instances on a single server. To - ``TUTOR_ROOT``: so that configuration, environment and data are not mixed up between platforms. - ``LOCAL_PROJECT_NAME``: the various docker-compose projects cannot share the same name. -- ``NGINX_HTTP_PORT``: ports cannot be shared by two different containers. +- ``CADDY_HTTP_PORT``: exposed ports cannot be shared by two different containers. - ``LMS_HOST``, ``CMS_HOST``: the different platforms must be accessible from different domain (or subdomain) names. In addition, a web proxy must be setup on the host, as described :ref:`in the corresponding tutorial `. diff --git a/docs/tutorials/proxy.rst b/docs/tutorials/proxy.rst index c0f4e48..96ff8ce 100644 --- a/docs/tutorials/proxy.rst +++ b/docs/tutorials/proxy.rst @@ -5,11 +5,11 @@ Running Open edX behind a web proxy The containerized web server (`Caddy `__) needs to listen to ports 80 and 443 on the host. If there is already a webserver running on the host, such as Apache or Nginx, the caddy container will not be able to start. Tutor supports running behind a web proxy. To do so, add the following configuration:: - tutor config save --set RUN_CADDY=false --set NGINX_HTTP_PORT=81 + tutor config save --set ENABLE_WEB_PROXY=false --set CADDY_HTTP_PORT=81 -In this example, the nginx container port would be mapped to 81 instead of 80. You must then configure the web proxy on the host. As of v11.0.0, configuration files are no longer provided for automatic configuration of your web proxy. Basically, you should setup a reverse proxy to `localhost:NGINX_HTTP_PORT` from the following hosts: LMS_HOST, PREVIEW_LMS_HOST, CMS_HOST, as well as any additional host exposed by your plugins. +In this example, the caddy container port would be mapped to 81 instead of 80. You must then configure the web proxy on the host. As of v11.0.0, configuration files are no longer provided for automatic configuration of your web proxy. Basically, you should setup a reverse proxy to `localhost:CADDY_HTTP_PORT` from the following hosts: LMS_HOST, PREVIEW_LMS_HOST, CMS_HOST, as well as any additional host exposed by your plugins. .. warning:: - In this setup, the Nginx HTTP port will be exposed to the world. Make sure to configure your server firewall to block unwanted connections to your server's ``NGINX_HTTP_PORT``. Alternatively, you can configure the Nginx container to accept only local connections:: + In this setup, the Caddy HTTP port will be exposed to the world. Make sure to configure your server firewall to block unwanted connections to your server's ``CADDY_HTTP_PORT``. Alternatively, you can configure the Caddy container to accept only local connections:: - tutor config save --set NGINX_HTTP_PORT=127.0.0.1:81 + tutor config save --set CADDY_HTTP_PORT=127.0.0.1:81 diff --git a/tutor/commands/images.py b/tutor/commands/images.py index 0bda321..816cc6c 100644 --- a/tutor/commands/images.py +++ b/tutor/commands/images.py @@ -18,7 +18,6 @@ VENDOR_IMAGES = [ "elasticsearch", "mongodb", "mysql", - "nginx", "redis", "smtp", ] diff --git a/tutor/commands/k8s.py b/tutor/commands/k8s.py index 77bb0f8..dc97b50 100644 --- a/tutor/commands/k8s.py +++ b/tutor/commands/k8s.py @@ -165,14 +165,14 @@ def quickstart(context: click.Context, non_interactive: bool) -> None: config = interactive_config.update( context.obj.root, interactive=(not non_interactive) ) - if not config["RUN_CADDY"]: + if not config["ENABLE_WEB_PROXY"]: fmt.echo_alert( - "Potentially invalid configuration: RUN_CADDY=false\n" + "Potentially invalid configuration: ENABLE_WEB_PROXY=false\n" "This setting might have been defined because you previously set WEB_PROXY=true. This is no longer" " necessary in order to get Tutor to work on Kubernetes. In Tutor v11+ a Caddy-based load balancer is" " provided out of the box to handle SSL/TLS certificate generation at runtime. If you disable this" " service, you will have to configure an Ingress resource and a certificate manager yourself to redirect" - " traffic to the nginx service. See the Kubernetes section in the Tutor documentation for more" + " traffic to the caddy service. See the Kubernetes section in the Tutor documentation for more" " information." ) click.echo(fmt.title("Updating the current environment")) diff --git a/tutor/config.py b/tutor/config.py index fda00aa..7770c82 100644 --- a/tutor/config.py +++ b/tutor/config.py @@ -174,6 +174,12 @@ def upgrade_obsolete(config: Config) -> None: ]: if name in config: config[name.replace("ACTIVATE_", "RUN_")] = config.pop(name) + # Replace RUN_CADDY by ENABLE_WEB_PROXY + if "RUN_CADDY" in config: + config["ENABLE_WEB_PROXY"] = config.pop("RUN_CADDY") + # Replace RUN_CADDY by ENABLE_WEB_PROXY + if "NGINX_HTTP_PORT" in config: + config["CADDY_HTTP_PORT"] = config.pop("NGINX_HTTP_PORT") def convert_json2yml(root: str) -> None: diff --git a/tutor/env.py b/tutor/env.py index 31dd64d..f591db3 100644 --- a/tutor/env.py +++ b/tutor/env.py @@ -222,11 +222,9 @@ def save(root: str, config: Config) -> None: def upgrade_obsolete(root: str) -> None: - # tutor.conf was renamed to _tutor.conf in order to be the first config file loaded - # by nginx - nginx_tutor_conf = pathjoin(root, "apps", "nginx", "tutor.conf") - if os.path.exists(nginx_tutor_conf): - os.remove(nginx_tutor_conf) + """ + Add here ad-hoc commands to upgrade the environment. + """ def save_plugin_templates( diff --git a/tutor/templates/apps/caddy/Caddyfile b/tutor/templates/apps/caddy/Caddyfile index 0f33948..b8c22ae 100644 --- a/tutor/templates/apps/caddy/Caddyfile +++ b/tutor/templates/apps/caddy/Caddyfile @@ -1,13 +1,69 @@ -{{ LMS_HOST }}{% if not ENABLE_HTTPS %}:80{% endif %} { - reverse_proxy nginx:80 { +# Global configuration +{ + {{ patch("caddyfile-global")|indent(4) }} +} + +# proxy directive snippet (with logging) to be used as follows: +# +# import proxy "containername:port" +(proxy) { + log { + output stdout + format filter { + wrap json + fields { + common_log delete + request>headers delete + resp_headers delete + tls delete + } + } + } + + reverse_proxy {args.0} { header_up X-Forwarded-Port {{ 443 if ENABLE_HTTPS else 80 }} } } -{{ PREVIEW_LMS_HOST }}{% if not ENABLE_HTTPS %}:80{% endif %} { - reverse_proxy nginx:80 + +{% if ENABLE_HTTPS and ENABLE_WEB_PROXY %} +{% set port = "" %} +{# listening to https is disabled and we must only listen to http #} +{% else %} +{% set port = ":80" %} +{% endif %} + +{{ LMS_HOST }}{{ port }}, {{ PREVIEW_LMS_HOST }}{{ port }} { + @favicon_matcher { + path_regexp ^(.*)/favicon.ico$ + } + rewrite @favicon_matcher /static/images/favicon.ico + + # Limit profile image upload size + request_body /api/profile_images/*/*/upload { + max_size 1MB + } + request_body { + max_size 4MB + } + + import proxy "lms:8000" + + {{ patch("caddyfile-lms")|indent(4) }} } -{{ CMS_HOST }}{% if not ENABLE_HTTPS %}:80{% endif %} { - reverse_proxy nginx:80 + +{{ CMS_HOST }}{{ port }} { + @favicon_matcher { + path_regexp ^(.*)/favicon.ico$ + } + rewrite @favicon_matcher /static/images/favicon.ico + + request_body { + max_size 250MB + } + + import proxy "cms:8000" + + {{ patch("caddyfile-cms")|indent(4) }} } {{ patch("caddyfile") }} diff --git a/tutor/templates/apps/nginx/_tutor.conf b/tutor/templates/apps/nginx/_tutor.conf deleted file mode 100644 index 2822590..0000000 --- a/tutor/templates/apps/nginx/_tutor.conf +++ /dev/null @@ -1,10 +0,0 @@ -# Allow long domain names -server_names_hash_bucket_size 128; - -# Set a short ttl for proxies to allow restarts -resolver 127.0.0.11 [::1]:5353 valid=10s; - -# Configure logging to include scheme and server name -log_format tutor '$remote_addr - $remote_user [$time_local] $scheme://$host "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; \ No newline at end of file diff --git a/tutor/templates/apps/nginx/cms.conf b/tutor/templates/apps/nginx/cms.conf deleted file mode 100644 index cbe909a..0000000 --- a/tutor/templates/apps/nginx/cms.conf +++ /dev/null @@ -1,28 +0,0 @@ -{% if RUN_CMS %} -upstream cms-backend { - server cms:8000 fail_timeout=0; -} - -server { - listen 80; - server_name {{ CMS_HOST }}; - - access_log /var/log/nginx/access.log tutor; - client_max_body_size 250M; - server_tokens off; - - rewrite ^(.*)/favicon.ico$ /static/images/favicon.ico last; - - location @proxy_to_cms_app { - proxy_redirect off; - proxy_set_header Host $http_host; - proxy_pass http://cms-backend; - } - - location / { - try_files $uri @proxy_to_cms_app; - } - - {{ patch("nginx-cms")|indent(2) }} -} -{% endif %} diff --git a/tutor/templates/apps/nginx/extra.conf b/tutor/templates/apps/nginx/extra.conf deleted file mode 100644 index 73cf41b..0000000 --- a/tutor/templates/apps/nginx/extra.conf +++ /dev/null @@ -1 +0,0 @@ -{{ patch("nginx-extra") }} diff --git a/tutor/templates/apps/nginx/lms.conf b/tutor/templates/apps/nginx/lms.conf deleted file mode 100644 index 6b65d89..0000000 --- a/tutor/templates/apps/nginx/lms.conf +++ /dev/null @@ -1,47 +0,0 @@ -{% if RUN_LMS %} -upstream lms-backend { - server lms:8000 fail_timeout=0; -} - -server { - listen 80; - server_name {{ LMS_HOST }} {{ PREVIEW_LMS_HOST }}; - - access_log /var/log/nginx/access.log tutor; - client_max_body_size 4M; - server_tokens off; - - rewrite ^(.*)/favicon.ico$ /static/images/favicon.ico last; - - # Allow large cookies - proxy_buffer_size 8k; - - location @proxy_to_lms_app { - proxy_redirect off; - proxy_set_header Host $http_host; - proxy_pass http://lms-backend; - } - - location / { - try_files $uri @proxy_to_lms_app; - } - - # /login?next= can be used by 3rd party sites in tags to - # determine whether a user on their site is logged into edX. - # The most common image to use is favicon.ico. - location /login { - if ( $arg_next ~* "favicon.ico" ) { - return 403; - } - try_files $uri @proxy_to_lms_app; - } - - # Need a separate location for the image uploads endpoint to limit upload sizes - location ~ ^/api/profile_images/[^/]*/[^/]*/upload$ { - try_files $uri @proxy_to_lms_app; - client_max_body_size 1049576; - } - - {{ patch("nginx-lms")|indent(2) }} -} -{% endif %} diff --git a/tutor/templates/config.yml b/tutor/templates/config.yml index d8798ad..4055d3f 100644 --- a/tutor/templates/config.yml +++ b/tutor/templates/config.yml @@ -9,16 +9,15 @@ ID: "{{ 24|random_string }}" LMS_HOST: "www.myopenedx.com" # The following are default values -RUN_CADDY: true RUN_LMS: true RUN_CMS: true RUN_FORUM: true RUN_ELASTICSEARCH: true -ENABLE_HTTPS: false RUN_MONGODB: true RUN_MYSQL: true RUN_REDIS: true RUN_SMTP: true +CADDY_HTTP_PORT: 80 CMS_HOST: "studio.{{ LMS_HOST }}" PREVIEW_LMS_HOST: "preview.{{ LMS_HOST }}" CONTACT_EMAIL: "contact@{{ LMS_HOST }}" @@ -29,6 +28,7 @@ DOCKER_REGISTRY: "docker.io/" DOCKER_IMAGE_OPENEDX: "{{ DOCKER_REGISTRY }}overhangio/openedx:{{ TUTOR_VERSION }}" DOCKER_IMAGE_OPENEDX_DEV: "{{ DOCKER_REGISTRY }}overhangio/openedx-dev:{{ TUTOR_VERSION }}" DOCKER_IMAGE_CADDY: "{{ DOCKER_REGISTRY }}caddy:2.3.0" +DOCKER_IMAGE_ELASTICSEARCH: "{{ DOCKER_REGISTRY }}elasticsearch:7.10.1" DOCKER_IMAGE_FORUM: "{{ DOCKER_REGISTRY }}overhangio/openedx-forum:{{ TUTOR_VERSION }}" DOCKER_IMAGE_MONGODB: "{{ DOCKER_REGISTRY }}mongo:4.2.17" DOCKER_IMAGE_MYSQL: "{{ DOCKER_REGISTRY }}mysql:5.7.35" @@ -41,6 +41,8 @@ ELASTICSEARCH_HOST: "elasticsearch" ELASTICSEARCH_PORT: 9200 ELASTICSEARCH_SCHEME: "http" ELASTICSEARCH_HEAP_SIZE: 1g +ENABLE_HTTPS: false +ENABLE_WEB_PROXY: true FORUM_HOST: "forum" FORUM_MONGODB_DATABASE: "cs_comments_service" JWT_COMMON_AUDIENCE: "openedx" @@ -65,7 +67,6 @@ OPENEDX_COMMON_VERSION: "master" MYSQL_HOST: "mysql" MYSQL_PORT: 3306 MYSQL_ROOT_USERNAME: "root" -NGINX_HTTP_PORT: 80 PLATFORM_NAME: "My Open edX" PLUGINS: [] PREVIEW_LMS_HOST: "preview.{{ LMS_HOST }}" diff --git a/tutor/templates/k8s/deployments.yml b/tutor/templates/k8s/deployments.yml index c0d479b..9541ebc 100644 --- a/tutor/templates/k8s/deployments.yml +++ b/tutor/templates/k8s/deployments.yml @@ -1,4 +1,4 @@ -{% if RUN_CADDY %} +{% if ENABLE_WEB_PROXY %} --- apiVersion: apps/v1 kind: Deployment @@ -379,36 +379,6 @@ spec: ports: - containerPort: 25 {% endif %} ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: nginx - labels: - app.kubernetes.io/name: nginx -spec: - selector: - matchLabels: - app.kubernetes.io/name: nginx - template: - metadata: - labels: - app.kubernetes.io/name: nginx - spec: - containers: - - name: nginx - image: {{ DOCKER_IMAGE_NGINX }} - volumeMounts: - - mountPath: /etc/nginx/conf.d/ - name: config - {{ patch("k8s-deployments-nginx-volume-mounts")|indent(12) }} - ports: - - containerPort: 80 - volumes: - - name: config - configMap: - name: nginx-config - {{ patch("k8s-deployments-nginx-volumes")|indent(8) }} {% if RUN_REDIS %} --- apiVersion: apps/v1 diff --git a/tutor/templates/k8s/services.yml b/tutor/templates/k8s/services.yml index f50da21..2abcd71 100644 --- a/tutor/templates/k8s/services.yml +++ b/tutor/templates/k8s/services.yml @@ -1,4 +1,4 @@ -{% if RUN_CADDY %} +{% if ENABLE_WEB_PROXY %} --- apiVersion: v1 kind: Service @@ -98,18 +98,6 @@ spec: selector: app.kubernetes.io/name: mysql {% endif %} ---- -apiVersion: v1 -kind: Service -metadata: - name: nginx -spec: - type: NodePort - ports: - - port: 80 - name: http - selector: - app.kubernetes.io/name: nginx {% if RUN_REDIS %} --- apiVersion: v1 diff --git a/tutor/templates/k8s/volumes.yml b/tutor/templates/k8s/volumes.yml index 20d1dcf..ffb4b66 100644 --- a/tutor/templates/k8s/volumes.yml +++ b/tutor/templates/k8s/volumes.yml @@ -1,4 +1,4 @@ -{% if RUN_CADDY %} +{% if ENABLE_WEB_PROXY %} --- apiVersion: v1 kind: PersistentVolumeClaim diff --git a/tutor/templates/kustomization.yml b/tutor/templates/kustomization.yml index a337c7f..e34666c 100644 --- a/tutor/templates/kustomization.yml +++ b/tutor/templates/kustomization.yml @@ -34,9 +34,6 @@ configMapGenerator: - name: openedx-config files:{% for file in "apps/openedx/config"|walk_templates %} - {{ file }}{% endfor %} -- name: nginx-config - files:{% for file in "apps/nginx"|walk_templates %} - - {{ file }}{% endfor %} - name: redis-config files: - apps/redis/redis.conf diff --git a/tutor/templates/local/docker-compose.prod.yml b/tutor/templates/local/docker-compose.prod.yml index c184419..5f670e1 100644 --- a/tutor/templates/local/docker-compose.prod.yml +++ b/tutor/templates/local/docker-compose.prod.yml @@ -1,36 +1,23 @@ version: "3.7" services: - {% if RUN_CADDY %} - # Web proxy for SSL termination + # Web proxy for load balancing and SSL termination caddy: image: {{ DOCKER_IMAGE_CADDY }} restart: unless-stopped ports: - - "80:80" - {% if ENABLE_HTTPS %}- "443:443"{% endif %} + - "{{ CADDY_HTTP_PORT }}:80" + {% if ENABLE_HTTPS and ENABLE_WEB_PROXY %}- "443:443"{% endif %} volumes: - ../apps/caddy/Caddyfile:/etc/caddy/Caddyfile:ro - {% if ENABLE_HTTPS %}- ../../data/caddy:/data{% endif %} - {% endif %} - - # Web server - nginx: - image: {{ DOCKER_IMAGE_NGINX }} - restart: unless-stopped - {% if not RUN_CADDY %} - ports: - - "{{ NGINX_HTTP_PORT }}:80" - {% endif %} - {% if RUN_CADDY and not ENABLE_HTTPS %} + {% if ENABLE_HTTPS and ENABLE_WEB_PROXY %}- ../../data/caddy:/data{% endif %} + {% if not ENABLE_HTTPS %} networks: default: - # These aliases are for internal communication between containers when running locally with *.local.overhang.io hostnames. + # These aliases are for internal communication between containers when running locally + # with *.local.overhang.io hostnames. aliases: - "{{ LMS_HOST }}" - {{ patch("local-docker-compose-nginx-aliases")|indent(10) }} + {{ patch("local-docker-compose-caddy-aliases")|indent(10) }} {% endif %} - volumes: - - ../apps/nginx:/etc/nginx/conf.d/:ro - depends_on: {{ [("lms", RUN_LMS), ("cms", RUN_CMS)]|list_if }} - {{ patch("local-docker-compose-prod-services")|indent(2) }} \ No newline at end of file + {{ patch("local-docker-compose-prod-services")|indent(2) }} From f9402f7879b720bda2245c6aa81f381339fe5ee5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Thu, 23 Sep 2021 12:04:19 +0200 Subject: [PATCH 06/56] feat: run all services as unprivileged containers With this change, containers are no longer run as "root" but as unprivileged users. This is necessary in some environments, notably some Kubernetes clusters. To make this possible, we need to manually fix bind-mounted volumes in docker-compose. This is pretty much equivalent to the behaviour in Kubernetes, where permissions are fixed at runtime if the volume owner is incorrect. Thus, we have a consistent behaviour between docker-compose and Kubernetes. We achieve this by bind-mounting some repos inside "*-permissions" services. These services run as root user on docker-compose and will fix the required permissions, as per build/permissions/setowner.sh These services simply do not run on Kubernetes, where we don't rely on bind-mounted volumes. There, we make use of Kubernete's built-in volume ownership feature. With this change, we get rid of the "openedx-dev" Docker image, in the sense that it no longer has its own Dockerfile. Instead, the dev image is now simply a different target in the multi-layer openedx Docker image. This makes it much faster to build the openedx-dev image. Because we declare the APP_USER_ID in the dev/docker-compose.yml file, we need to pass the user ID from the host there. The only way to achieve that is with a tutor config variable. The downside of this approach is that the dev/docker-compose.yml file is no longer portable from one machine to the next. We consider that this is not such a big issue, as it affects the development environment only. We take this opportunity to replace the base image of the "forum" image. There is now no need to re-install ruby inside the image. The total image size is only decreased by 10%, but re-building the image is faster. In order to run the smtp service as non-root, we switch from namshi/smtp to devture/exim-relay. This change should be backward-compatible. Note that the nginx container remains privileged. We could switch to nginxinc/nginx-unprivileged, but it's probably not worth the effort, as we are considering to get rid of the nginx container altogether. Close #323. --- CHANGELOG-nightly.md | 5 +- docs/configuration.rst | 2 +- docs/dev.rst | 2 +- docs/tutorials/theming.rst | 4 -- tutor/bindmounts.py | 1 + tutor/commands/images.py | 9 +-- tutor/env.py | 1 + tutor/jobs.py | 1 + tutor/templates/build/forum/Dockerfile | 37 +++++----- tutor/templates/build/openedx-dev/Dockerfile | 34 ---------- .../build/openedx-dev/bin/create-user.sh | 11 --- .../openedx-dev/bin/docker-entrypoint.sh | 19 ------ tutor/templates/build/openedx/Dockerfile | 68 ++++++++++++++----- tutor/templates/build/permissions/Dockerfile | 7 ++ tutor/templates/build/permissions/setowner.sh | 14 ++++ tutor/templates/config.yml | 7 +- tutor/templates/dev/docker-compose.yml | 11 +++ tutor/templates/k8s/deployments.yml | 63 +++++++++++++++-- tutor/templates/k8s/services.yml | 4 +- tutor/templates/local/docker-compose.yml | 60 +++++++++++++++- 20 files changed, 230 insertions(+), 130 deletions(-) delete mode 100644 tutor/templates/build/openedx-dev/Dockerfile delete mode 100755 tutor/templates/build/openedx-dev/bin/create-user.sh delete mode 100644 tutor/templates/build/openedx-dev/bin/docker-entrypoint.sh create mode 100644 tutor/templates/build/permissions/Dockerfile create mode 100644 tutor/templates/build/permissions/setowner.sh diff --git a/CHANGELOG-nightly.md b/CHANGELOG-nightly.md index 4844fda..8e6ffe1 100644 --- a/CHANGELOG-nightly.md +++ b/CHANGELOG-nightly.md @@ -2,7 +2,10 @@ Note: Breaking changes between versions are indicated by "💥". +- 💥[Improvement] Run all services as unprivileged containers, for better security. This has multiple consequences: + - The "openedx-dev" image is now built with `tutor dev dc build lms`. + - The "smtp" service now runs the "devture/exim-relay" Docker image, which is unprivileged. Also, the default SMTP port is now 8025. - 💥[Feature] Get rid of the nginx container and service, which is now replaced by Caddy. this has the following consequences: - Patches "nginx-cms", "nginx-lms", "nginx-extra", "local-docker-compose-nginx-aliases" are replaced by "caddyfile-cms", "caddyfile-lms", "caddyfile", " local-docker-compose-caddy-aliases". - Patches "k8s-deployments-nginx-volume-mounts", "k8s-deployments-nginx-volumes" were obsolete and are removed. - - The `NGINX_HTTP_PORT` setting is renamed to `CADDY_HTTP_PORT`. \ No newline at end of file + - The `NGINX_HTTP_PORT` setting is renamed to `CADDY_HTTP_PORT`. diff --git a/docs/configuration.rst b/docs/configuration.rst index f5e0a89..16aafee 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -328,7 +328,7 @@ And djangojs.po:: Then you will have to re-build the openedx Docker image:: - tutor images build openedx openedx-dev + tutor images build openedx Beware that this will take a long time! Unfortunately it's difficult to accelerate this process, as translation files need to be compiled prior to collecting the assets. In development it's possible to accelerate the iteration loop -- but that exercise is left to the reader. diff --git a/docs/dev.rst b/docs/dev.rst index 526b413..9eed8c2 100644 --- a/docs/dev.rst +++ b/docs/dev.rst @@ -19,7 +19,7 @@ Once the local platform has been configured, you should stop it so that it does Finally, you should build the ``openedx-dev`` docker image:: - tutor images build openedx-dev + tutor dev dc build lms This ``openedx-dev`` development image differs from the ``openedx`` production image: diff --git a/docs/tutorials/theming.rst b/docs/tutorials/theming.rst index 4f6cfb2..bc514b9 100644 --- a/docs/tutorials/theming.rst +++ b/docs/tutorials/theming.rst @@ -50,10 +50,6 @@ The LMS can then be accessed at http://local.overhang.io:8000. You will then hav tutor dev settheme mythemename -Re-build development docker image (and compile assets):: - - tutor images build openedx-dev - Watch the themes folders for changes (in a different terminal):: tutor dev run watchthemes diff --git a/tutor/bindmounts.py b/tutor/bindmounts.py index 2324d61..ea64db9 100644 --- a/tutor/bindmounts.py +++ b/tutor/bindmounts.py @@ -39,6 +39,7 @@ chown -R {user_id} {volumes_path}/{volume_name}""".format( "run", "--rm", "--no-deps", + "--user=0", "--volume", "{}:{}".format(volumes_root_path, container_volumes_root_path), service, diff --git a/tutor/commands/images.py b/tutor/commands/images.py index 816cc6c..5982f4c 100644 --- a/tutor/commands/images.py +++ b/tutor/commands/images.py @@ -8,11 +8,9 @@ from .. import exceptions from .. import images from .. import plugins from ..types import Config -from .. import utils from .context import Context -BASE_IMAGE_NAMES = ["openedx", "forum"] -DEV_IMAGE_NAMES = ["openedx-dev"] +BASE_IMAGE_NAMES = ["openedx", "forum", "permissions"] VENDOR_IMAGES = [ "caddy", "elasticsearch", @@ -127,11 +125,6 @@ def build_image(root: str, config: Config, image: str, *args: str) -> None: tutor_env.pathjoin(root, "plugins", plugin, "build", img), tag, *args ) - # Build dev images with user id argument - dev_build_arg = ["--build-arg", "USERID={}".format(utils.get_user_id())] - for img, tag in iter_images(config, image, DEV_IMAGE_NAMES): - images.build(tutor_env.pathjoin(root, "build", img), tag, *dev_build_arg, *args) - def pull_image(config: Config, image: str) -> None: for _img, tag in iter_images(config, image, all_image_names(config)): diff --git a/tutor/env.py b/tutor/env.py index f591db3..7639415 100644 --- a/tutor/env.py +++ b/tutor/env.py @@ -59,6 +59,7 @@ class Renderer: environment.globals["rsa_import_key"] = utils.rsa_import_key environment.filters["rsa_private_key"] = utils.rsa_private_key environment.filters["walk_templates"] = self.walk_templates + environment.globals["HOST_USER_ID"] = utils.get_user_id() environment.globals["TUTOR_APP"] = __app__.replace("-", "_") environment.globals["TUTOR_VERSION"] = __version__ self.environment = environment diff --git a/tutor/jobs.py b/tutor/jobs.py index f9b0adf..0c16ab2 100644 --- a/tutor/jobs.py +++ b/tutor/jobs.py @@ -36,6 +36,7 @@ class BaseJobRunner: def initialise(runner: BaseJobRunner, limit_to: Optional[str] = None) -> None: fmt.echo_info("Initialising all services...") if limit_to is None or limit_to == "mysql": + fmt.echo_info("Initialising mysql...") runner.run_job_from_template("mysql", "hooks", "mysql", "init") for plugin_name, hook in runner.iter_plugin_hooks("pre-init"): if limit_to is None or limit_to == plugin_name: diff --git a/tutor/templates/build/forum/Dockerfile b/tutor/templates/build/forum/Dockerfile index 0bbbbd2..cdb059e 100644 --- a/tutor/templates/build/forum/Dockerfile +++ b/tutor/templates/build/forum/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.io/ubuntu:20.04 +FROM docker.io/ruby:2.5.7-slim-stretch MAINTAINER Overhang.io ENV DEBIAN_FRONTEND=noninteractive @@ -12,32 +12,27 @@ RUN wget -O /tmp/dockerize.tar.gz https://github.com/jwilder/dockerize/releases/ && tar -C /usr/local/bin -xzvf /tmp/dockerize.tar.gz \ && rm /tmp/dockerize.tar.gz -RUN mkdir /openedx +# Create unprivileged "app" user +RUN useradd --home-dir /app --create-home --shell /bin/bash --uid 1000 app -# Install ruby-build for building specific version of ruby -# The ruby-build version should be periodically updated to reflect the latest release -ARG RUBY_BUILD_VERSION=v20200401 -RUN git clone https://github.com/rbenv/ruby-build.git --branch $RUBY_BUILD_VERSION /openedx/ruby-build -WORKDIR /openedx/ruby-build -RUN PREFIX=/usr/local ./install.sh +# Copy custom scripts +COPY ./bin /app/bin +RUN chmod a+x /app/bin/* +ENV PATH :${PATH} -# Install ruby and some specific dependencies -ARG RUBY_VERSION=2.5.7 -ARG BUNDLER_VERSION=1.17.3 -ARG RAKE_VERSION=13.0.1 -RUN ruby-build $RUBY_VERSION /openedx/ruby -ENV PATH "/openedx/ruby/bin:$PATH" -RUN gem install bundler -v $BUNDLER_VERSION -RUN gem install rake -v $RAKE_VERSION +# From then on, run as unprivileged app user +USER app + +# Install rake and bundler +ENV PATH "/app/bin:/app/.gem/ruby/2.5.0/bin:$PATH" +RUN gem install --user-install bundler --version 1.17.3 +RUN gem install --user-install rake --version 13.0.1 # Install forum -RUN git clone https://github.com/edx/cs_comments_service.git --branch {{ OPENEDX_COMMON_VERSION }} --depth 1 /openedx/cs_comments_service -WORKDIR /openedx/cs_comments_service +RUN git clone https://github.com/edx/cs_comments_service.git --branch {{ OPENEDX_COMMON_VERSION }} --depth 1 /app/cs_comments_service +WORKDIR /app/cs_comments_service RUN bundle install --deployment -COPY ./bin /openedx/bin -RUN chmod a+x /openedx/bin/* -ENV PATH /openedx/bin:${PATH} ENTRYPOINT ["docker-entrypoint.sh"] ENV SINATRA_ENV staging diff --git a/tutor/templates/build/openedx-dev/Dockerfile b/tutor/templates/build/openedx-dev/Dockerfile deleted file mode 100644 index 8e55797..0000000 --- a/tutor/templates/build/openedx-dev/Dockerfile +++ /dev/null @@ -1,34 +0,0 @@ -FROM {{ DOCKER_IMAGE_OPENEDX }} as base -MAINTAINER Overhang.io - -# Install useful system requirements -RUN apt update && \ - apt install -y vim iputils-ping dnsutils telnet \ - && rm -rf /var/lib/apt/lists/* - -# Install dev python requirements -RUN pip install -r requirements/edx/development.txt -RUN pip install ipdb==0.13.4 ipython==7.27.0 - -{{ patch("openedx-dev-dockerfile-post-python-requirements") }} - -# Recompile static assets: in development mode all static assets are stored in edx-platform, -# and the location of these files is stored in webpack-stats.json. If we don't recompile -# static assets, then production assets will be served instead. -RUN rm -r /openedx/staticfiles && \ - mkdir /openedx/staticfiles && \ - openedx-assets webpack --env=dev - -# Copy new entrypoint (to take care of permission issues at runtime) -COPY ./bin /openedx/bin -RUN chmod a+x /openedx/bin/* - -# Configure new user -ARG USERID=1000 -RUN create-user.sh $USERID - -######## Development image -FROM base as dev - -# Default django settings -ENV SETTINGS tutor.development diff --git a/tutor/templates/build/openedx-dev/bin/create-user.sh b/tutor/templates/build/openedx-dev/bin/create-user.sh deleted file mode 100755 index 4b91302..0000000 --- a/tutor/templates/build/openedx-dev/bin/create-user.sh +++ /dev/null @@ -1,11 +0,0 @@ -#! /bin/sh -e -USERID=$1 - -if [ "$USERID" != "" ] && [ "$USERID" != "0" ] -then - echo "Creating 'openedx' user with id $USERID" - useradd --home-dir /openedx --uid $USERID openedx - chown -R openedx:openedx /openedx -else - echo "Running as root" -fi \ No newline at end of file diff --git a/tutor/templates/build/openedx-dev/bin/docker-entrypoint.sh b/tutor/templates/build/openedx-dev/bin/docker-entrypoint.sh deleted file mode 100644 index 7c10997..0000000 --- a/tutor/templates/build/openedx-dev/bin/docker-entrypoint.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/sh -e -export DJANGO_SETTINGS_MODULE=$SERVICE_VARIANT.envs.$SETTINGS - -if id -u openedx > /dev/null 2>&1; then - # Change owners of mounted volumes - echo "Setting file permissions for user openedx..." - find /openedx \ - -not -path "/openedx/edx-platform/*" \ - -not -user openedx \ - -writable \ - -exec chown openedx:openedx {} \+ - echo "File permissions set." - - # Run CMD as user openedx - exec chroot --userspec="openedx:openedx" --skip-chdir / env HOME=/openedx "$@" -else - echo "Running openedx-dev as root user" - exec "$@" -fi diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index b314e7c..fbecf89 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -117,14 +117,19 @@ RUN apt update && \ apt install -y gettext gfortran graphviz graphviz-dev libffi-dev libfreetype6-dev libgeos-dev libjpeg8-dev liblapack-dev libmysqlclient-dev libpng-dev libsqlite3-dev libxmlsec1-dev lynx ntp pkg-config rdfind && \ rm -rf /var/lib/apt/lists/* -COPY --from=dockerize /usr/local/bin/dockerize /usr/local/bin/dockerize -COPY --from=code /openedx/edx-platform /openedx/edx-platform -COPY --from=locales /openedx/locale/contrib/locale /openedx/locale/contrib/locale -COPY --from=python /opt/pyenv /opt/pyenv -COPY --from=python-requirements /openedx/venv /openedx/venv -COPY --from=python-requirements /openedx/requirements /openedx/requirements -COPY --from=nodejs-requirements /openedx/nodeenv /openedx/nodeenv -COPY --from=nodejs-requirements /openedx/edx-platform/node_modules /openedx/edx-platform/node_modules +# From then on, run as unprivileged "app" user +ARG APP_USER_ID=1000 +RUN useradd --home-dir /openedx --create-home --shell /bin/bash --uid ${APP_USER_ID} app +USER ${APP_USER_ID} + +COPY --chown=app:app --from=dockerize /usr/local/bin/dockerize /usr/local/bin/dockerize +COPY --chown=app:app --from=code /openedx/edx-platform /openedx/edx-platform +COPY --chown=app:app --from=locales /openedx/locale /openedx/locale +COPY --chown=app:app --from=python /opt/pyenv /opt/pyenv +COPY --chown=app:app --from=python-requirements /openedx/venv /openedx/venv +COPY --chown=app:app --from=python-requirements /openedx/requirements /openedx/requirements +COPY --chown=app:app --from=nodejs-requirements /openedx/nodeenv /openedx/nodeenv +COPY --chown=app:app --from=nodejs-requirements /openedx/edx-platform/node_modules /openedx/edx-platform/node_modules ENV PATH /openedx/venv/bin:./node_modules/.bin:/openedx/nodeenv/bin:${PATH} ENV VIRTUAL_ENV /openedx/venv/ @@ -136,16 +141,16 @@ RUN pip install -r requirements/edx/local.in # Create folder that will store lms/cms.env.json files, as well as # the tutor-specific settings files. RUN mkdir -p /openedx/config ./lms/envs/tutor ./cms/envs/tutor -COPY revisions.yml /openedx/config/ +COPY --chown=app:app revisions.yml /openedx/config/ ENV LMS_CFG /openedx/config/lms.env.json ENV STUDIO_CFG /openedx/config/cms.env.json ENV REVISION_CFG /openedx/config/revisions.yml -COPY settings/lms/*.py ./lms/envs/tutor/ -COPY settings/cms/*.py ./cms/envs/tutor/ +COPY --chown=app:app settings/lms/*.py ./lms/envs/tutor/ +COPY --chown=app:app settings/cms/*.py ./cms/envs/tutor/ # Copy user-specific locales to /openedx/locale/user/locale and compile them -RUN mkdir -p /openedx/locale/user -COPY ./locale/ /openedx/locale/user/locale/ +RUN mkdir /openedx/locale/user +COPY --chown=app:app ./locale/ /openedx/locale/user/locale/ RUN cd /openedx/locale/user && \ django-admin.py compilemessages -v1 @@ -156,7 +161,7 @@ RUN ./manage.py lms --settings=tutor.i18n compilejsi18n RUN ./manage.py cms --settings=tutor.i18n compilejsi18n # Copy scripts -COPY ./bin /openedx/bin +COPY --chown=app:app ./bin /openedx/bin RUN chmod a+x /openedx/bin/* ENV PATH /openedx/bin:${PATH} @@ -178,7 +183,7 @@ RUN openedx-assets xmodule \ && openedx-assets npm \ && openedx-assets webpack --env=prod \ && openedx-assets common -COPY ./themes/ /openedx/themes/ +COPY --chown=app:app ./themes/ /openedx/themes/ RUN openedx-assets themes \ && openedx-assets collect --settings=tutor.assets \ # De-duplicate static assets with symlinks @@ -195,9 +200,40 @@ ENV SETTINGS tutor.production # Entrypoint will set right environment variables ENTRYPOINT ["docker-entrypoint.sh"] +EXPOSE 8000 + +###### Intermediate image with dev/test dependencies +FROM production as development + +# Install useful system requirements (as root) +USER root +RUN apt update && \ + apt install -y vim iputils-ping dnsutils telnet \ + && rm -rf /var/lib/apt/lists/* +USER app + +# Install dev python requirements +RUN pip install -r requirements/edx/development.txt +RUN pip install ipdb==0.13.4 ipython==7.27.0 + +# Recompile static assets: in development mode all static assets are stored in edx-platform, +# and the location of these files is stored in webpack-stats.json. If we don't recompile +# static assets, then production assets will be served instead. +RUN rm -r /openedx/staticfiles && \ + mkdir /openedx/staticfiles && \ + openedx-assets webpack --env=dev + +{{ patch("openedx-dev-dockerfile-post-python-requirements") }} + +# Default django settings +ENV SETTINGS tutor.development + +CMD ./manage.py $SERVICE_VARIANT runserver 0.0.0.0:8000 + +###### Final image with production cmd +FROM production as final # Run server -EXPOSE 8000 CMD uwsgi \ --static-map /static=/openedx/staticfiles/ \ --static-map /media=/openedx/media/ \ diff --git a/tutor/templates/build/permissions/Dockerfile b/tutor/templates/build/permissions/Dockerfile new file mode 100644 index 0000000..1ddcc5f --- /dev/null +++ b/tutor/templates/build/permissions/Dockerfile @@ -0,0 +1,7 @@ +from docker.io/alpine:3.13.6 +MAINTAINER Overhang.io + +COPY ./setowner.sh /usr/local/bin/setowner +RUN chmod a+x /usr/local/bin/setowner + +ENTRYPOINT ["setowner"] diff --git a/tutor/templates/build/permissions/setowner.sh b/tutor/templates/build/permissions/setowner.sh new file mode 100644 index 0000000..f0a3ea9 --- /dev/null +++ b/tutor/templates/build/permissions/setowner.sh @@ -0,0 +1,14 @@ +#! /bin/sh +set -e +user_id="$1" +shift +for path in $@; do + path_user_id="$(stat -c '%u' $path)" + if [ "$path_user_id" != "$user_id" ] + then + echo "$path changing UID from $path_user_id to $user_id..." + chown --recursive $user_id $path + else + echo "$path already owned by $user_id" + fi +done diff --git a/tutor/templates/config.yml b/tutor/templates/config.yml index 4055d3f..3a16b0e 100644 --- a/tutor/templates/config.yml +++ b/tutor/templates/config.yml @@ -26,7 +26,7 @@ OPENEDX_AWS_SECRET_ACCESS_KEY: "" DEV_PROJECT_NAME: "tutor_dev" DOCKER_REGISTRY: "docker.io/" DOCKER_IMAGE_OPENEDX: "{{ DOCKER_REGISTRY }}overhangio/openedx:{{ TUTOR_VERSION }}" -DOCKER_IMAGE_OPENEDX_DEV: "{{ DOCKER_REGISTRY }}overhangio/openedx-dev:{{ TUTOR_VERSION }}" +DOCKER_IMAGE_OPENEDX_DEV: "openedx-dev" DOCKER_IMAGE_CADDY: "{{ DOCKER_REGISTRY }}caddy:2.3.0" DOCKER_IMAGE_ELASTICSEARCH: "{{ DOCKER_REGISTRY }}elasticsearch:7.10.1" DOCKER_IMAGE_FORUM: "{{ DOCKER_REGISTRY }}overhangio/openedx-forum:{{ TUTOR_VERSION }}" @@ -34,8 +34,9 @@ DOCKER_IMAGE_MONGODB: "{{ DOCKER_REGISTRY }}mongo:4.2.17" DOCKER_IMAGE_MYSQL: "{{ DOCKER_REGISTRY }}mysql:5.7.35" DOCKER_IMAGE_ELASTICSEARCH: "{{ DOCKER_REGISTRY }}elasticsearch:7.10.1" DOCKER_IMAGE_NGINX: "{{ DOCKER_REGISTRY }}nginx:1.21.1" +DOCKER_IMAGE_PERMISSIONS: "{{ DOCKER_REGISTRY }}alpine:3.13.6" DOCKER_IMAGE_REDIS: "{{ DOCKER_REGISTRY }}redis:6.2.6" -DOCKER_IMAGE_SMTP: "{{ DOCKER_REGISTRY }}namshi/smtp:latest" +DOCKER_IMAGE_SMTP: "{{ DOCKER_REGISTRY }}devture/exim-relay:4.94.2-r0-4" LOCAL_PROJECT_NAME: "{{ TUTOR_APP }}_local" ELASTICSEARCH_HOST: "elasticsearch" ELASTICSEARCH_PORT: 9200 @@ -75,7 +76,7 @@ REDIS_PORT: 6379 REDIS_USERNAME: "" REDIS_PASSWORD: "" SMTP_HOST: "smtp" -SMTP_PORT: 25 +SMTP_PORT: 8025 SMTP_USERNAME: "" SMTP_PASSWORD: "" SMTP_USE_TLS: false diff --git a/tutor/templates/dev/docker-compose.yml b/tutor/templates/dev/docker-compose.yml index ee65403..095d474 100644 --- a/tutor/templates/dev/docker-compose.yml +++ b/tutor/templates/dev/docker-compose.yml @@ -3,6 +3,11 @@ version: "3.7" x-openedx-service: &openedx-service image: {{ DOCKER_IMAGE_OPENEDX_DEV }} + build: + context: ../build/openedx/ + target: development + args: + APP_USER_ID: "{{ HOST_USER_ID }}" environment: SETTINGS: ${TUTOR_EDX_PLATFORM_SETTINGS:-tutor.development} volumes: @@ -16,6 +21,12 @@ x-openedx-service: - ../build/openedx/requirements:/openedx/requirements services: + lms-permissions: + command: ["{{ HOST_USER_ID }}", "/openedx/data", "/openedx/media"] + + cms-permissions: + command: ["{{ HOST_USER_ID }}", "/openedx/data", "/openedx/media"] + lms: <<: *openedx-service command: ./manage.py lms runserver 0.0.0.0:8000 diff --git a/tutor/templates/k8s/deployments.yml b/tutor/templates/k8s/deployments.yml index 9541ebc..ddfa4f1 100644 --- a/tutor/templates/k8s/deployments.yml +++ b/tutor/templates/k8s/deployments.yml @@ -51,6 +51,9 @@ spec: labels: app.kubernetes.io/name: cms spec: + securityContext: + runAsUser: 1000 + runAsGroup: 1000 containers: - name: cms image: {{ DOCKER_IMAGE_OPENEDX }} @@ -69,6 +72,8 @@ spec: resources: requests: memory: 2Gi + securityContext: + allowPrivilegeEscalation: false volumes: - name: settings-lms configMap: @@ -95,6 +100,9 @@ spec: labels: app.kubernetes.io/name: cms-worker spec: + securityContext: + runAsUser: 1000 + runAsGroup: 1000 containers: - name: cms-worker image: {{ DOCKER_IMAGE_OPENEDX }} @@ -102,8 +110,6 @@ spec: env: - name: SERVICE_VARIANT value: cms - - name: C_FORCE_ROOT - value: "1" volumeMounts: - mountPath: /openedx/edx-platform/lms/envs/tutor/ name: settings-lms @@ -111,6 +117,8 @@ spec: name: settings-cms - mountPath: /openedx/config name: config + securityContext: + allowPrivilegeEscalation: false volumes: - name: settings-lms configMap: @@ -139,6 +147,9 @@ spec: labels: app.kubernetes.io/name: forum spec: + securityContext: + runAsUser: 1000 + runAsGroup: 1000 containers: - name: forum image: {{ DOCKER_IMAGE_FORUM }} @@ -155,6 +166,8 @@ spec: value: "{{ MONGODB_PORT }}" - name: MONGODB_DATABASE value: "{{ FORUM_MONGODB_DATABASE }}" + securityContext: + allowPrivilegeEscalation: false {% endif %} {% if RUN_LMS %} --- @@ -173,6 +186,9 @@ spec: labels: app.kubernetes.io/name: lms spec: + securityContext: + runAsUser: 1000 + runAsGroup: 1000 containers: - name: lms image: {{ DOCKER_IMAGE_OPENEDX }} @@ -188,6 +204,8 @@ spec: resources: requests: memory: 2Gi + securityContext: + allowPrivilegeEscalation: false volumes: - name: settings-lms configMap: @@ -214,6 +232,9 @@ spec: labels: app.kubernetes.io/name: lms-worker spec: + securityContext: + runAsUser: 1000 + runAsGroup: 1000 containers: - name: lms-worker image: {{ DOCKER_IMAGE_OPENEDX }} @@ -221,8 +242,6 @@ spec: env: - name: SERVICE_VARIANT value: lms - - name: C_FORCE_ROOT - value: "1" volumeMounts: - mountPath: /openedx/edx-platform/lms/envs/tutor/ name: settings-lms @@ -230,6 +249,8 @@ spec: name: settings-cms - mountPath: /openedx/config name: config + securityContext: + allowPrivilegeEscalation: false volumes: - name: settings-lms configMap: @@ -260,6 +281,11 @@ spec: labels: app.kubernetes.io/name: elasticsearch spec: + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + fsGroupChangePolicy: "OnRootMismatch" containers: - name: elasticsearch image: {{ DOCKER_IMAGE_ELASTICSEARCH }} @@ -276,6 +302,8 @@ spec: value: "1" ports: - containerPort: 9200 + securityContext: + allowPrivilegeEscalation: false volumeMounts: - mountPath: /usr/share/elasticsearch/data name: data @@ -303,6 +331,11 @@ spec: labels: app.kubernetes.io/name: mongodb spec: + securityContext: + runAsUser: 999 + runAsGroup: 999 + fsGroup: 999 + fsGroupChangePolicy: "OnRootMismatch" containers: - name: mongodb image: {{ DOCKER_IMAGE_MONGODB }} @@ -312,7 +345,8 @@ spec: volumeMounts: - mountPath: /data/db name: data - + securityContext: + allowPrivilegeEscalation: false volumes: - name: data persistentVolumeClaim: @@ -337,6 +371,11 @@ spec: labels: app.kubernetes.io/name: mysql spec: + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + fsGroupChangePolicy: "OnRootMismatch" containers: - name: mysql image: {{ DOCKER_IMAGE_MYSQL }} @@ -351,6 +390,8 @@ spec: volumeMounts: - mountPath: /var/lib/mysql name: data + securityContext: + allowPrivilegeEscalation: false volumes: - name: data persistentVolumeClaim: @@ -373,11 +414,14 @@ spec: labels: app.kubernetes.io/name: smtp spec: + securityContext: + runAsUser: 100 + runAsGroup: 101 containers: - name: smtp image: {{ DOCKER_IMAGE_SMTP }} ports: - - containerPort: 25 + - containerPort: 8025 {% endif %} {% if RUN_REDIS %} --- @@ -398,6 +442,11 @@ spec: labels: app.kubernetes.io/name: redis spec: + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + fsGroupChangePolicy: "OnRootMismatch" containers: - name: redis image: {{ DOCKER_IMAGE_REDIS }} @@ -410,6 +459,8 @@ spec: name: config - mountPath: /openedx/redis/data name: data + securityContext: + allowPrivilegeEscalation: false volumes: - name: config configMap: diff --git a/tutor/templates/k8s/services.yml b/tutor/templates/k8s/services.yml index 2abcd71..68e78ec 100644 --- a/tutor/templates/k8s/services.yml +++ b/tutor/templates/k8s/services.yml @@ -121,9 +121,9 @@ metadata: spec: type: NodePort ports: - - port: 25 + - port: 8025 protocol: TCP selector: app.kubernetes.io/name: smtp {% endif %} -{{ patch("k8s-services") }} \ No newline at end of file +{{ patch("k8s-services") }} diff --git a/tutor/templates/local/docker-compose.yml b/tutor/templates/local/docker-compose.yml index cef04c2..e79294a 100644 --- a/tutor/templates/local/docker-compose.yml +++ b/tutor/templates/local/docker-compose.yml @@ -9,6 +9,16 @@ services: # Use WiredTiger in all environments, just like at edx.org command: mongod --nojournal --storageEngine wiredTiger restart: unless-stopped + user: "999:999" + privileged: false + volumes: + - ../../data/mongodb:/data/db + depends_on: + - mongodb-permissions + mongodb-permissions: + image: {{ DOCKER_IMAGE_PERMISSIONS }} + command: ["999", "/data/db"] + restart: on-failure volumes: - ../../data/mongodb:/data/db {% endif %} @@ -18,10 +28,18 @@ services: image: {{ DOCKER_IMAGE_MYSQL }} command: mysqld --character-set-server=utf8 --collation-server=utf8_general_ci restart: unless-stopped + user: "1000:1000" + privileged: false volumes: - ../../data/mysql:/var/lib/mysql environment: MYSQL_ROOT_PASSWORD: "{{ MYSQL_ROOT_PASSWORD }}" + mysql-permissions: + image: {{ DOCKER_IMAGE_PERMISSIONS }} + command: ["1000", "/var/lib/mysql"] + restart: on-failure + volumes: + - ../../data/mysql:/var/lib/mysql {% endif %} {% if RUN_ELASTICSEARCH %} @@ -32,12 +50,20 @@ services: - bootstrap.memory_lock=true - discovery.type=single-node - "ES_JAVA_OPTS=-Xms{{ ELASTICSEARCH_HEAP_SIZE }} -Xmx{{ ELASTICSEARCH_HEAP_SIZE }}" - - TAKE_FILE_OWNERSHIP=1 ulimits: memlock: soft: -1 hard: -1 restart: unless-stopped + user: "1000:1000" + volumes: + - ../../data/elasticsearch:/usr/share/elasticsearch/data + depends_on: + - elasticsearch-permissions + elasticsearch-permissions: + image: {{ DOCKER_IMAGE_PERMISSIONS }} + command: ["1000", "/usr/share/elasticsearch/data"] + restart: on-failure volumes: - ../../data/elasticsearch:/usr/share/elasticsearch/data {% endif %} @@ -46,17 +72,29 @@ services: redis: image: {{ DOCKER_IMAGE_REDIS }} working_dir: /openedx/redis/data + user: "1000:1000" volumes: - ../apps/redis/redis.conf:/openedx/redis/config/redis.conf:ro - ../../data/redis:/openedx/redis/data command: redis-server /openedx/redis/config/redis.conf restart: unless-stopped + depends_on: + - redis-permissions + redis-permissions: + image: {{ DOCKER_IMAGE_PERMISSIONS }} + command: ["1000", "/openedx/redis/data"] + restart: on-failure + volumes: + - ../../data/redis:/openedx/redis/data {% endif %} {% if RUN_SMTP %} smtp: image: {{ DOCKER_IMAGE_SMTP }} restart: unless-stopped + user: "100:101" + environment: + HOSTNAME: "{{ LMS_HOST }}" {% endif %} ############# Forum @@ -91,6 +129,7 @@ services: - ../../data/lms:/openedx/data - ../../data/openedx-media:/openedx/media depends_on: + - lms-permissions {% if RUN_MYSQL %}- mysql{% endif %} {% if RUN_ELASTICSEARCH %}- elasticsearch{% endif %} {% if RUN_FORUM %}- forum{% endif %} @@ -98,6 +137,14 @@ services: {% if RUN_REDIS %}- redis{% endif %} {% if RUN_SMTP %}- smtp{% endif %} {{ patch("local-docker-compose-lms-dependencies")|indent(6) }} + lms-permissions: + image: {{ DOCKER_IMAGE_PERMISSIONS }} + command: ["1000", "/openedx/data", "/openedx/media"] + restart: on-failure + volumes: + - ../../data/redis:/openedx/redis/data + - ../../data/lms:/openedx/data + - ../../data/openedx-media:/openedx/media {% endif %} {% if RUN_CMS %} @@ -115,6 +162,7 @@ services: - ../../data/cms:/openedx/data - ../../data/openedx-media:/openedx/media depends_on: + - cms-permissions {% if RUN_MYSQL %}- mysql{% endif %} {% if RUN_ELASTICSEARCH %}- elasticsearch{% endif %} {% if RUN_MONGODB %}- mongodb{% endif %} @@ -122,6 +170,14 @@ services: {% if RUN_SMTP %}- smtp{% endif %} {% if RUN_LMS %}- lms{% endif %} {{ patch("local-docker-compose-cms-dependencies")|indent(6) }} + cms-permissions: + image: {{ DOCKER_IMAGE_PERMISSIONS }} + command: ["1000", "/openedx/data", "/openedx/media"] + restart: on-failure + volumes: + - ../../data/redis:/openedx/redis/data + - ../../data/cms:/openedx/data + - ../../data/openedx-media:/openedx/media {% endif %} ############# LMS and CMS workers @@ -132,7 +188,6 @@ services: environment: SERVICE_VARIANT: lms SETTINGS: ${TUTOR_EDX_PLATFORM_SETTINGS:-tutor.production} - C_FORCE_ROOT: "1" # run celery tasks as root #nofear command: celery worker --app=lms.celery --loglevel=info --hostname=edx.lms.core.default.%%h --maxtasksperchild=100 --exclude-queues=edx.cms.core.default restart: unless-stopped volumes: @@ -151,7 +206,6 @@ services: environment: SERVICE_VARIANT: cms SETTINGS: ${TUTOR_EDX_PLATFORM_SETTINGS:-tutor.production} - C_FORCE_ROOT: "1" # run celery tasks as root #nofear command: celery worker --app=cms.celery --loglevel=info --hostname=edx.cms.core.default.%%h --maxtasksperchild 100 --exclude-queues=edx.lms.core.default restart: unless-stopped volumes: From f6789150ee38eb7bdb4af684092d622a70db3e31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Mon, 25 Oct 2021 16:56:37 +0200 Subject: [PATCH 07/56] fix: permissions image name --- tutor/templates/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tutor/templates/config.yml b/tutor/templates/config.yml index 3a16b0e..eb28c0a 100644 --- a/tutor/templates/config.yml +++ b/tutor/templates/config.yml @@ -34,7 +34,7 @@ DOCKER_IMAGE_MONGODB: "{{ DOCKER_REGISTRY }}mongo:4.2.17" DOCKER_IMAGE_MYSQL: "{{ DOCKER_REGISTRY }}mysql:5.7.35" DOCKER_IMAGE_ELASTICSEARCH: "{{ DOCKER_REGISTRY }}elasticsearch:7.10.1" DOCKER_IMAGE_NGINX: "{{ DOCKER_REGISTRY }}nginx:1.21.1" -DOCKER_IMAGE_PERMISSIONS: "{{ DOCKER_REGISTRY }}alpine:3.13.6" +DOCKER_IMAGE_PERMISSIONS: "{{ DOCKER_REGISTRY }}overhangio/openedx-permissions:{{ TUTOR_VERSION }}" DOCKER_IMAGE_REDIS: "{{ DOCKER_REGISTRY }}redis:6.2.6" DOCKER_IMAGE_SMTP: "{{ DOCKER_REGISTRY }}devture/exim-relay:4.94.2-r0-4" LOCAL_PROJECT_NAME: "{{ TUTOR_APP }}_local" From 7a01f9d00936fe37cd81a2eaf2ed78db9202b266 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Mon, 1 Nov 2021 09:10:36 +0100 Subject: [PATCH 08/56] fix: always run Caddy on Kubernetes Caddy should always be running, even when ENABLE_WEB_PROXY is false. It's the service that should not always be running. --- CHANGELOG-nightly.md | 1 + tutor/templates/k8s/deployments.yml | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG-nightly.md b/CHANGELOG-nightly.md index 8e6ffe1..d6618d2 100644 --- a/CHANGELOG-nightly.md +++ b/CHANGELOG-nightly.md @@ -2,6 +2,7 @@ Note: Breaking changes between versions are indicated by "💥". +- [Bugfix] Fix running Caddy container in k8s, which should always be the case even if `ENABLE_WEB_PROXY` is false. - 💥[Improvement] Run all services as unprivileged containers, for better security. This has multiple consequences: - The "openedx-dev" image is now built with `tutor dev dc build lms`. - The "smtp" service now runs the "devture/exim-relay" Docker image, which is unprivileged. Also, the default SMTP port is now 8025. diff --git a/tutor/templates/k8s/deployments.yml b/tutor/templates/k8s/deployments.yml index ddfa4f1..77b58bc 100644 --- a/tutor/templates/k8s/deployments.yml +++ b/tutor/templates/k8s/deployments.yml @@ -1,4 +1,3 @@ -{% if ENABLE_WEB_PROXY %} --- apiVersion: apps/v1 kind: Deployment @@ -21,19 +20,24 @@ spec: volumeMounts: - mountPath: /etc/caddy/ name: config + {%- if ENABLE_WEB_PROXY %} - mountPath: /data/ name: data + {%- endif %} ports: - containerPort: 80 + {%- if ENABLE_WEB_PROXY %} - containerPort: 443 + {%- endif %} volumes: - name: config configMap: name: caddy-config + {%- if ENABLE_WEB_PROXY %} - name: data persistentVolumeClaim: claimName: caddy -{% endif %} + {%- endif %} {% if RUN_CMS %} --- apiVersion: apps/v1 From 999c23d1ff615884f0c1de8f423d2742ece89b42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Tue, 2 Nov 2021 18:15:06 +0100 Subject: [PATCH 09/56] chore: get rid of tutor-openedx In the past, tutor was installed with "pip install tutor-openedx". For some time (since v12.0.2), "tutor" was installed as a dependency of "tutor-openedx". Now is the time to get rid of that old package. The standard way of installing tutor is now with "pip install tutor". --- .gitignore | 3 -- CHANGELOG-nightly.md | 1 + Makefile | 10 +--- tutor-openedx/MANIFEST.in | 0 tutor-openedx/README.rst | 3 -- tutor-openedx/setup.py | 64 -------------------------- tutor-openedx/tutoropenedx/__init__.py | 0 7 files changed, 3 insertions(+), 78 deletions(-) delete mode 100644 tutor-openedx/MANIFEST.in delete mode 100644 tutor-openedx/README.rst delete mode 100644 tutor-openedx/setup.py delete mode 100644 tutor-openedx/tutoropenedx/__init__.py diff --git a/.gitignore b/.gitignore index 347fc4d..8e55cc4 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,3 @@ __pycache__ /build/ /dist/ /release_description.md - -# Copied at build-time -/tutor-openedx/tutoropenedx/__about__.py diff --git a/CHANGELOG-nightly.md b/CHANGELOG-nightly.md index d6618d2..7a7077d 100644 --- a/CHANGELOG-nightly.md +++ b/CHANGELOG-nightly.md @@ -2,6 +2,7 @@ Note: Breaking changes between versions are indicated by "💥". +- 💥[Improvement] Get rid of the "tutor-openedx" package, which is no longer supported. - [Bugfix] Fix running Caddy container in k8s, which should always be the case even if `ENABLE_WEB_PROXY` is false. - 💥[Improvement] Run all services as unprivileged containers, for better security. This has multiple consequences: - The "openedx-dev" image is now built with `tutor dev dc build lms`. diff --git a/Makefile b/Makefile index 9ddd9c3..7356469 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .DEFAULT_GOAL := help .PHONY: docs -SRC_DIRS = ./tutor ./tests ./bin ./tutor-openedx +SRC_DIRS = ./tutor ./tests ./bin BLACK_OPTS = --exclude templates ${SRC_DIRS} ###### Development @@ -18,18 +18,13 @@ upgrade-requirements: ## Upgrade requirements files pip-compile --upgrade requirements/dev.in pip-compile --upgrade requirements/docs.in -build-pythonpackage: build-pythonpackage-tutor build-pythonpackage-tutor-openedx ## Build python packages ready to upload to pypi +build-pythonpackage: build-pythonpackage-tutor ## Build python packages ready to upload to pypi build-pythonpackage-tutor: ## Build the "tutor" python package for upload to pypi python setup.py sdist -build-pythonpackage-tutor-openedx: ## Build the obsolete "tutor-openedx" python package for upload to pypi - cp tutor/__about__.py tutor-openedx/tutoropenedx/ - cd tutor-openedx && python setup.py sdist --dist-dir ../dist - push-pythonpackage: ## Push python package to pypi twine upload --skip-existing dist/tutor-$(shell make version).tar.gz - twine upload --skip-existing dist/tutor-openedx-$(shell make version).tar.gz test: test-lint test-unit test-types test-format test-pythonpackage ## Run all tests by decreasing order or priority @@ -47,7 +42,6 @@ test-types: ## Check type definitions test-pythonpackage: build-pythonpackage ## Test that package can be uploaded to pypi twine check dist/tutor-$(shell make version).tar.gz - twine check dist/tutor-openedx-$(shell make version).tar.gz format: ## Format code automatically black $(BLACK_OPTS) diff --git a/tutor-openedx/MANIFEST.in b/tutor-openedx/MANIFEST.in deleted file mode 100644 index e69de29..0000000 diff --git a/tutor-openedx/README.rst b/tutor-openedx/README.rst deleted file mode 100644 index 5d4014e..0000000 --- a/tutor-openedx/README.rst +++ /dev/null @@ -1,3 +0,0 @@ -WARNING: This project has moved to https://pypi.org/project/tutor/. You should now install Tutor with:: - - pip install tutor diff --git a/tutor-openedx/setup.py b/tutor-openedx/setup.py deleted file mode 100644 index c9cf051..0000000 --- a/tutor-openedx/setup.py +++ /dev/null @@ -1,64 +0,0 @@ -import io -import os -import sys -from setuptools import setup - -HERE = os.path.abspath(os.path.dirname(__file__)) - - -def load_readme(): - with io.open(os.path.join(HERE, "README.rst"), "rt", encoding="utf8") as f: - return f.read() - - -def load_about(): - about = {} - with io.open( - os.path.join(HERE, "tutoropenedx", "__about__.py"), "rt", encoding="utf-8" - ) as f: - exec(f.read(), about) # pylint: disable=exec-used - return about - - -ABOUT = load_about() - -setup( - name="tutor-openedx", - version=ABOUT["__version__"], - url="https://docs.tutor.overhang.io/", - project_urls={ - "Documentation": "https://docs.tutor.overhang.io/", - "Code": "https://github.com/overhangio/tutor", - "Issue tracker": "https://github.com/overhangio/tutor/issues", - "Community": "https://discuss.overhang.io", - }, - license="AGPLv3", - author="Overhang.io", - author_email="contact@overhang.io", - description="The Docker-based Open edX distribution designed for peace of mind", - long_description=load_readme(), - long_description_content_type="text/x-rst", - packages=["tutoropenedx"], - python_requires=">=3.5", - install_requires=["tutor=={}".format(ABOUT["__version__"])], - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "License :: OSI Approved :: GNU Affero General Public License v3", - "Operating System :: OS Independent", - "Programming Language :: Python", - "Programming Language :: Python :: 3.5", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - ], -) -sys.stderr.write( - """ -Installing Tutor from tutor-openedx is deprecated. You should instead install the "tutor" package with: - - pip install tutor -""" -) diff --git a/tutor-openedx/tutoropenedx/__init__.py b/tutor-openedx/tutoropenedx/__init__.py deleted file mode 100644 index e69de29..0000000 From 791bca156409074a5fa59926e229b2870daadcaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Tue, 2 Nov 2021 18:24:38 +0100 Subject: [PATCH 10/56] doc: remove now irrelevant comment --- tutor/commands/compose.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tutor/commands/compose.py b/tutor/commands/compose.py index a78274c..693987e 100644 --- a/tutor/commands/compose.py +++ b/tutor/commands/compose.py @@ -36,9 +36,7 @@ class ComposeJobRunner(jobs.BaseComposeJobRunner): def run_job(self, service: str, command: str) -> int: """ Run the "{{ service }}-job" service from local/docker-compose.jobs.yml with the - specified command. For backward-compatibility reasons, if the corresponding - service does not exist, run the service from good old regular - docker-compose.yml. + specified command. """ run_command = [] for docker_compose_path in self.docker_compose_job_files: From 7a3026efe627bf07352a5380c933fe19f282e01f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Mon, 25 Oct 2021 19:32:13 +0200 Subject: [PATCH 11/56] feat: move all forum-related code to a dedicated plugin Forum is an optional feature, and as such it deserves its own plugin. Starting from Maple, users will be able to install the forum from https://github.com/overhangio/tutor-forum/ Close #450. --- CHANGELOG-nightly.md | 1 + Makefile | 2 +- bin/main.py | 1 + docs/configuration.rst | 11 +---- docs/plugins/api.rst | 2 +- requirements/plugins.txt | 1 + tutor/commands/images.py | 2 +- tutor/config.py | 1 - tutor/jobs.py | 2 +- .../apps/openedx/config/lms.env.json | 2 - .../apps/openedx/settings/lms/development.py | 1 - tutor/templates/build/forum/Dockerfile | 48 ------------------- .../build/forum/bin/docker-entrypoint.sh | 16 ------- tutor/templates/config.yml | 4 -- tutor/templates/k8s/deployments.yml | 39 --------------- tutor/templates/k8s/jobs.yml | 25 ---------- tutor/templates/k8s/services.yml | 14 ------ tutor/templates/local/docker-compose.jobs.yml | 10 ---- tutor/templates/local/docker-compose.yml | 16 ------- 19 files changed, 8 insertions(+), 190 deletions(-) delete mode 100644 tutor/templates/build/forum/Dockerfile delete mode 100755 tutor/templates/build/forum/bin/docker-entrypoint.sh diff --git a/CHANGELOG-nightly.md b/CHANGELOG-nightly.md index 7a7077d..4ae842e 100644 --- a/CHANGELOG-nightly.md +++ b/CHANGELOG-nightly.md @@ -2,6 +2,7 @@ Note: Breaking changes between versions are indicated by "💥". +- 💥[Improvement] Move the Open edX forum to a [dedicated plugin](https://github.com/overhangio/tutor-forum/) (#450). - 💥[Improvement] Get rid of the "tutor-openedx" package, which is no longer supported. - [Bugfix] Fix running Caddy container in k8s, which should always be the case even if `ENABLE_WEB_PROXY` is false. - 💥[Improvement] Run all services as unprivileged containers, for better security. This has multiple consequences: diff --git a/Makefile b/Makefile index 7356469..60c6106 100644 --- a/Makefile +++ b/Makefile @@ -92,7 +92,7 @@ ci-test-bundle: ## Run basic tests on bundle yes "" | ./dist/tutor config save --interactive ./dist/tutor config save ./dist/tutor plugins list - ./dist/tutor plugins enable android discovery ecommerce license mfe minio notes webui xqueue + ./dist/tutor plugins enable android discovery ecommerce forum license mfe minio notes webui xqueue ./dist/tutor plugins list ./dist/tutor license --help diff --git a/bin/main.py b/bin/main.py index e259de2..4c29441 100755 --- a/bin/main.py +++ b/bin/main.py @@ -6,6 +6,7 @@ for plugin_name in [ "android", "discovery", "ecommerce", + "forum", "license", "mfe", "minio", diff --git a/docs/configuration.rst b/docs/configuration.rst index 16aafee..0648545 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -42,7 +42,6 @@ Individual service activation - ``RUN_LMS`` (default: ``true``) - ``RUN_CMS`` (default: ``true``) -- ``RUN_FORUM`` (default: ``true``) - ``RUN_ELASTICSEARCH`` (default: ``true``) - ``RUN_MONGODB`` (default: ``true``) - ``RUN_MYSQL`` (default: ``true``) @@ -61,9 +60,8 @@ Custom images ************* - ``DOCKER_IMAGE_OPENEDX`` (default: ``"{{ DOCKER_REGISTRY }}overhangio/openedx:{{ TUTOR_VERSION }}"``) -- ``DOCKER_IMAGE_FORUM`` (default: ``"{{ DOCKER_REGISTRY }}overhangio/openedx-forum:{{ TUTOR_VERSION }}"``) -These configuration parameters define which image to run for each service. By default, the docker image tag matches the Tutor version it was built with. +This configuration parameter defines the name of the Docker image to run for the lms and cms containers. By default, the Docker image tag matches the Tutor version it was built with. Custom registry *************** @@ -165,13 +163,6 @@ SMTP Note that the SMTP server shipped with Tutor by default does not implement TLS. With external servers, only one of SSL or TLS should be enabled, at most. -Forum -***** - -- ``RUN_FORUM`` (default: ``true``) -- ``FORUM_HOST`` (default: ``"forum"``) -- ``FORUM_MONGODB_DATABASE`` (default: ``"cs_comments_service"``) - SSL/TLS certificates for HTTPS access ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/plugins/api.rst b/docs/plugins/api.rst index cc85a9d..d989e95 100644 --- a/docs/plugins/api.rst +++ b/docs/plugins/api.rst @@ -83,7 +83,7 @@ Example:: During initialisation, "myservice1" and "myservice2" will be run in sequence with the commands defined in the templates ``myplugin/hooks/myservice1/init`` and ``myplugin/hooks/myservice2/init``. -To initialise a "foo" service, Tutor runs the "foo-job" service that is found in the ``env/local/docker-compose.jobs.yml`` file. By default, Tutor comes with a few services in this file: mysql-job, lms-job, cms-job, forum-job. If your plugin requires running custom services during initialisation, you will need to add them to the ``docker-compose.jobs.yml`` template. To do so, just use the "local-docker-compose-jobs-services" patch. +To initialise a "foo" service, Tutor runs the "foo-job" service that is found in the ``env/local/docker-compose.jobs.yml`` file. By default, Tutor comes with a few services in this file: mysql-job, lms-job, cms-job. If your plugin requires running custom services during initialisation, you will need to add them to the ``docker-compose.jobs.yml`` template. To do so, just use the "local-docker-compose-jobs-services" patch. In Kubernetes, the approach is the same, except that jobs are implemented as actual job objects in the ``k8s/jobs.yml`` template. To add your own services there, your plugin should implement the "k8s-jobs" patch. diff --git a/requirements/plugins.txt b/requirements/plugins.txt index da66ffd..85053a3 100644 --- a/requirements/plugins.txt +++ b/requirements/plugins.txt @@ -2,6 +2,7 @@ tutor-android>=12.0.0,<13.0.0 tutor-discovery>=12.0.0,<13.0.0 tutor-ecommerce>=12.0.0,<13.0.0 +tutor-forum>=12.0.0,<13.0.0 tutor-license>=12.0.0,<13.0.0 tutor-mfe>=12.0.0,<13.0.0 tutor-minio>=12.0.0,<13.0.0 diff --git a/tutor/commands/images.py b/tutor/commands/images.py index 5982f4c..4a0fdd3 100644 --- a/tutor/commands/images.py +++ b/tutor/commands/images.py @@ -10,7 +10,7 @@ from .. import plugins from ..types import Config from .context import Context -BASE_IMAGE_NAMES = ["openedx", "forum", "permissions"] +BASE_IMAGE_NAMES = ["openedx", "permissions"] VENDOR_IMAGES = [ "caddy", "elasticsearch", diff --git a/tutor/config.py b/tutor/config.py index 7770c82..3511ede 100644 --- a/tutor/config.py +++ b/tutor/config.py @@ -165,7 +165,6 @@ def upgrade_obsolete(config: Config) -> None: for name in [ "ACTIVATE_LMS", "ACTIVATE_CMS", - "ACTIVATE_FORUM", "ACTIVATE_ELASTICSEARCH", "ACTIVATE_MONGODB", "ACTIVATE_MYSQL", diff --git a/tutor/jobs.py b/tutor/jobs.py index 9c5afd6..b8afa07 100644 --- a/tutor/jobs.py +++ b/tutor/jobs.py @@ -63,7 +63,7 @@ def initialise(runner: BaseJobRunner, limit_to: Optional[str] = None) -> None: runner.run_job_from_template( service, plugin_name, "hooks", service, "pre-init" ) - for service in ["lms", "cms", "forum"]: + for service in ["lms", "cms"]: if limit_to is None or limit_to == service: fmt.echo_info("Initialising {}...".format(service)) runner.run_job_from_template(service, "hooks", service, "init") diff --git a/tutor/templates/apps/openedx/config/lms.env.json b/tutor/templates/apps/openedx/config/lms.env.json index 3b73079..e39533a 100644 --- a/tutor/templates/apps/openedx/config/lms.env.json +++ b/tutor/templates/apps/openedx/config/lms.env.json @@ -35,8 +35,6 @@ "CELERY_BROKER_USER": "{{ REDIS_USERNAME }}", "CELERY_BROKER_PASSWORD": "{{ REDIS_PASSWORD }}", "ALTERNATE_WORKER_QUEUES": "cms", - "COMMENTS_SERVICE_URL": "http://{{ FORUM_HOST }}:4567", - "COMMENTS_SERVICE_KEY": "forumapikey", "ENABLE_COMPREHENSIVE_THEMING": true, "COMPREHENSIVE_THEME_DIRS": ["/openedx/themes"], "STATIC_ROOT_BASE": "/openedx/staticfiles", diff --git a/tutor/templates/apps/openedx/settings/lms/development.py b/tutor/templates/apps/openedx/settings/lms/development.py index 09c92a4..1038890 100644 --- a/tutor/templates/apps/openedx/settings/lms/development.py +++ b/tutor/templates/apps/openedx/settings/lms/development.py @@ -18,7 +18,6 @@ CMS_ROOT_URL = "http://{}".format(CMS_BASE) LOGIN_REDIRECT_WHITELIST.append(CMS_BASE) FEATURES['ENABLE_COURSEWARE_MICROFRONTEND'] = False -COMMENTS_SERVICE_URL = "http://{{ FORUM_HOST }}:4567" LOGGING["loggers"]["oauth2_provider"] = { "handlers": ["console"], diff --git a/tutor/templates/build/forum/Dockerfile b/tutor/templates/build/forum/Dockerfile deleted file mode 100644 index cdb059e..0000000 --- a/tutor/templates/build/forum/Dockerfile +++ /dev/null @@ -1,48 +0,0 @@ -FROM docker.io/ruby:2.5.7-slim-stretch -MAINTAINER Overhang.io - -ENV DEBIAN_FRONTEND=noninteractive -RUN apt update && \ - apt upgrade -y && \ - apt install -y git wget autoconf bison build-essential libssl-dev libyaml-dev libreadline6-dev zlib1g-dev libncurses5-dev libffi-dev libgdbm-dev - -# Install dockerize to wait for mongodb/elasticsearch availability -ARG DOCKERIZE_VERSION=v0.6.1 -RUN wget -O /tmp/dockerize.tar.gz https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \ - && tar -C /usr/local/bin -xzvf /tmp/dockerize.tar.gz \ - && rm /tmp/dockerize.tar.gz - -# Create unprivileged "app" user -RUN useradd --home-dir /app --create-home --shell /bin/bash --uid 1000 app - -# Copy custom scripts -COPY ./bin /app/bin -RUN chmod a+x /app/bin/* -ENV PATH :${PATH} - -# From then on, run as unprivileged app user -USER app - -# Install rake and bundler -ENV PATH "/app/bin:/app/.gem/ruby/2.5.0/bin:$PATH" -RUN gem install --user-install bundler --version 1.17.3 -RUN gem install --user-install rake --version 13.0.1 - -# Install forum -RUN git clone https://github.com/edx/cs_comments_service.git --branch {{ OPENEDX_COMMON_VERSION }} --depth 1 /app/cs_comments_service -WORKDIR /app/cs_comments_service -RUN bundle install --deployment - -ENTRYPOINT ["docker-entrypoint.sh"] - -ENV SINATRA_ENV staging -ENV NEW_RELIC_ENABLE false -ENV API_KEY forumapikey -ENV SEARCH_SERVER "http://elasticsearch:9200" -ENV MONGODB_AUTH "" -ENV MONGOID_AUTH_MECH "" -ENV MONGODB_HOST "mongodb" -ENV MONGODB_PORT "27017" -ENV MONGODB_DATABASE "cs_comments_service" -EXPOSE 4567 -CMD ./bin/unicorn -c config/unicorn_tcp.rb -I '.' diff --git a/tutor/templates/build/forum/bin/docker-entrypoint.sh b/tutor/templates/build/forum/bin/docker-entrypoint.sh deleted file mode 100755 index 4f5ac7c..0000000 --- a/tutor/templates/build/forum/bin/docker-entrypoint.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -e - -export MONGOHQ_URL="mongodb://$MONGODB_AUTH$MONGODB_HOST:$MONGODB_PORT/$MONGODB_DATABASE" -# the search server variable was renamed after the upgrade to elasticsearch 7 -export SEARCH_SERVER_ES7="$SEARCH_SERVER" - -# make sure that there is an actual authentication mechanism in place, if necessary -if [ -n "$MONGODB_AUTH" ] -then - export MONGOID_AUTH_MECH=":scram" -fi - -echo "Waiting for mongodb/elasticsearch..." -dockerize -wait tcp://$MONGODB_HOST:$MONGODB_PORT -wait $SEARCH_SERVER -wait-retry-interval 5s -timeout 600s - -exec "$@" diff --git a/tutor/templates/config.yml b/tutor/templates/config.yml index eb28c0a..acd4882 100644 --- a/tutor/templates/config.yml +++ b/tutor/templates/config.yml @@ -11,7 +11,6 @@ LMS_HOST: "www.myopenedx.com" # The following are default values RUN_LMS: true RUN_CMS: true -RUN_FORUM: true RUN_ELASTICSEARCH: true RUN_MONGODB: true RUN_MYSQL: true @@ -29,7 +28,6 @@ DOCKER_IMAGE_OPENEDX: "{{ DOCKER_REGISTRY }}overhangio/openedx:{{ TUTOR_VERSION DOCKER_IMAGE_OPENEDX_DEV: "openedx-dev" DOCKER_IMAGE_CADDY: "{{ DOCKER_REGISTRY }}caddy:2.3.0" DOCKER_IMAGE_ELASTICSEARCH: "{{ DOCKER_REGISTRY }}elasticsearch:7.10.1" -DOCKER_IMAGE_FORUM: "{{ DOCKER_REGISTRY }}overhangio/openedx-forum:{{ TUTOR_VERSION }}" DOCKER_IMAGE_MONGODB: "{{ DOCKER_REGISTRY }}mongo:4.2.17" DOCKER_IMAGE_MYSQL: "{{ DOCKER_REGISTRY }}mysql:5.7.35" DOCKER_IMAGE_ELASTICSEARCH: "{{ DOCKER_REGISTRY }}elasticsearch:7.10.1" @@ -44,8 +42,6 @@ ELASTICSEARCH_SCHEME: "http" ELASTICSEARCH_HEAP_SIZE: 1g ENABLE_HTTPS: false ENABLE_WEB_PROXY: true -FORUM_HOST: "forum" -FORUM_MONGODB_DATABASE: "cs_comments_service" JWT_COMMON_AUDIENCE: "openedx" JWT_COMMON_ISSUER: "{% if ENABLE_HTTPS %}https{% else %}http{% endif %}://{{ LMS_HOST }}/oauth2" JWT_COMMON_SECRET_KEY: "{{ OPENEDX_SECRET_KEY }}" diff --git a/tutor/templates/k8s/deployments.yml b/tutor/templates/k8s/deployments.yml index 77b58bc..3841d78 100644 --- a/tutor/templates/k8s/deployments.yml +++ b/tutor/templates/k8s/deployments.yml @@ -134,45 +134,6 @@ spec: configMap: name: openedx-config {% endif %} -{% if RUN_FORUM %} ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: forum - labels: - app.kubernetes.io/name: forum -spec: - selector: - matchLabels: - app.kubernetes.io/name: forum - template: - metadata: - labels: - app.kubernetes.io/name: forum - spec: - securityContext: - runAsUser: 1000 - runAsGroup: 1000 - containers: - - name: forum - image: {{ DOCKER_IMAGE_FORUM }} - ports: - - containerPort: 4567 - env: - - name: SEARCH_SERVER - value: "{{ ELASTICSEARCH_SCHEME }}://{{ ELASTICSEARCH_HOST }}:{{ ELASTICSEARCH_PORT }}" - - name: MONGODB_AUTH - value: "{% if MONGODB_USERNAME and MONGODB_PASSWORD %}{{ MONGODB_USERNAME}}:{{ MONGODB_PASSWORD }}@{% endif %}" - - name: MONGODB_HOST - value: "{{ MONGODB_HOST }}" - - name: MONGODB_PORT - value: "{{ MONGODB_PORT }}" - - name: MONGODB_DATABASE - value: "{{ FORUM_MONGODB_DATABASE }}" - securityContext: - allowPrivilegeEscalation: false -{% endif %} {% if RUN_LMS %} --- apiVersion: apps/v1 diff --git a/tutor/templates/k8s/jobs.yml b/tutor/templates/k8s/jobs.yml index 96a4d3a..d514d01 100644 --- a/tutor/templates/k8s/jobs.yml +++ b/tutor/templates/k8s/jobs.yml @@ -77,30 +77,5 @@ spec: containers: - name: mysql image: {{ DOCKER_IMAGE_MYSQL }} ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: forum-job - labels: - app.kubernetes.io/component: job -spec: - template: - spec: - restartPolicy: Never - containers: - - name: forum - image: {{ DOCKER_IMAGE_FORUM }} - env: - - name: SEARCH_SERVER - value: "{{ ELASTICSEARCH_SCHEME }}://{{ ELASTICSEARCH_HOST }}:{{ ELASTICSEARCH_PORT }}" - - name: MONGODB_AUTH - value: "{% if MONGODB_USERNAME and MONGODB_PASSWORD %}{{ MONGODB_USERNAME}}:{{ MONGODB_PASSWORD }}@{% endif %}" - - name: MONGODB_HOST - value: "{{ MONGODB_HOST }}" - - name: MONGODB_PORT - value: "{{ MONGODB_PORT }}" - - name: MONGODB_DATABASE - value: "{{ FORUM_MONGODB_DATABASE }}" {{ patch("k8s-jobs") }} diff --git a/tutor/templates/k8s/services.yml b/tutor/templates/k8s/services.yml index 68e78ec..b768098 100644 --- a/tutor/templates/k8s/services.yml +++ b/tutor/templates/k8s/services.yml @@ -28,20 +28,6 @@ spec: selector: app.kubernetes.io/name: cms {% endif %} -{% if RUN_FORUM %} ---- -apiVersion: v1 -kind: Service -metadata: - name: forum -spec: - type: NodePort - ports: - - port: 4567 - protocol: TCP - selector: - app.kubernetes.io/name: forum -{% endif %} {% if RUN_LMS %} --- apiVersion: v1 diff --git a/tutor/templates/local/docker-compose.jobs.yml b/tutor/templates/local/docker-compose.jobs.yml index 26b1237..880660f 100644 --- a/tutor/templates/local/docker-compose.jobs.yml +++ b/tutor/templates/local/docker-compose.jobs.yml @@ -27,14 +27,4 @@ services: - ../apps/openedx/config/:/openedx/config/:ro depends_on: {{ [("mysql", RUN_MYSQL)]|list_if }} - forum-job: - image: {{ DOCKER_IMAGE_FORUM }} - environment: - SEARCH_SERVER: "{{ ELASTICSEARCH_SCHEME }}://{{ ELASTICSEARCH_HOST }}:{{ ELASTICSEARCH_PORT }}" - MONGODB_AUTH: "{% if MONGODB_USERNAME and MONGODB_PASSWORD %}{{ MONGODB_USERNAME}}:{{ MONGODB_PASSWORD }}@{% endif %}" - MONGODB_HOST: "{{ MONGODB_HOST }}" - MONGODB_PORT: "{{ MONGODB_PORT }}" - MONGODB_DATABASE: "{{ FORUM_MONGODB_DATABASE }}" - depends_on: {{ [("elasticsearch", RUN_ELASTICSEARCH), ("mongodb", RUN_MONGODB)]|list_if }} - {{ patch("local-docker-compose-jobs-services")|indent(4) }} diff --git a/tutor/templates/local/docker-compose.yml b/tutor/templates/local/docker-compose.yml index e79294a..163a70e 100644 --- a/tutor/templates/local/docker-compose.yml +++ b/tutor/templates/local/docker-compose.yml @@ -97,21 +97,6 @@ services: HOSTNAME: "{{ LMS_HOST }}" {% endif %} - ############# Forum - - {% if RUN_FORUM %} - forum: - image: {{ DOCKER_IMAGE_FORUM }} - environment: - SEARCH_SERVER: "{{ ELASTICSEARCH_SCHEME }}://{{ ELASTICSEARCH_HOST }}:{{ ELASTICSEARCH_PORT }}" - MONGODB_AUTH: "{% if MONGODB_USERNAME and MONGODB_PASSWORD %}{{ MONGODB_USERNAME}}:{{ MONGODB_PASSWORD }}@{% endif %}" - MONGODB_HOST: "{{ MONGODB_HOST }}" - MONGODB_PORT: "{{ MONGODB_PORT }}" - MONGODB_DATABASE: "{{ FORUM_MONGODB_DATABASE }}" - restart: unless-stopped - depends_on: {{ [("elasticsearch", RUN_ELASTICSEARCH), ("mongodb", RUN_MONGODB)]|list_if }} - {% endif %} - ############# LMS and CMS {% if RUN_LMS %} @@ -132,7 +117,6 @@ services: - lms-permissions {% if RUN_MYSQL %}- mysql{% endif %} {% if RUN_ELASTICSEARCH %}- elasticsearch{% endif %} - {% if RUN_FORUM %}- forum{% endif %} {% if RUN_MONGODB %}- mongodb{% endif %} {% if RUN_REDIS %}- redis{% endif %} {% if RUN_SMTP %}- smtp{% endif %} From 72baae0e27379821652ee7db495708f9cb6396ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Tue, 9 Nov 2021 11:30:00 +0100 Subject: [PATCH 12/56] fix: enable plugins to implement the "caddyfile" patch When nginx was removed in favour of caddy, we decided that plugin implementations of the "caddyfile" patch should make use of the "port" local variable. However, local variables are not available from inside plugin patches, which are rendered outside of the context of the parent templates. For a more extensive description of the problem, see: https://github.com/overhangio/tutor-mfe/pull/23#issuecomment-964016190 We still want to make it easy for developers to decide what should the port be for caddy hosts. To do so, we make use of environment variables that are passed at runtime to the caddy container. Thus, a regular plugin patch should look like this: {{ PLUGIN_HOST }}{$default_site_port} { import proxy "myplugin:8000" } --- CHANGELOG-nightly.md | 1 + tutor/templates/apps/caddy/Caddyfile | 11 ++--------- tutor/templates/k8s/deployments.yml | 3 +++ tutor/templates/local/docker-compose.prod.yml | 2 ++ 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/CHANGELOG-nightly.md b/CHANGELOG-nightly.md index 4ae842e..16bb0ff 100644 --- a/CHANGELOG-nightly.md +++ b/CHANGELOG-nightly.md @@ -2,6 +2,7 @@ Note: Breaking changes between versions are indicated by "💥". +- [Bugfix] Make it possible for plugins to implement the "caddyfile" patch without relying on the "port" local variable. - 💥[Improvement] Move the Open edX forum to a [dedicated plugin](https://github.com/overhangio/tutor-forum/) (#450). - 💥[Improvement] Get rid of the "tutor-openedx" package, which is no longer supported. - [Bugfix] Fix running Caddy container in k8s, which should always be the case even if `ENABLE_WEB_PROXY` is false. diff --git a/tutor/templates/apps/caddy/Caddyfile b/tutor/templates/apps/caddy/Caddyfile index b8c22ae..ae92508 100644 --- a/tutor/templates/apps/caddy/Caddyfile +++ b/tutor/templates/apps/caddy/Caddyfile @@ -25,14 +25,7 @@ } } -{% if ENABLE_HTTPS and ENABLE_WEB_PROXY %} -{% set port = "" %} -{# listening to https is disabled and we must only listen to http #} -{% else %} -{% set port = ":80" %} -{% endif %} - -{{ LMS_HOST }}{{ port }}, {{ PREVIEW_LMS_HOST }}{{ port }} { +{{ LMS_HOST }}{$default_site_port}, {{ PREVIEW_LMS_HOST }}{$default_site_port} { @favicon_matcher { path_regexp ^(.*)/favicon.ico$ } @@ -51,7 +44,7 @@ {{ patch("caddyfile-lms")|indent(4) }} } -{{ CMS_HOST }}{{ port }} { +{{ CMS_HOST }}{$default_site_port} { @favicon_matcher { path_regexp ^(.*)/favicon.ico$ } diff --git a/tutor/templates/k8s/deployments.yml b/tutor/templates/k8s/deployments.yml index 3841d78..0694c21 100644 --- a/tutor/templates/k8s/deployments.yml +++ b/tutor/templates/k8s/deployments.yml @@ -17,6 +17,9 @@ spec: containers: - name: caddy image: {{ DOCKER_IMAGE_CADDY }} + env: + - name: default_site_port + value: "{% if not ENABLE_HTTPS or not ENABLE_WEB_PROXY %}:80{% endif %}" volumeMounts: - mountPath: /etc/caddy/ name: config diff --git a/tutor/templates/local/docker-compose.prod.yml b/tutor/templates/local/docker-compose.prod.yml index 5f670e1..b753bfa 100644 --- a/tutor/templates/local/docker-compose.prod.yml +++ b/tutor/templates/local/docker-compose.prod.yml @@ -7,6 +7,8 @@ services: ports: - "{{ CADDY_HTTP_PORT }}:80" {% if ENABLE_HTTPS and ENABLE_WEB_PROXY %}- "443:443"{% endif %} + environment: + default_site_port: "{% if not ENABLE_HTTPS or not ENABLE_WEB_PROXY %}:80{% endif %}" volumes: - ../apps/caddy/Caddyfile:/etc/caddy/Caddyfile:ro {% if ENABLE_HTTPS and ENABLE_WEB_PROXY %}- ../../data/caddy:/data{% endif %} From 0153e7a690613b16d10d0555cc73f9f97e3ae8d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Tue, 9 Nov 2021 11:42:39 +0100 Subject: [PATCH 13/56] fix: https test --- tests/test_env.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_env.py b/tests/test_env.py index 17528f2..5a03d5d 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -94,7 +94,7 @@ class EnvTests(unittest.TestCase): with patch.object(fmt, "STDOUT"): env.save(root, config) with open(os.path.join(root, "env", "apps", "caddy", "Caddyfile")) as f: - self.assertIn("www.myopenedx.com {", f.read()) + self.assertIn("www.myopenedx.com{$default_site_port}", f.read()) def test_patch(self) -> None: patches = {"plugin1": "abcd", "plugin2": "efgh"} From ddbfdb919ed718be04172fe43d5bbbb0a97ca2e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Mon, 15 Nov 2021 06:38:04 +0100 Subject: [PATCH 14/56] fix: disable forum feature by default --- tutor/templates/apps/openedx/settings/partials/common_all.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tutor/templates/apps/openedx/settings/partials/common_all.py b/tutor/templates/apps/openedx/settings/partials/common_all.py index 82c1190..ac63b81 100644 --- a/tutor/templates/apps/openedx/settings/partials/common_all.py +++ b/tutor/templates/apps/openedx/settings/partials/common_all.py @@ -152,6 +152,9 @@ JWT_AUTH["JWT_ISSUERS"] = [ } ] +# Enable/Disable some features globally +FEATURES["ENABLE_DISCUSSION_SERVICE"] = False + # Disable codejail support # explicitely configuring python is necessary to prevent unsafe calls import codejail.jail_code From e0335bbd2b896b187f2415ba195c435b619b7a4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Tue, 16 Nov 2021 10:04:12 +0100 Subject: [PATCH 15/56] fix: get rid of useless redis folder in permission setting Adding these volumes was a mistake. --- tutor/templates/local/docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/tutor/templates/local/docker-compose.yml b/tutor/templates/local/docker-compose.yml index 163a70e..087c3ca 100644 --- a/tutor/templates/local/docker-compose.yml +++ b/tutor/templates/local/docker-compose.yml @@ -126,7 +126,6 @@ services: command: ["1000", "/openedx/data", "/openedx/media"] restart: on-failure volumes: - - ../../data/redis:/openedx/redis/data - ../../data/lms:/openedx/data - ../../data/openedx-media:/openedx/media {% endif %} @@ -159,7 +158,6 @@ services: command: ["1000", "/openedx/data", "/openedx/media"] restart: on-failure volumes: - - ../../data/redis:/openedx/redis/data - ../../data/cms:/openedx/data - ../../data/openedx-media:/openedx/media {% endif %} From 1ddf6b1271aba197db8a4f90a5fcea35f5141482 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Thu, 18 Nov 2021 11:54:42 +0100 Subject: [PATCH 16/56] fix: don't attempt to security patch edx-platform --- tutor/templates/build/openedx/Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index 7b7fb9d..217329c 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -44,9 +44,6 @@ RUN git config --global user.email "tutor@overhang.io" \ {{ patch("openedx-dockerfile-git-patches-default") }} {% else %} # Patch edx-platform -# edx-proctoring security fix https://github.com/edx/edx-platform/pull/29347/ -RUN git fetch --depth=2 https://github.com/edx/edx-platform d61dcac29d1651956623c150be53a8bbe69e9346 \ - && git cherry-pick d61dcac29d1651956623c150be53a8bbe69e9346 {% endif %} {# Example: RUN git fetch --depth=2 https://github.com/edx/edx-platform && git cherry-pick #} From 55582575f0281e2be6688879adb1208e07546fcb Mon Sep 17 00:00:00 2001 From: Florian Haas Date: Thu, 18 Nov 2021 14:07:40 +0100 Subject: [PATCH 17/56] fix: Stop creating immutable resource label that breaks "tutor k8s" on Tutor version changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Through the commonLabels directive in kustomization.yml, all resources get a label named "app.kubernetes.io/version", which is being set to the Tutor version at the time of initial deployment. When the user then subsequently progresses to a new Tutor version, Kubernetes attempts to update this label — but for Deployment, ReplicaSet, and DaemonSet resources, this is no longer allowed as of https://github.com/kubernetes/kubernetes/issues/50808. This causes "tutor k8s start" (at the "kubectl apply --kustomize" step) to break with errors such as: Deployment.apps "redis" is invalid: spec.selector: Invalid value: v1.LabelSelector{MatchLabels:map[string]string{"app.kubernetes.io/instance":"openedx-JIONBLbtByCGUYgHgr4tDWu1", "app.kubernetes.io/managed-by":"tutor", "app.kubernetes.io/name":"redis", "app.kubernetes.io/part-of":"openedx", "app.kubernetes.io/version":"12.1.7"}, MatchExpressions:[]v1.LabelSelectorRequirement(nil)}: field is immutable Simply removing the app.kubernetes.io/version label from kustomization.yml will permanently fix this issue for newly created Kubernetes deployments, which will "survive" any future Tutor version changes thereafter. However, *existing* production Open edX deployments will need to throw the affected Deployments away, and re-create them. Also, add the Tutor version as a resource annotation instead, using the commonAnnotations directive. See also: https://github.com/kubernetes/client-go/issues/508 https://kubectl.docs.kubernetes.io/references/kustomize/kustomization/commonlabels/ https://kubectl.docs.kubernetes.io/references/kustomize/kustomization/commonannotations/ Fixes #531. --- CHANGELOG-nightly.md | 1 + tutor/templates/kustomization.yml | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG-nightly.md b/CHANGELOG-nightly.md index 16bb0ff..dc3af97 100644 --- a/CHANGELOG-nightly.md +++ b/CHANGELOG-nightly.md @@ -2,6 +2,7 @@ Note: Breaking changes between versions are indicated by "💥". +- 💥[Bugfix] No longer track the Tutor version number in resource labels (and label selectors, which breaks the update of Deployment resources), but instead do so in resource annotations. - [Bugfix] Make it possible for plugins to implement the "caddyfile" patch without relying on the "port" local variable. - 💥[Improvement] Move the Open edX forum to a [dedicated plugin](https://github.com/overhangio/tutor-forum/) (#450). - 💥[Improvement] Get rid of the "tutor-openedx" package, which is no longer supported. diff --git a/tutor/templates/kustomization.yml b/tutor/templates/kustomization.yml index e34666c..df7367d 100644 --- a/tutor/templates/kustomization.yml +++ b/tutor/templates/kustomization.yml @@ -12,11 +12,16 @@ resources: # namespace to deploy all Resources to namespace: {{ K8S_NAMESPACE }} -# labels added to all Resources +# annotations added to all Resources +# https://kubectl.docs.kubernetes.io/references/kustomize/kustomization/commonannotations/ +commonAnnotations: + app.kubernetes.io/version: {{ TUTOR_VERSION }} + +# labels (and label selectors) added to all Resources # https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/ +# https://kubectl.docs.kubernetes.io/references/kustomize/kustomization/commonlabels/ commonLabels: app.kubernetes.io/instance: openedx-{{ ID }} - app.kubernetes.io/version: {{ TUTOR_VERSION }} app.kubernetes.io/part-of: openedx app.kubernetes.io/managed-by: tutor {{ patch("kustomization-commonlabels")|indent(2) }} From a074ff34c79a43d11974ec90557d93ab816123ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Mon, 29 Nov 2021 14:12:22 +0100 Subject: [PATCH 18/56] fix: docker-compose project name in dev on nightly Project name was incorrectly set to "tutor_dev" instead of "tutor_nightly_dev". --- CHANGELOG-nightly.md | 1 + tutor/templates/config.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG-nightly.md b/CHANGELOG-nightly.md index dc3af97..f240b3c 100644 --- a/CHANGELOG-nightly.md +++ b/CHANGELOG-nightly.md @@ -2,6 +2,7 @@ Note: Breaking changes between versions are indicated by "💥". +- [Bugfix] Fix docker-compose project name in development on nightly branch. - 💥[Bugfix] No longer track the Tutor version number in resource labels (and label selectors, which breaks the update of Deployment resources), but instead do so in resource annotations. - [Bugfix] Make it possible for plugins to implement the "caddyfile" patch without relying on the "port" local variable. - 💥[Improvement] Move the Open edX forum to a [dedicated plugin](https://github.com/overhangio/tutor-forum/) (#450). diff --git a/tutor/templates/config.yml b/tutor/templates/config.yml index acd4882..58af558 100644 --- a/tutor/templates/config.yml +++ b/tutor/templates/config.yml @@ -22,7 +22,7 @@ PREVIEW_LMS_HOST: "preview.{{ LMS_HOST }}" CONTACT_EMAIL: "contact@{{ LMS_HOST }}" OPENEDX_AWS_ACCESS_KEY: "" OPENEDX_AWS_SECRET_ACCESS_KEY: "" -DEV_PROJECT_NAME: "tutor_dev" +DEV_PROJECT_NAME: "{{ TUTOR_APP }}_dev" DOCKER_REGISTRY: "docker.io/" DOCKER_IMAGE_OPENEDX: "{{ DOCKER_REGISTRY }}overhangio/openedx:{{ TUTOR_VERSION }}" DOCKER_IMAGE_OPENEDX_DEV: "openedx-dev" From 1f8555b80e35c16569127bedcc358120ab93b184 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Tue, 30 Nov 2021 10:47:08 +0100 Subject: [PATCH 19/56] chore: drop python 3.5 compatibility Python 3.5 has reached end of life in September 3.5. Anyway, Tutor was not compatible because some dev dependencies, such as astroid 2.8.3, are no longer available in 3.5. This means that we can now start using many python 3.6 niceties, such as f-strings \o/ --- CHANGELOG-nightly.md | 1 + setup.py | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG-nightly.md b/CHANGELOG-nightly.md index f240b3c..665d430 100644 --- a/CHANGELOG-nightly.md +++ b/CHANGELOG-nightly.md @@ -2,6 +2,7 @@ Note: Breaking changes between versions are indicated by "💥". +- 💥[Improvement] Drop Python 3.5 compatibility. - [Bugfix] Fix docker-compose project name in development on nightly branch. - 💥[Bugfix] No longer track the Tutor version number in resource labels (and label selectors, which breaks the update of Deployment resources), but instead do so in resource annotations. - [Bugfix] Make it possible for plugins to implement the "caddyfile" patch without relying on the "port" local variable. diff --git a/setup.py b/setup.py index 982c941..2f77aaa 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ setup( long_description_content_type="text/x-rst", packages=find_packages(exclude=["tests*"]), include_package_data=True, - python_requires=">=3.5", + python_requires=">=3.6", install_requires=load_requirements(), entry_points={"console_scripts": ["tutor=tutor.commands.cli:main"]}, classifiers=[ @@ -61,7 +61,6 @@ setup( "License :: OSI Approved :: GNU Affero General Public License v3", "Operating System :: OS Independent", "Programming Language :: Python", - "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", From 0052481d42b8caf87cb6b0076c9da61a344f73b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Mon, 8 Nov 2021 15:41:35 +0100 Subject: [PATCH 20/56] fix: lint unused arguments in code base --- Makefile | 2 +- tutor/bindmounts.py | 4 ++-- tutor/env.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 910f93c..a7268ba 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ test-format: ## Run code formatting tests black --check --diff $(BLACK_OPTS) test-lint: ## Run code linting tests - pylint --errors-only --enable=unused-import --ignore=templates ${SRC_DIRS} + pylint --errors-only --enable=unused-import,unused-argument --ignore=templates ${SRC_DIRS} test-unit: ## Run unit tests python -m unittest discover tests diff --git a/tutor/bindmounts.py b/tutor/bindmounts.py index bbefca7..3f25d6e 100644 --- a/tutor/bindmounts.py +++ b/tutor/bindmounts.py @@ -73,8 +73,8 @@ def parse_volumes(docker_compose_args: List[str]) -> Tuple[List[str], List[str]] @click.option("-v", "--volume", "volumes", multiple=True) @click.argument("args", nargs=-1) def custom_docker_compose( - volumes: List[str], args: List[str] - ) -> None: # pylint: disable=unused-argument + volumes: List[str], args: List[str] # pylint: disable=unused-argument + ) -> None: pass if isinstance(docker_compose_args, tuple): diff --git a/tutor/env.py b/tutor/env.py index 7639415..4cd8c48 100644 --- a/tutor/env.py +++ b/tutor/env.py @@ -222,7 +222,7 @@ def save(root: str, config: Config) -> None: fmt.echo_info("Environment generated in {}".format(base_dir(root))) -def upgrade_obsolete(root: str) -> None: +def upgrade_obsolete(_root: str) -> None: """ Add here ad-hoc commands to upgrade the environment. """ From 092dfbff67cdc3a298fec8229cc6d09930b3bb45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Mon, 8 Nov 2021 14:46:38 +0100 Subject: [PATCH 21/56] refactor: clarify configuration management Previously, configuration management was very confusing because we kept mixing "base" and "defaults" configuration: - It was difficult to make the difference between core settings that were necessary (e.g: passwords) as opposed to others that could simply be defaulted to. - The order of settings in config.yml mattered: config entries that depended on other needed to be defined later. As a consequence, Tutor was not compatible with Python 3.5, where dict entries are not sorted. --- tests/test_config.py | 41 +-- tests/test_env.py | 13 +- tests/test_plugins.py | 57 ++-- tutor/commands/config.py | 14 +- tutor/commands/k8s.py | 10 +- tutor/commands/local.py | 2 +- tutor/commands/plugins.py | 6 +- tutor/config.py | 247 ++++++++++-------- tutor/interactive.py | 24 +- tutor/templates/config/base.yml | 6 + .../{config.yml => config/defaults.yml} | 34 +-- 11 files changed, 233 insertions(+), 221 deletions(-) create mode 100644 tutor/templates/config/base.yml rename tutor/templates/{config.yml => config/defaults.yml} (86%) diff --git a/tests/test_config.py b/tests/test_config.py index 3fab30b..30f934f 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -2,6 +2,8 @@ import unittest from unittest.mock import Mock, patch import tempfile +import click + from tutor import config as tutor_config from tutor import interactive from tutor.types import get_typed, Config @@ -9,7 +11,7 @@ from tutor.types import get_typed, Config class ConfigTests(unittest.TestCase): def test_version(self) -> None: - defaults = tutor_config.load_defaults() + defaults = tutor_config.get_defaults({}) self.assertNotIn("TUTOR_VERSION", defaults) def test_merge(self) -> None: @@ -18,22 +20,21 @@ class ConfigTests(unittest.TestCase): tutor_config.merge(config1, config2) self.assertEqual({"x": "y"}, config1) - def test_merge_render(self) -> None: + def test_merge_not_render(self) -> None: config: Config = {} - defaults = tutor_config.load_defaults() + base = tutor_config.get_base({}) with patch.object(tutor_config.utils, "random_string", return_value="abcd"): - tutor_config.merge(config, defaults) + tutor_config.merge(config, base) - self.assertEqual("abcd", config["MYSQL_ROOT_PASSWORD"]) + # Check that merge does not perform a rendering + self.assertNotEqual("abcd", config["MYSQL_ROOT_PASSWORD"]) @patch.object(tutor_config.fmt, "echo") - def test_update_twice(self, _: Mock) -> None: + def test_save_load(self, _: Mock) -> None: with tempfile.TemporaryDirectory() as root: - tutor_config.update(root) - config1 = tutor_config.load_user(root) - - tutor_config.update(root) - config2 = tutor_config.load_user(root) + config1 = tutor_config.load_minimal(root) + tutor_config.save_config_file(root, config1) + config2 = tutor_config.load_minimal(root) self.assertEqual(config1, config2) @@ -44,28 +45,32 @@ class ConfigTests(unittest.TestCase): tutor_config.utils, "random_string" ) as mock_random_string: mock_random_string.return_value = "abcd" - config1, _defaults1 = tutor_config.load_all(root) + config1 = tutor_config.load_full(root) password1 = config1["MYSQL_ROOT_PASSWORD"] config1.pop("MYSQL_ROOT_PASSWORD") tutor_config.save_config_file(root, config1) mock_random_string.return_value = "efgh" - config2, _defaults2 = tutor_config.load_all(root) + config2 = tutor_config.load_full(root) password2 = config2["MYSQL_ROOT_PASSWORD"] self.assertEqual("abcd", password1) self.assertEqual("efgh", password2) - def test_interactive_load_all(self) -> None: + def test_interactive(self) -> None: + def mock_prompt(*_args: None, **kwargs: str) -> str: + return kwargs["default"] + with tempfile.TemporaryDirectory() as rootdir: - config, defaults = interactive.load_all(rootdir, interactive=False) + with patch.object(click, "prompt", new=mock_prompt): + with patch.object(click, "confirm", new=mock_prompt): + config = interactive.load_user_config(rootdir, interactive=True) self.assertIn("MYSQL_ROOT_PASSWORD", config) self.assertEqual(8, len(get_typed(config, "MYSQL_ROOT_PASSWORD", str))) - self.assertNotIn("LMS_HOST", config) - self.assertEqual("www.myopenedx.com", defaults["LMS_HOST"]) - self.assertEqual("studio.{{ LMS_HOST }}", defaults["CMS_HOST"]) + self.assertEqual("www.myopenedx.com", config["LMS_HOST"]) + self.assertEqual("studio.www.myopenedx.com", config["CMS_HOST"]) def test_is_service_activated(self) -> None: config: Config = {"RUN_SERVICE1": True, "RUN_SERVICE2": False} diff --git a/tests/test_env.py b/tests/test_env.py index 5a03d5d..b0e7395 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -63,7 +63,10 @@ class EnvTests(unittest.TestCase): def test_render_file(self) -> None: config: Config = {} - tutor_config.merge(config, tutor_config.load_defaults()) + tutor_config.update_with_base(config) + tutor_config.update_with_defaults(config) + tutor_config.render_full(config) + config["MYSQL_ROOT_PASSWORD"] = "testpassword" rendered = env.render_file(config, "hooks", "mysql", "init") self.assertIn("testpassword", rendered) @@ -75,10 +78,8 @@ class EnvTests(unittest.TestCase): ) def test_save_full(self) -> None: - defaults = tutor_config.load_defaults() with tempfile.TemporaryDirectory() as root: - config = tutor_config.load_current(root, defaults) - tutor_config.merge(config, defaults) + config = tutor_config.load_full(root) with patch.object(fmt, "STDOUT"): env.save(root, config) self.assertTrue( @@ -86,10 +87,8 @@ class EnvTests(unittest.TestCase): ) def test_save_full_with_https(self) -> None: - defaults = tutor_config.load_defaults() with tempfile.TemporaryDirectory() as root: - config = tutor_config.load_current(root, defaults) - tutor_config.merge(config, defaults) + config = tutor_config.load_full(root) config["ENABLE_HTTPS"] = True with patch.object(fmt, "STDOUT"): env.save(root, config) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 54a98a6..38f756e 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -128,9 +128,6 @@ class PluginsTests(unittest.TestCase): self.assertEqual([], patches) def test_configure(self) -> None: - config: Config = {"ID": "id"} - defaults: Config = {} - class plugin1: config: Config = { "add": {"PARAM1": "value1", "PARAM2": "value2"}, @@ -143,37 +140,31 @@ class PluginsTests(unittest.TestCase): "iter_enabled", return_value=[plugins.BasePlugin("plugin1", plugin1)], ): - tutor_config.load_plugins(config, defaults) + base = tutor_config.get_base({}) + defaults = tutor_config.get_defaults({}) - self.assertEqual( - { - "ID": "id", - "PARAM3": "value3", - "PLUGIN1_PARAM1": "value1", - "PLUGIN1_PARAM2": "value2", - }, - config, - ) - self.assertEqual({"PLUGIN1_PARAM4": "value4"}, defaults) + self.assertEqual(base["PARAM3"], "value3") + self.assertEqual(base["PLUGIN1_PARAM1"], "value1") + self.assertEqual(base["PLUGIN1_PARAM2"], "value2") + self.assertEqual(defaults["PLUGIN1_PARAM4"], "value4") def test_configure_set_does_not_override(self) -> None: - config: Config = {"ID": "oldid"} + config: Config = {"ID1": "oldid"} class plugin1: - config: Config = {"set": {"ID": "newid"}} + config: Config = {"set": {"ID1": "newid", "ID2": "id2"}} with patch.object( plugins.Plugins, "iter_enabled", return_value=[plugins.BasePlugin("plugin1", plugin1)], ): - tutor_config.load_plugins(config, {}) + tutor_config.update_with_base(config) - self.assertEqual({"ID": "oldid"}, config) + self.assertEqual("oldid", config["ID1"]) + self.assertEqual("id2", config["ID2"]) def test_configure_set_random_string(self) -> None: - config: Config = {} - class plugin1: config: Config = {"set": {"PARAM1": "{{ 128|random_string }}"}} @@ -182,12 +173,13 @@ class PluginsTests(unittest.TestCase): "iter_enabled", return_value=[plugins.BasePlugin("plugin1", plugin1)], ): - tutor_config.load_plugins(config, {}) + config = tutor_config.get_base({}) + tutor_config.render_full(config) + self.assertEqual(128, len(get_typed(config, "PARAM1", str))) def test_configure_default_value_with_previous_definition(self) -> None: - config: Config = {} - defaults: Config = {"PARAM1": "value"} + config: Config = {"PARAM1": "value"} class plugin1: config: Config = {"defaults": {"PARAM2": "{{ PARAM1 }}"}} @@ -197,10 +189,10 @@ class PluginsTests(unittest.TestCase): "iter_enabled", return_value=[plugins.BasePlugin("plugin1", plugin1)], ): - tutor_config.load_plugins(config, defaults) - self.assertEqual("{{ PARAM1 }}", defaults["PLUGIN1_PARAM2"]) + tutor_config.update_with_defaults(config) + self.assertEqual("{{ PARAM1 }}", config["PLUGIN1_PARAM2"]) - def test_configure_add_twice(self) -> None: + def test_config_load_from_plugins(self) -> None: config: Config = {} class plugin1: @@ -211,19 +203,12 @@ class PluginsTests(unittest.TestCase): "iter_enabled", return_value=[plugins.BasePlugin("plugin1", plugin1)], ): - tutor_config.load_plugins(config, {}) + tutor_config.update_with_base(config) + tutor_config.update_with_defaults(config) + tutor_config.render_full(config) value1 = get_typed(config, "PLUGIN1_PARAM1", str) - with patch.object( - plugins.Plugins, - "iter_enabled", - return_value=[plugins.BasePlugin("plugin1", plugin1)], - ): - tutor_config.load_plugins(config, {}) - value2 = get_typed(config, "PLUGIN1_PARAM1", str) self.assertEqual(10, len(value1)) - self.assertEqual(10, len(value2)) - self.assertEqual(value1, value2) def test_hooks(self) -> None: class plugin1: diff --git a/tutor/commands/config.py b/tutor/commands/config.py index de68128..964813d 100644 --- a/tutor/commands/config.py +++ b/tutor/commands/config.py @@ -50,16 +50,16 @@ def save( unset_vars: List[str], env_only: bool, ) -> None: - config, defaults = interactive_config.load_all( - context.root, interactive=interactive - ) + config = interactive_config.load_user_config(context.root, interactive=interactive) if set_vars: - tutor_config.merge(config, dict(set_vars), force=True) + for key, value in dict(set_vars).items(): + config[key] = env.render_unknown(config, value) for key in unset_vars: config.pop(key, None) if not env_only: tutor_config.save_config_file(context.root, config) - tutor_config.merge(config, defaults) + + tutor_config.render_full(config) env.save(context.root, config) @@ -78,8 +78,8 @@ def save( def render(context: Context, extra_configs: List[str], src: str, dst: str) -> None: config = tutor_config.load(context.root) for extra_config in extra_configs: - tutor_config.merge( - config, tutor_config.load_config_file(extra_config), force=True + config.update( + env.render_unknown(config, tutor_config.get_yaml_file(extra_config)) ) renderer = env.Renderer(config, [src]) diff --git a/tutor/commands/k8s.py b/tutor/commands/k8s.py index dc97b50..18443bf 100644 --- a/tutor/commands/k8s.py +++ b/tutor/commands/k8s.py @@ -8,11 +8,11 @@ from .. import config as tutor_config from .. import env as tutor_env from .. import exceptions from .. import fmt -from .. import interactive as interactive_config from .. import jobs from .. import serialize from ..types import Config, get_typed from .. import utils +from .config import save as config_save_command from .context import Context @@ -162,9 +162,13 @@ def k8s() -> None: @click.pass_context def quickstart(context: click.Context, non_interactive: bool) -> None: click.echo(fmt.title("Interactive platform configuration")) - config = interactive_config.update( - context.obj.root, interactive=(not non_interactive) + context.invoke( + config_save_command, + interactive=(not non_interactive), + set_vars=[], + unset_vars=[], ) + config = tutor_config.load(context.obj.root) if not config["ENABLE_WEB_PROXY"]: fmt.echo_alert( "Potentially invalid configuration: ENABLE_WEB_PROXY=false\n" diff --git a/tutor/commands/local.py b/tutor/commands/local.py index 8f9b97c..c706561 100644 --- a/tutor/commands/local.py +++ b/tutor/commands/local.py @@ -109,7 +109,7 @@ Your Open edX platform is ready and can be accessed at the following urls: @click.option("-I", "--non-interactive", is_flag=True, help="Run non-interactively") @click.pass_context def upgrade(context: click.Context, from_version: str, non_interactive: bool) -> None: - config = tutor_config.load_no_check(context.obj.root) + config = tutor_config.load_full(context.obj.root) if not non_interactive: question = """You are about to upgrade your Open edX platform. It is strongly recommended to make a backup before upgrading. To do so, run: diff --git a/tutor/commands/plugins.py b/tutor/commands/plugins.py index 54c9c86..6fe1fad 100644 --- a/tutor/commands/plugins.py +++ b/tutor/commands/plugins.py @@ -28,7 +28,7 @@ def plugins_command() -> None: @click.command(name="list", help="List installed plugins") @click.pass_obj def list_command(context: Context) -> None: - config = tutor_config.load_user(context.root) + config = tutor_config.load_full(context.root) for plugin in plugins.iter_installed(): status = "" if plugins.is_enabled(config, plugin.name) else " (disabled)" print( @@ -42,7 +42,7 @@ def list_command(context: Context) -> None: @click.argument("plugin_names", metavar="plugin", nargs=-1) @click.pass_obj def enable(context: Context, plugin_names: List[str]) -> None: - config = tutor_config.load_user(context.root) + config = tutor_config.load_full(context.root) for plugin in plugin_names: plugins.enable(config, plugin) fmt.echo_info("Plugin {} enabled".format(plugin)) @@ -59,7 +59,7 @@ def enable(context: Context, plugin_names: List[str]) -> None: @click.argument("plugin_names", metavar="plugin", nargs=-1) @click.pass_obj def disable(context: Context, plugin_names: List[str]) -> None: - config = tutor_config.load_user(context.root) + config = tutor_config.load_full(context.root) disable_all = "all" in plugin_names for plugin in plugins.iter_enabled(config): if disable_all or plugin.name in plugin_names: diff --git a/tutor/config.py b/tutor/config.py index 3511ede..4973d24 100644 --- a/tutor/config.py +++ b/tutor/config.py @@ -1,134 +1,178 @@ import os -from typing import Tuple from . import env, exceptions, fmt, plugins, serialize, utils from .types import Config, cast_config -def update(root: str) -> Config: - """ - Load and save the configuration. - """ - config, defaults = load_all(root) - save_config_file(root, config) - merge(config, defaults) - return config - - def load(root: str) -> Config: """ - Load full configuration. This will raise an exception if there is no current - configuration in the project root. + Load full configuration. + + This will raise an exception if there is no current configuration in the + project root. A warning will also be printed if the version from disk + differs from the package version. """ - check_existing_config(root) - return load_no_check(root) + if not os.path.exists(config_path(root)): + raise exceptions.TutorError( + "Project root does not exist. Make sure to generate the initial " + "configuration with `tutor config save --interactive` or `tutor local " + "quickstart` prior to running other commands." + ) + env.check_is_up_to_date(root) + convert_json2yml(root) + return load_full(root) -def load_no_check(root: str) -> Config: - config, defaults = load_all(root) - merge(config, defaults) +def load_minimal(root: str) -> Config: + """ + Load a minimal configuration composed of the user and the base config. + + This configuration is not suitable for rendering templates, as it is incomplete. + """ + config = get_user(root) + update_with_base(config) + render_full(config) return config -def load_all(root: str) -> Tuple[Config, Config]: +def load_full(root: str) -> Config: """ - Return: - current (dict): params currently saved in config.yml - defaults (dict): default values of params which might be missing from the - current config + Load a full configuration, with user, base and defaults. """ - defaults = load_defaults() - current = load_current(root, defaults) - return current, defaults + config = get_user(root) + update_with_base(config) + update_with_defaults(config) + render_full(config) + return config -def merge(config: Config, defaults: Config, force: bool = False) -> None: +def update_with_base(config: Config) -> None: """ - Merge default values with user configuration and perform rendering of "{{...}}" - values. + Add base configuration to the config object. + + Note that configuration entries are unrendered at this point. """ - for key, value in defaults.items(): - if force or key not in config: - config[key] = env.render_unknown(config, value) + base = get_base(config) + merge(config, base) -def load_defaults() -> Config: - config = serialize.load(env.read_template_file("config.yml")) +def update_with_defaults(config: Config) -> None: + """ + Add default configuration to the config object. + + Note that configuration entries are unrendered at this point. + """ + defaults = get_defaults(config) + merge(config, defaults) + + +def update_with_env(config: Config) -> None: + """ + Override config values from environment variables. + """ + overrides = {} + for k in config.keys(): + env_var = "TUTOR_" + k + if env_var in os.environ: + overrides[k] = serialize.parse(os.environ[env_var]) + config.update(overrides) + + +def get_user(root: str) -> Config: + """ + Get the user configuration from the tutor root. + + Overrides from environment variables are loaded as well. + """ + path = config_path(root) + config = {} + if os.path.exists(path): + config = get_yaml_file(path) + upgrade_obsolete(config) + update_with_env(config) + return config + + +def get_base(config: Config) -> Config: + """ + Load the base configuration. + + Entries in this configuration are unrendered. + """ + base = get_template("base.yml") + + # Load base values from plugins + for plugin in plugins.iter_enabled(config): + # Add new config key/values + for key, value in plugin.config_add.items(): + new_key = plugin.config_key(key) + base[new_key] = value + + # Set existing config key/values + for key, value in plugin.config_set.items(): + base[key] = value + + return base + + +def get_defaults(config: Config) -> Config: + """ + Get default configuration, including from plugins. + + Entries in this configuration are unrendered. + """ + defaults = get_template("defaults.yml") + + for plugin in plugins.iter_enabled(config): + # Create new defaults + for key, value in plugin.config_defaults.items(): + defaults[plugin.config_key(key)] = value + + update_with_env(defaults) + return defaults + + +def get_template(filename: str) -> Config: + """ + Get one of the configuration templates. + + Entries in this configuration are unrendered. + """ + config = serialize.load(env.read_template_file("config", filename)) return cast_config(config) -def load_config_file(path: str) -> Config: +def get_yaml_file(path: str) -> Config: + """ + Load config from yaml file. + """ with open(path) as f: config = serialize.load(f.read()) return cast_config(config) -def load_current(root: str, defaults: Config) -> Config: +def merge(config: Config, base: Config) -> None: """ - Load the configuration currently stored on disk. - Note: this modifies the defaults with the plugin default values. + Merge base values with user configuration. Values are only added if not + already present. + + Note that this function does not perform the rendering step of the + configuration entries. """ - convert_json2yml(root) - config = load_user(root) - load_env(config, defaults) - load_required(config, defaults) - load_plugins(config, defaults) - return config - - -def load_user(root: str) -> Config: - path = config_path(root) - if not os.path.exists(path): - return {} - - config = load_config_file(path) - upgrade_obsolete(config) - return config - - -def load_env(config: Config, defaults: Config) -> None: - for k in defaults.keys(): - env_var = "TUTOR_" + k - if env_var in os.environ: - config[k] = serialize.parse(os.environ[env_var]) - - -def load_required(config: Config, defaults: Config) -> None: - """ - All these keys must be present in the user's config.yml. This includes all values - that are generated once and must be kept after that, such as passwords. - """ - for key in [ - "OPENEDX_SECRET_KEY", - "MYSQL_ROOT_PASSWORD", - "OPENEDX_MYSQL_PASSWORD", - "ID", - "JWT_RSA_PRIVATE_KEY", - ]: + for key, value in base.items(): if key not in config: - config[key] = env.render_unknown(config, defaults[key]) + config[key] = value -def load_plugins(config: Config, defaults: Config) -> None: +def render_full(config: Config) -> None: """ - Add, override and set new defaults from plugins. + Fill and render an existing configuration with defaults. + + It is generally necessary to apply this function before rendering templates, + otherwise configuration entries may not be rendered. """ - for plugin in plugins.iter_enabled(config): - # Add new config key/values - for key, value in plugin.config_add.items(): - new_key = plugin.config_key(key) - if new_key not in config: - config[new_key] = env.render_unknown(config, value) - - # Create new defaults - for key, value in plugin.config_defaults.items(): - defaults[plugin.config_key(key)] = value - - # Set existing config key/values: here, we do not override existing values - # This must come last, as overridden values may depend on plugin defaults - for key, value in plugin.config_set.items(): - if key not in config: - config[key] = env.render_unknown(config, value) + for key, value in config.items(): + config[key] = env.render_unknown(config, value) def is_service_activated(config: Config, service: str) -> bool: @@ -194,7 +238,7 @@ def convert_json2yml(root: str) -> None: root ) ) - config = load_config_file(json_path) + config = get_yaml_file(json_path) save_config_file(root, config) os.remove(json_path) fmt.echo_info( @@ -210,18 +254,5 @@ def save_config_file(root: str, config: Config) -> None: fmt.echo_info("Configuration saved to {}".format(path)) -def check_existing_config(root: str) -> None: - """ - Check there is a configuration on disk and the current environment is up-to-date. - """ - if not os.path.exists(config_path(root)): - raise exceptions.TutorError( - "Project root does not exist. Make sure to generate the initial " - "configuration with `tutor config save --interactive` or `tutor local " - "quickstart` prior to running other commands." - ) - env.check_is_up_to_date(root) - - def config_path(root: str) -> str: return os.path.join(root, "config.yml") diff --git a/tutor/interactive.py b/tutor/interactive.py index 048e10d..25ba3d9 100644 --- a/tutor/interactive.py +++ b/tutor/interactive.py @@ -1,34 +1,24 @@ -from typing import List, Tuple +from typing import List import click from . import config as tutor_config from . import env, exceptions, fmt -from .__about__ import __version__ from .types import Config, get_typed -def update(root: str, interactive: bool = True) -> Config: - """ - Load and save the configuration. - """ - config, defaults = load_all(root, interactive=interactive) - tutor_config.save_config_file(root, config) - tutor_config.merge(config, defaults) - return config - - -def load_all(root: str, interactive: bool = True) -> Tuple[Config, Config]: +def load_user_config(root: str, interactive: bool = True) -> Config: """ Load configuration and interactively ask questions to collect param values from the user. """ - config, defaults = tutor_config.load_all(root) + config = tutor_config.load_minimal(root) if interactive: - ask_questions(config, defaults) - return config, defaults + ask_questions(config) + return config -def ask_questions(config: Config, defaults: Config) -> None: +def ask_questions(config: Config) -> None: + defaults = tutor_config.get_defaults(config) run_for_prod = config.get("LMS_HOST") != "local.overhang.io" run_for_prod = click.confirm( fmt.question( diff --git a/tutor/templates/config/base.yml b/tutor/templates/config/base.yml new file mode 100644 index 0000000..59a1c98 --- /dev/null +++ b/tutor/templates/config/base.yml @@ -0,0 +1,6 @@ +--- +ID: "{{ 24|random_string }}" +JWT_RSA_PRIVATE_KEY: "{{ 2048|rsa_private_key }}" +MYSQL_ROOT_PASSWORD: "{{ 8|random_string }}" +OPENEDX_MYSQL_PASSWORD: "{{ 8|random_string }}" +OPENEDX_SECRET_KEY: "{{ 24|random_string }}" diff --git a/tutor/templates/config.yml b/tutor/templates/config/defaults.yml similarity index 86% rename from tutor/templates/config.yml rename to tutor/templates/config/defaults.yml index f2ee6ce..fd31c00 100644 --- a/tutor/templates/config.yml +++ b/tutor/templates/config/defaults.yml @@ -1,27 +1,10 @@ --- -# These configuration values must be stored in the user's config.yml. -MYSQL_ROOT_PASSWORD: "{{ 8|random_string }}" -OPENEDX_MYSQL_PASSWORD: "{{ 8|random_string }}" -OPENEDX_SECRET_KEY: "{{ 24|random_string }}" -ID: "{{ 24|random_string }}" - +# This file includes all Tutor setting defaults. Settings that do not have a +# default value, such as passwords, should be stored in base.yml. # This must be defined early -LMS_HOST: "www.myopenedx.com" - -# The following are default values -RUN_LMS: true -RUN_CMS: true -RUN_ELASTICSEARCH: true -RUN_MONGODB: true -RUN_MYSQL: true -RUN_REDIS: true -RUN_SMTP: true CADDY_HTTP_PORT: 80 CMS_HOST: "studio.{{ LMS_HOST }}" -PREVIEW_LMS_HOST: "preview.{{ LMS_HOST }}" CONTACT_EMAIL: "contact@{{ LMS_HOST }}" -OPENEDX_AWS_ACCESS_KEY: "" -OPENEDX_AWS_SECRET_ACCESS_KEY: "" DEV_PROJECT_NAME: "{{ TUTOR_APP }}_dev" DOCKER_REGISTRY: "docker.io/" DOCKER_IMAGE_OPENEDX: "{{ DOCKER_REGISTRY }}overhangio/openedx:{{ TUTOR_VERSION }}" @@ -35,7 +18,6 @@ DOCKER_IMAGE_NGINX: "{{ DOCKER_REGISTRY }}nginx:1.21.1" DOCKER_IMAGE_PERMISSIONS: "{{ DOCKER_REGISTRY }}overhangio/openedx-permissions:{{ TUTOR_VERSION }}" DOCKER_IMAGE_REDIS: "{{ DOCKER_REGISTRY }}redis:6.2.6" DOCKER_IMAGE_SMTP: "{{ DOCKER_REGISTRY }}devture/exim-relay:4.94.2-r0-4" -LOCAL_PROJECT_NAME: "{{ TUTOR_APP }}_local" ELASTICSEARCH_HOST: "elasticsearch" ELASTICSEARCH_PORT: 9200 ELASTICSEARCH_SCHEME: "http" @@ -45,14 +27,17 @@ ENABLE_WEB_PROXY: true JWT_COMMON_AUDIENCE: "openedx" JWT_COMMON_ISSUER: "{% if ENABLE_HTTPS %}https{% else %}http{% endif %}://{{ LMS_HOST }}/oauth2" JWT_COMMON_SECRET_KEY: "{{ OPENEDX_SECRET_KEY }}" -JWT_RSA_PRIVATE_KEY: "{{ 2048|rsa_private_key }}" K8S_NAMESPACE: "openedx" LANGUAGE_CODE: "en" +LMS_HOST: "www.myopenedx.com" +LOCAL_PROJECT_NAME: "{{ TUTOR_APP }}_local" MONGODB_HOST: "mongodb" MONGODB_DATABASE: "openedx" MONGODB_PORT: 27017 MONGODB_USERNAME: "" MONGODB_PASSWORD: "" +OPENEDX_AWS_ACCESS_KEY: "" +OPENEDX_AWS_SECRET_ACCESS_KEY: "" OPENEDX_CACHE_REDIS_DB: 1 OPENEDX_CELERY_REDIS_DB: 0 OPENEDX_CMS_UWSGI_WORKERS: 2 @@ -73,6 +58,13 @@ REDIS_HOST: "redis" REDIS_PORT: 6379 REDIS_USERNAME: "" REDIS_PASSWORD: "" +RUN_CMS: true +RUN_ELASTICSEARCH: true +RUN_LMS: true +RUN_MONGODB: true +RUN_MYSQL: true +RUN_REDIS: true +RUN_SMTP: true SMTP_HOST: "smtp" SMTP_PORT: 8025 SMTP_USERNAME: "" From 31eab76632773224e481d1ec784d45b45b673434 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Wed, 8 Dec 2021 11:20:41 +0100 Subject: [PATCH 22/56] fix: breaking "config save" "config save" was not loading the full configuration prior to saving the environment. --- tutor/commands/config.py | 3 ++- tutor/config.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tutor/commands/config.py b/tutor/commands/config.py index 964813d..177a0a3 100644 --- a/tutor/commands/config.py +++ b/tutor/commands/config.py @@ -59,7 +59,8 @@ def save( if not env_only: tutor_config.save_config_file(context.root, config) - tutor_config.render_full(config) + # Reload configuration, without version checking + config = tutor_config.load_full(context.root) env.save(context.root, config) diff --git a/tutor/config.py b/tutor/config.py index 4973d24..edc9414 100644 --- a/tutor/config.py +++ b/tutor/config.py @@ -19,7 +19,6 @@ def load(root: str) -> Config: "quickstart` prior to running other commands." ) env.check_is_up_to_date(root) - convert_json2yml(root) return load_full(root) @@ -39,6 +38,7 @@ def load_full(root: str) -> Config: """ Load a full configuration, with user, base and defaults. """ + convert_json2yml(root) config = get_user(root) update_with_base(config) update_with_defaults(config) From 3ad68ab782ceeb6ffb8d313d58302ebdcd3e05b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Wed, 8 Dec 2021 13:07:24 +0100 Subject: [PATCH 23/56] fix: do not save the full config on "plugins en/disable" This was saving the full configuration to config.yml, resulting in many incorrect configuration values... --- tutor/commands/plugins.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tutor/commands/plugins.py b/tutor/commands/plugins.py index 6fe1fad..fab677e 100644 --- a/tutor/commands/plugins.py +++ b/tutor/commands/plugins.py @@ -42,7 +42,7 @@ def list_command(context: Context) -> None: @click.argument("plugin_names", metavar="plugin", nargs=-1) @click.pass_obj def enable(context: Context, plugin_names: List[str]) -> None: - config = tutor_config.load_full(context.root) + config = tutor_config.load_minimal(context.root) for plugin in plugin_names: plugins.enable(config, plugin) fmt.echo_info("Plugin {} enabled".format(plugin)) @@ -59,7 +59,7 @@ def enable(context: Context, plugin_names: List[str]) -> None: @click.argument("plugin_names", metavar="plugin", nargs=-1) @click.pass_obj def disable(context: Context, plugin_names: List[str]) -> None: - config = tutor_config.load_full(context.root) + config = tutor_config.load_minimal(context.root) disable_all = "all" in plugin_names for plugin in plugins.iter_enabled(config): if disable_all or plugin.name in plugin_names: From e5b63604de2c92e0f53532a96a30afbd5ae02925 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Mon, 29 Nov 2021 22:17:12 +0100 Subject: [PATCH 24/56] security: convert NodePort to ClusterIP for better isolation On some providers (notably: DigitalOcean) NodePort services are not exposed to the outside world. But this is not what the Kubernetes spec describes: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types Thus, there is a risk that NodePort services are exposed to the outside world in some context. To avoid this, we convert all NodePort to ClusterIP resources. --- CHANGELOG-nightly.md | 1 + tutor/templates/k8s/services.yml | 14 +++++++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/CHANGELOG-nightly.md b/CHANGELOG-nightly.md index 665d430..d78f0e7 100644 --- a/CHANGELOG-nightly.md +++ b/CHANGELOG-nightly.md @@ -2,6 +2,7 @@ Note: Breaking changes between versions are indicated by "💥". +- [Security] On Kubernetes, convert all NodePort services to ClusterIP to guarantee network isolation from outside the cluster. - 💥[Improvement] Drop Python 3.5 compatibility. - [Bugfix] Fix docker-compose project name in development on nightly branch. - 💥[Bugfix] No longer track the Tutor version number in resource labels (and label selectors, which breaks the update of Deployment resources), but instead do so in resource annotations. diff --git a/tutor/templates/k8s/services.yml b/tutor/templates/k8s/services.yml index b768098..2e00b9b 100644 --- a/tutor/templates/k8s/services.yml +++ b/tutor/templates/k8s/services.yml @@ -21,7 +21,7 @@ kind: Service metadata: name: cms spec: - type: NodePort + type: ClusterIP ports: - port: 8000 protocol: TCP @@ -35,7 +35,7 @@ kind: Service metadata: name: lms spec: - type: NodePort + type: ClusterIP ports: - port: 8000 protocol: TCP @@ -49,7 +49,7 @@ kind: Service metadata: name: elasticsearch spec: - type: NodePort + type: ClusterIP ports: - port: 9200 protocol: TCP @@ -63,7 +63,7 @@ kind: Service metadata: name: mongodb spec: - type: NodePort + type: ClusterIP ports: - port: 27017 protocol: TCP @@ -77,7 +77,7 @@ kind: Service metadata: name: mysql spec: - type: NodePort + type: ClusterIP ports: - port: 3306 protocol: TCP @@ -91,7 +91,7 @@ kind: Service metadata: name: redis spec: - type: NodePort + type: ClusterIP ports: - port: {{ REDIS_PORT }} protocol: TCP @@ -105,7 +105,7 @@ kind: Service metadata: name: smtp spec: - type: NodePort + type: ClusterIP ports: - port: 8025 protocol: TCP From 39ca60f168f19529b3df2ca7016f919d2f4fcbca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Mon, 29 Nov 2021 21:55:13 +0100 Subject: [PATCH 25/56] feat: better support of caddy as a k8s load balancer This introduces quite a few changes to make it easier to run Caddy as a load balancer in Kubernetes: - Make it possible to start/stop a selection of resources with ``tutor k8s start/stop [names...]``. - Make it easy to deploy an independent LoadBalancer by converting the caddy service to a NodePort when ``ENABLE_WEB_PROXY=false``. - Add a ``app.kubernetes.io/component: loadbalancer`` label to the LoadBalancer service. - Add ``app.kubernetes.io/name`` labels to all services. - Preserve the LoadBalancer service in ``tutor k8s stop`` commands. - Wait for the caddy deployment to be ready before running initialisation jobs. Close #532. --- CHANGELOG-nightly.md | 7 +++ docs/k8s.rst | 18 +++++- tutor/commands/k8s.py | 99 ++++++++++++++++++++++---------- tutor/templates/k8s/services.yml | 34 +++++++++++ 4 files changed, 126 insertions(+), 32 deletions(-) diff --git a/CHANGELOG-nightly.md b/CHANGELOG-nightly.md index d78f0e7..1bca6d2 100644 --- a/CHANGELOG-nightly.md +++ b/CHANGELOG-nightly.md @@ -2,6 +2,13 @@ Note: Breaking changes between versions are indicated by "💥". +- [Feature] Better support of Caddy as a load balancer in Kubernetes: + - Make it possible to start/stop a selection of resources with ``tutor k8s start/stop [names...]``. + - Make it easy to deploy an independent LoadBalancer by converting the caddy service to a ClusterIP when ``ENABLE_WEB_PROXY=false``. + - Add a ``app.kubernetes.io/component: loadbalancer`` label to the LoadBalancer service. + - Add ``app.kubernetes.io/name`` labels to all services. + - Preserve the LoadBalancer service in ``tutor k8s stop`` commands. + - Wait for the caddy deployment to be ready before running initialisation jobs. - [Security] On Kubernetes, convert all NodePort services to ClusterIP to guarantee network isolation from outside the cluster. - 💥[Improvement] Drop Python 3.5 compatibility. - [Bugfix] Fix docker-compose project name in development on nightly branch. diff --git a/docs/k8s.rst b/docs/k8s.rst index b44a66f..0316b0b 100644 --- a/docs/k8s.rst +++ b/docs/k8s.rst @@ -27,10 +27,22 @@ The Kubernetes cluster should have at least 4Gb of RAM on each node. When runnin .. image:: img/virtualbox-minikube-system.png :alt: Virtualbox memory settings for Minikube -Ingress controller and SSL/TLS certificates -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Load Balancer and SSL/TLS certificates +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -As of Tutor v11, it is no longer required to setup an Ingress controller to access your platform. Instead Caddy exposes a LoadBalancer service and SSL/TLS certificates are transparently generated at runtime. +By default, Tutor deploys a `LoadBalancer `__ service that exposes the Caddy deployment to the outside world. As in the local installation, this service is responsible for transparently generating SSL/TLS certificates at runtime. You will need to point your DNS records to this LoadBalancer object before the platform can work correctly. Thus, you should first start the Caddy load balancer, with:: + + tutor k8s start caddy + +Get the external IP of this services:: + + kubectl --namespace openedx get services/caddy + +Use this external IP to configure your DNS records. Once the DNS records are configured, you should verify that the Caddy container has properly generated the SSL/TLS certificates by checking the container logs:: + + tutor k8s logs -f caddy + +If, for some reason, you would like to deploy your own load balancer, you should set ``ENABLE_WEB_PROXY=false`` just like in the :ref:`local installation `. Then, point your load balancer at the "caddy" service, which will be a `ClusterIP `__. S3-like object storage with `MinIO `_ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/tutor/commands/k8s.py b/tutor/commands/k8s.py index 18443bf..9613529 100644 --- a/tutor/commands/k8s.py +++ b/tutor/commands/k8s.py @@ -198,9 +198,16 @@ def quickstart(context: click.Context, non_interactive: bool) -> None: ) -@click.command(help="Run all configured Open edX services") +@click.command( + short_help="Run all configured Open edX resources", + help=( + "Run all configured Open edX resources. You may limit this command to " + "some resources by passing name arguments." + ), +) +@click.argument("names", metavar="name", nargs=-1) @click.pass_obj -def start(context: Context) -> None: +def start(context: Context, names: List[str]) -> None: config = tutor_config.load(context.root) # Create namespace, if necessary # Note that this step should not be run for some users, in particular those @@ -218,34 +225,68 @@ def start(context: Context) -> None: "--selector", "app.kubernetes.io/component=namespace", ) - # Create volumes - utils.kubectl( - "apply", - "--kustomize", - tutor_env.pathjoin(context.root), - "--wait", - "--selector", - "app.kubernetes.io/component=volume", - ) - # Create everything else except jobs - utils.kubectl( - "apply", - "--kustomize", - tutor_env.pathjoin(context.root), - "--selector", - "app.kubernetes.io/component notin (job,volume,namespace)", - ) + + names = names or ["all"] + for name in names: + if name == "all": + # Create volumes + utils.kubectl( + "apply", + "--kustomize", + tutor_env.pathjoin(context.root), + "--wait", + "--selector", + "app.kubernetes.io/component=volume", + ) + # Create everything else except jobs + utils.kubectl( + "apply", + "--kustomize", + tutor_env.pathjoin(context.root), + "--selector", + "app.kubernetes.io/component notin (job,volume,namespace)", + ) + else: + utils.kubectl( + "apply", + "--kustomize", + tutor_env.pathjoin(context.root), + "--selector", + "app.kubernetes.io/name={}".format(name), + ) -@click.command(help="Stop a running platform") +@click.command( + short_help="Stop a running platform", + help=( + "Stop a running platform by deleting all resources, except for volumes. " + "You may limit this command to some resources by passing name arguments." + ), +) +@click.argument("names", metavar="name", nargs=-1) @click.pass_obj -def stop(context: Context) -> None: +def stop(context: Context, names: List[str]) -> None: config = tutor_config.load(context.root) - utils.kubectl( - "delete", - *resource_selector(config), - "deployments,services,configmaps,jobs", - ) + names = names or ["all"] + resource_types = "deployments,services,configmaps,jobs" + not_lb_selector = "app.kubernetes.io/component!=loadbalancer" + for name in names: + if name == "all": + utils.kubectl( + "delete", + *resource_selector(config, not_lb_selector), + resource_types, + ) + else: + utils.kubectl( + "delete", + *resource_selector( + config, + not_lb_selector, + "app.kubernetes.io/name={}".format(name), + ), + resource_types, + ) @click.command(help="Reboot an existing platform") @@ -290,9 +331,9 @@ def delete(context: Context, yes: bool) -> None: def init(context: Context, limit: Optional[str]) -> None: config = tutor_config.load(context.root) runner = K8sJobRunner(context.root, config) - for service in ["mysql", "elasticsearch", "mongodb"]: - if tutor_config.is_service_activated(config, service): - wait_for_pod_ready(config, service) + for name in ["caddy", "elasticsearch", "mysql", "mongodb"]: + if tutor_config.is_service_activated(config, name): + wait_for_pod_ready(config, name) jobs.initialise(runner, limit_to=limit) diff --git a/tutor/templates/k8s/services.yml b/tutor/templates/k8s/services.yml index 2e00b9b..180032b 100644 --- a/tutor/templates/k8s/services.yml +++ b/tutor/templates/k8s/services.yml @@ -4,13 +4,33 @@ apiVersion: v1 kind: Service metadata: name: caddy + labels: + app.kubernetes.io/name: caddy + app.kubernetes.io/component: loadbalancer spec: type: LoadBalancer ports: - port: 80 name: http + {%- if ENABLE_HTTPS %} - port: 443 name: https + {%- endif %} + selector: + app.kubernetes.io/name: caddy +{% else %} +--- +apiVersion: v1 +kind: Service +metadata: + name: caddy + labels: + app.kubernetes.io/name: caddy +spec: + type: ClusterIP + ports: + - port: {{ CADDY_HTTP_PORT }} + name: http selector: app.kubernetes.io/name: caddy {% endif %} @@ -20,6 +40,8 @@ apiVersion: v1 kind: Service metadata: name: cms + labels: + app.kubernetes.io/name: cms spec: type: ClusterIP ports: @@ -34,6 +56,8 @@ apiVersion: v1 kind: Service metadata: name: lms + labels: + app.kubernetes.io/name: lms spec: type: ClusterIP ports: @@ -48,6 +72,8 @@ apiVersion: v1 kind: Service metadata: name: elasticsearch + labels: + app.kubernetes.io/name: elasticsearch spec: type: ClusterIP ports: @@ -62,6 +88,8 @@ apiVersion: v1 kind: Service metadata: name: mongodb + labels: + app.kubernetes.io/name: mongodb spec: type: ClusterIP ports: @@ -76,6 +104,8 @@ apiVersion: v1 kind: Service metadata: name: mysql + labels: + app.kubernetes.io/name: mysql spec: type: ClusterIP ports: @@ -90,6 +120,8 @@ apiVersion: v1 kind: Service metadata: name: redis + labels: + app.kubernetes.io/name: redis spec: type: ClusterIP ports: @@ -104,6 +136,8 @@ apiVersion: v1 kind: Service metadata: name: smtp + labels: + app.kubernetes.io/name: smtp spec: type: ClusterIP ports: From 1fe311d351059211be43335d000b7692666548b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Tue, 30 Nov 2021 18:02:14 +0100 Subject: [PATCH 26/56] feat: `k8s scale` command + tutorial - Add a `tutor k8s scale lms 11` command - Create a "Running Open edX at scale" tutorial --- docs/configuration.rst | 2 + docs/tutorials.rst | 1 + docs/tutorials/scale.rst | 85 ++++++++++++++++++++++++++++++++++++++++ tutor/commands/k8s.py | 48 +++++++++++++++++------ 4 files changed, 125 insertions(+), 11 deletions(-) create mode 100644 docs/tutorials/scale.rst diff --git a/docs/configuration.rst b/docs/configuration.rst index abbd235..55369ec 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -74,6 +74,8 @@ You may want to pull/push images from/to a custom docker registry. For instance, (the trailing ``/`` is important) +.. _openedx_configuration: + Open edX customisation ~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/tutorials.rst b/docs/tutorials.rst index 0bd5103..5f9dfea 100644 --- a/docs/tutorials.rst +++ b/docs/tutorials.rst @@ -18,6 +18,7 @@ System administration .. toctree:: :maxdepth: 2 + tutorials/scale tutorials/portainer tutorials/podman tutorials/proxy diff --git a/docs/tutorials/scale.rst b/docs/tutorials/scale.rst new file mode 100644 index 0000000..3645168 --- /dev/null +++ b/docs/tutorials/scale.rst @@ -0,0 +1,85 @@ +.. _scale: + +Running Open edX at scale +========================= + +Does Open edX scale? This is the $10⁶ question when it comes to Tutor and Open edX deployments. The short answer is "yes". The longer answer is also "yes", but the details will very much depend on what we mean by "scaling". + +Depending on the context, "scaling" can imply different things: + +1. `Vertical scaling `__: increasing platform capacity by allocating more resources to a single server. +2. `Horizontal scaling `__: the ability to serve an infinitely increasing number of users with consistent performance and linear costs. +3. `High availability (HA) `__: the ability of the platform to remain fully functional despite one or more components being unavailable. + +All of these can be achieved with Tutor and Open edX, but the method to attain either differs greatly. First of all, the range of available solutions will depend on which deployment target is used. Tutor supports installations of Open edX on a single server with the :ref:`"local" ` deployment target, where Docker containers are orchestrated by docker-compose. On a single server, by definition, the server is a single point of failure (`SPOF `__). Thus, high availability is out of the question with a single server. To achieve high availability, it is necessary to deploy to a cluster of multiple servers. But while docker-compose is a great tool for managing single-server deployments, it is simply inappropriate for deploying to a cluster. Tutor also supports deploying to a Kubernetes cluster (see :ref:`k8s`). This is the recommended solution to deploy Open edX "at scale". + +Scaling with a single server +---------------------------- + +Options are limited when it comes to scaling an Open edX platform deployed on a single-server. High availability is out of the question and the number of users that your platform can serve simultaneously will be limited by the server capacity. + +Fortunately, Open edX was designed to run at scale -- most notably at `edX.org `__, but also on large national education platforms. Thus, performance will not be limited by the backend software, but only by the hardware. + +Increasing web server capacity +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +As the server CPU and memory are increased, the request throughput can be increased by adjusting the number of uWSGI workers (see :ref:`configuration docs `). By default, the "lms" and "cms" containers each spawn 2 uWSGI workers. The number of workers should be increased if you observe an increase of the latency of user requests but CPU usage remains below 100%. To increase the number of workers for the LMS and the CMS, run for example:: + + tutor config save \ + --set OPENEDX_LMS_UWSGI_WORKERS=8 \ + --set OPENEDX_CMS_UWSGI_WORKERS=4 + tutor local restart lms cms + +The right values will very much depend on your server available memory and CPU performance, as well as the maximum number of simultaneous users who use your platform. As an example data point, it was reported that a large Open edX platform can serve up to 500k unique users per week on a virtual server with 8 vCPU and 16 GB memory. + +Offloading data storage +~~~~~~~~~~~~~~~~~~~~~~~ + +Aside from web workers, the most resource-intensive services are in the data persistence layer. They are, by decreasing resource usage: + +- `Elasticsearch `__: indexing of course contents and forum topics, mostly for search. Elasticsearch is never a source of truth in Open edX, and the data can thus be trashed and re-created safely. +- `MySQL `__: structured, consistent data storage which is the default destination of all data. +- `Mongodb `__: structured storage of course data. +- `Redis `__: caching and asynchronous task management. +- `MinIO `__: S3-like object storage for user-uploaded files, which is enabled by the `tutor-minio `__ plugin. It is possible to replace MinIO by direct filesystem storage (the default), but scaling will then become much more difficult down the road. + +When attempting to scale a single-server deployment, we recommend to start by offloading some of these stateful data storage components, in the same order of priority. There are multiple benefits: + +1. It will free up some resource both for the web workers and the data storage components. +2. It is a first step towards horizontal scaling of the web workers. +3. It becomes possible to either install every component as a separate service or rely on 3rd-party SaaS with high availability. + +Moving each of the data storage components is a fairly straightforward process, although details vary for every component. For instance, for the MySQL database, start by disabling the locally running MySQL instance:: + + tutor config save --set RUN_MYSQL=false + +Then, migrate the data located at ``$(tutor config printroot)/data/mysql`` to the new MySQL instance. Configure the Open edX platform to point at the new database:: + + tutor config save \ + --set MYSQL_HOST=yourdb.com \ + --set MYSQL_PORT=3306 \ + --set MYSQL_ROOT_USERNAME=root \ + --set MYSQL_ROOT_PASSWORD=p4ssw0rd + +The changes will be taken into account the next time the platform is restarted. + +Beware that moving the data components to dedicated servers has the potential of creating new single points of failures (`SPOF `__). To avoid this situation, each component should be installed as a highly available service (or as highly available SaaS). + +Scaling with multiple servers +----------------------------- + +Horizontally scaling web services +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +As the number of users of a web platform increases, they put increased pressure on the web workers that respond to their requests. Thus, in most cases, web worker performance is the first bottleneck that system administrators have to face when their service becomes more popular. Initially, any given Kubernetes-based Tutor platform ships with one replica for each deployment. To increase (or reduce) the number of replicas for any given service, run ``tutor k8s scale ``. Behind the scenes, this command will trigger a ``kubectl scale --replicas=...`` command that will seamlessly increase the number of pods for that deployment. + +In Open edX there are multiple web services that are exposed to the outside world. The ones that usually receive the most traffic are, in decreasing order, the LMS, the CMS and the forum (assuming the `tutor-forum `__ plugin was enabled). As an example, all three deployment replicas can be scaled by running:: + + tutor k8s scale lms 8 + tutor k8s scale cms 4 + tutor k8s scale forum 2 + +Highly-available architecture, autoscaling, ... +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +There is only so much that Tutor can do for you, and scaling some components falls beyond the scope of Tutor. For instance, it is your responsibility to make sure that your Kubernetes cluster has a `highly available control plane `__ and `topology `__. Also, it is possible to achieve `autoscaling `__; but it is your responsibility to setup latency metrics collection and to configure the scaling policies. diff --git a/tutor/commands/k8s.py b/tutor/commands/k8s.py index 9613529..166016f 100644 --- a/tutor/commands/k8s.py +++ b/tutor/commands/k8s.py @@ -296,17 +296,6 @@ def reboot(context: click.Context) -> None: context.invoke(start) -def resource_selector(config: Config, *selectors: str) -> List[str]: - """ - Convenient utility for filtering only the resources that belong to this project. - """ - selector = ",".join( - ["app.kubernetes.io/instance=openedx-" + get_typed(config, "ID", str)] - + list(selectors) - ) - return ["--namespace", k8s_namespace(config), "--selector=" + selector] - - @click.command(help="Completely delete an existing platform") @click.option("-y", "--yes", is_flag=True, help="Do not ask for confirmation") @click.pass_obj @@ -337,6 +326,24 @@ def init(context: Context, limit: Optional[str]) -> None: jobs.initialise(runner, limit_to=limit) +@click.command(help="Scale the number of replicas of a given deployment") +@click.argument("deployment") +@click.argument("replicas", type=int) +@click.pass_obj +def scale(context: Context, deployment: str, replicas: int) -> None: + config = tutor_config.load(context.root) + utils.kubectl( + "scale", + # Note that we don't use the full resource selector because selectors + # are not compatible with the deployment/ argument. + *resource_namespace_selector( + config, + ), + "--replicas={}".format(replicas), + "deployment/{}".format(deployment), + ) + + @click.command(help="Create an Open edX user and interactively set their password") @click.option("--superuser", is_flag=True, help="Make superuser") @click.option("--staff", is_flag=True, help="Make staff user") @@ -562,6 +569,24 @@ def wait_for_pod_ready(config: Config, service: str) -> None: ) +def resource_selector(config: Config, *selectors: str) -> List[str]: + """ + Convenient utility to filter the resources that belong to this project. + """ + selector = ",".join( + ["app.kubernetes.io/instance=openedx-" + get_typed(config, "ID", str)] + + list(selectors) + ) + return resource_namespace_selector(config) + ["--selector=" + selector] + + +def resource_namespace_selector(config: Config) -> List[str]: + """ + Convenient utility to filter the resources that belong to this project namespace. + """ + return ["--namespace", k8s_namespace(config)] + + def k8s_namespace(config: Config) -> str: return get_typed(config, "K8S_NAMESPACE", str) @@ -572,6 +597,7 @@ k8s.add_command(stop) k8s.add_command(reboot) k8s.add_command(delete) k8s.add_command(init) +k8s.add_command(scale) k8s.add_command(createuser) k8s.add_command(importdemocourse) k8s.add_command(settheme) From bf40724dd71d2fa6be253a5fdc301f13152cf3be Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Wed, 15 Dec 2021 11:33:48 -0800 Subject: [PATCH 27/56] fix: a migration on nightly/master open edx now requires MongoDB --- CHANGELOG-nightly.md | 1 + tutor/templates/hooks/lms/init | 1 + tutor/templates/local/docker-compose.jobs.yml | 4 ++-- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG-nightly.md b/CHANGELOG-nightly.md index 1bca6d2..a6181c0 100644 --- a/CHANGELOG-nightly.md +++ b/CHANGELOG-nightly.md @@ -2,6 +2,7 @@ Note: Breaking changes between versions are indicated by "💥". +- [Bugfix] Start MongoDB when running migrations, because a new data migration fails if MongoDB is not running - [Feature] Better support of Caddy as a load balancer in Kubernetes: - Make it possible to start/stop a selection of resources with ``tutor k8s start/stop [names...]``. - Make it easy to deploy an independent LoadBalancer by converting the caddy service to a ClusterIP when ``ENABLE_WEB_PROXY=false``. diff --git a/tutor/templates/hooks/lms/init b/tutor/templates/hooks/lms/init index 3b25b85..b6b332e 100644 --- a/tutor/templates/hooks/lms/init +++ b/tutor/templates/hooks/lms/init @@ -1,4 +1,5 @@ dockerize -wait tcp://{{ MYSQL_HOST }}:{{ MYSQL_PORT }} -timeout 20s +dockerize -wait tcp://{{ MONGODB_HOST }}:{{ MONGODB_PORT }} -timeout 20s echo "Loading settings $DJANGO_SETTINGS_MODULE" diff --git a/tutor/templates/local/docker-compose.jobs.yml b/tutor/templates/local/docker-compose.jobs.yml index 01fb809..a55cd13 100644 --- a/tutor/templates/local/docker-compose.jobs.yml +++ b/tutor/templates/local/docker-compose.jobs.yml @@ -14,7 +14,7 @@ services: - ../apps/openedx/settings/lms:/openedx/edx-platform/lms/envs/tutor:ro - ../apps/openedx/settings/cms:/openedx/edx-platform/cms/envs/tutor:ro - ../apps/openedx/config:/openedx/config:ro - depends_on: {{ [("mysql", RUN_MYSQL)]|list_if }} + depends_on: {{ [("mysql", RUN_MYSQL), ("mongodb", RUN_MONGODB)]|list_if }} cms-job: image: {{ DOCKER_IMAGE_OPENEDX }} @@ -25,6 +25,6 @@ services: - ../apps/openedx/settings/lms:/openedx/edx-platform/lms/envs/tutor:ro - ../apps/openedx/settings/cms:/openedx/edx-platform/cms/envs/tutor:ro - ../apps/openedx/config:/openedx/config:ro - depends_on: {{ [("mysql", RUN_MYSQL)]|list_if }} + depends_on: {{ [("mysql", RUN_MYSQL), ("mongodb", RUN_MONGODB)]|list_if }} {{ patch("local-docker-compose-jobs-services")|indent(4) }} From 71b4c14d6919436e80421ee9678bbdb709e2051d Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Tue, 11 Jan 2022 10:40:18 -0500 Subject: [PATCH 28/56] chore: set CMS_CFG instead of STUDIO_CFG In the LMS/CMS Dockerfile, the env var STUDIO_CFG is set in order to point CMS at its configuration json/yaml file. Since https://github.com/edx/edx-platform/pull/29534 (which introduced 0013-cms-vs-studio.rst), the STUDIO_CFG variable has been deprecated in favor of CMS_CFG. This change updates the Dockerfile to reflect the new preferred environment variable. The only noticeable impact of this change is that it will remove a depreation warning from Django startup for tutor uses running off of Open edX master. --- CHANGELOG-nightly.md | 1 + tutor/templates/build/openedx/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG-nightly.md b/CHANGELOG-nightly.md index a6181c0..af3f2a0 100644 --- a/CHANGELOG-nightly.md +++ b/CHANGELOG-nightly.md @@ -2,6 +2,7 @@ Note: Breaking changes between versions are indicated by "💥". +- [Improvement] Point CMS at its config file using ``CMS_CFG`` environment variable instead of deprecated ``STUDIO_CFG``. - [Bugfix] Start MongoDB when running migrations, because a new data migration fails if MongoDB is not running - [Feature] Better support of Caddy as a load balancer in Kubernetes: - Make it possible to start/stop a selection of resources with ``tutor k8s start/stop [names...]``. diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index bc227c0..81619dc 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -143,7 +143,7 @@ RUN pip install -r requirements/edx/local.in RUN mkdir -p /openedx/config ./lms/envs/tutor ./cms/envs/tutor COPY --chown=app:app revisions.yml /openedx/config/ ENV LMS_CFG /openedx/config/lms.env.json -ENV STUDIO_CFG /openedx/config/cms.env.json +ENV CMS_CFG /openedx/config/cms.env.json ENV REVISION_CFG /openedx/config/revisions.yml COPY --chown=app:app settings/lms/*.py ./lms/envs/tutor/ COPY --chown=app:app settings/cms/*.py ./cms/envs/tutor/ From 76ef7de74f4afd26c0218884436581f4d4a701ed Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Fri, 18 Mar 2022 10:17:20 -0400 Subject: [PATCH 29/56] fix: lms-worker/cms-worker command update for Celery 5 Before edx-platform was upgraded to Celery 5, lms-worker and cms-worker could be invoked using this syntax: celery worker --app=APP --maxtasksperchild=N Since the recent Celery 5 upgrade (edx-platform commit 0588c92), though, this fails with the messages: You are using `--app` as an option of the worker sub-command: celery worker --app celeryapp <...> The support for this usage was removed in Celery 5.0. Instead you should use `--app` as a global option: celery --app celeryapp worker <...> and: Error: No such option: --maxtasksperchild (Possible options: --max-memory-per-child, --max-tasks-per-child) So, this commit changes the lms-worker and cms-worker invocations to: celery --app=APP --max-tasks-per-child=N --- CHANGELOG-nightly.md | 1 + tutor/templates/k8s/deployments.yml | 4 ++-- tutor/templates/local/docker-compose.yml | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG-nightly.md b/CHANGELOG-nightly.md index af3f2a0..80f2aea 100644 --- a/CHANGELOG-nightly.md +++ b/CHANGELOG-nightly.md @@ -2,6 +2,7 @@ Note: Breaking changes between versions are indicated by "💥". +- [Bugfix] Update ``celery`` invocations for lms-worker and cms-worker to be compatible with Celery 5 CLI. - [Improvement] Point CMS at its config file using ``CMS_CFG`` environment variable instead of deprecated ``STUDIO_CFG``. - [Bugfix] Start MongoDB when running migrations, because a new data migration fails if MongoDB is not running - [Feature] Better support of Caddy as a load balancer in Kubernetes: diff --git a/tutor/templates/k8s/deployments.yml b/tutor/templates/k8s/deployments.yml index 823cdf5..1810778 100644 --- a/tutor/templates/k8s/deployments.yml +++ b/tutor/templates/k8s/deployments.yml @@ -113,7 +113,7 @@ spec: containers: - name: cms-worker image: {{ DOCKER_IMAGE_OPENEDX }} - args: ["celery", "worker", "--app=cms.celery", "--loglevel=info", "--hostname=edx.cms.core.default.%%h", "--maxtasksperchild", "100", "--exclude-queues=edx.lms.core.default"] + args: ["celery", "--app=cms.celery", "worker", "--loglevel=info", "--hostname=edx.cms.core.default.%%h", "--max-tasks-per-child", "100", "--exclude-queues=edx.lms.core.default"] env: - name: SERVICE_VARIANT value: cms @@ -206,7 +206,7 @@ spec: containers: - name: lms-worker image: {{ DOCKER_IMAGE_OPENEDX }} - args: ["celery", "worker", "--app=lms.celery", "--loglevel=info", "--hostname=edx.lms.core.default.%%h", "--maxtasksperchild=100", "--exclude-queues=edx.cms.core.default"] + args: ["celery", "--app=lms.celery", "worker", "--loglevel=info", "--hostname=edx.lms.core.default.%%h", "--max-tasks-per-child=100", "--exclude-queues=edx.cms.core.default"] env: - name: SERVICE_VARIANT value: lms diff --git a/tutor/templates/local/docker-compose.yml b/tutor/templates/local/docker-compose.yml index b5c97d3..cddbe65 100644 --- a/tutor/templates/local/docker-compose.yml +++ b/tutor/templates/local/docker-compose.yml @@ -170,7 +170,7 @@ services: environment: SERVICE_VARIANT: lms SETTINGS: ${TUTOR_EDX_PLATFORM_SETTINGS:-tutor.production} - command: celery worker --app=lms.celery --loglevel=info --hostname=edx.lms.core.default.%%h --maxtasksperchild=100 --exclude-queues=edx.cms.core.default + command: celery --app=lms.celery worker --loglevel=info --hostname=edx.lms.core.default.%%h --max-tasks-per-child=100 --exclude-queues=edx.cms.core.default restart: unless-stopped volumes: - ../apps/openedx/settings/lms:/openedx/edx-platform/lms/envs/tutor:ro @@ -188,7 +188,7 @@ services: environment: SERVICE_VARIANT: cms SETTINGS: ${TUTOR_EDX_PLATFORM_SETTINGS:-tutor.production} - command: celery worker --app=cms.celery --loglevel=info --hostname=edx.cms.core.default.%%h --maxtasksperchild 100 --exclude-queues=edx.lms.core.default + command: celery --app=cms.celery worker --loglevel=info --hostname=edx.cms.core.default.%%h --max-tasks-per-child 100 --exclude-queues=edx.lms.core.default restart: unless-stopped volumes: - ../apps/openedx/settings/lms:/openedx/edx-platform/lms/envs/tutor:ro From cdcd5c7b3e07c61595580dd213bf0bf73dbbb137 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Thu, 24 Mar 2022 08:27:11 +0100 Subject: [PATCH 30/56] v13.1.8 (2022-03-18) - [Bugfix] Fix "evalsymlink failure" during `k8s quickstart` (#611). - [Bugfix] Fix "TypeError: upgrade() got an unexpected keyword argument 'non_interactive'" during `local upgrade`. --- CHANGELOG.md | 3 ++- tutor/__about__.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1aa28f2..06b7063 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,10 +4,11 @@ Note: Breaking changes between versions are indicated by "💥". ## Unreleased +## v13.1.8 (2022-03-18) + - [Bugfix] Fix "evalsymlink failure" during `k8s quickstart` (#611). - [Bugfix] Fix "TypeError: upgrade() got an unexpected keyword argument 'non_interactive'" during `local upgrade`. - ## v13.1.7 (2022-03-17) - [Bugfix] Fix dockerize on arm64 by switching to the [powerman/dockerize](https://github.com/powerman/dockerize) fork (#591). diff --git a/tutor/__about__.py b/tutor/__about__.py index 45cbe60..e1ffb83 100644 --- a/tutor/__about__.py +++ b/tutor/__about__.py @@ -2,7 +2,7 @@ import os # Increment this version number to trigger a new release. See # docs/tutor.html#versioning for information on the versioning scheme. -__version__ = "13.1.7" +__version__ = "13.1.8" # The version suffix will be appended to the actual version, separated by a # dash. Use this suffix to differentiate between the actual released version and From cbe753bc739e0d654dc64a63b691215731cee37a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Mon, 28 Mar 2022 10:52:26 +0200 Subject: [PATCH 31/56] docs: trim changelog-nightly Older changes have been moved to CHANGELOG.md, so we can remove them from this file. --- CHANGELOG-nightly.md | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/CHANGELOG-nightly.md b/CHANGELOG-nightly.md index 80f2aea..6fcfda7 100644 --- a/CHANGELOG-nightly.md +++ b/CHANGELOG-nightly.md @@ -5,25 +5,3 @@ Note: Breaking changes between versions are indicated by "💥". - [Bugfix] Update ``celery`` invocations for lms-worker and cms-worker to be compatible with Celery 5 CLI. - [Improvement] Point CMS at its config file using ``CMS_CFG`` environment variable instead of deprecated ``STUDIO_CFG``. - [Bugfix] Start MongoDB when running migrations, because a new data migration fails if MongoDB is not running -- [Feature] Better support of Caddy as a load balancer in Kubernetes: - - Make it possible to start/stop a selection of resources with ``tutor k8s start/stop [names...]``. - - Make it easy to deploy an independent LoadBalancer by converting the caddy service to a ClusterIP when ``ENABLE_WEB_PROXY=false``. - - Add a ``app.kubernetes.io/component: loadbalancer`` label to the LoadBalancer service. - - Add ``app.kubernetes.io/name`` labels to all services. - - Preserve the LoadBalancer service in ``tutor k8s stop`` commands. - - Wait for the caddy deployment to be ready before running initialisation jobs. -- [Security] On Kubernetes, convert all NodePort services to ClusterIP to guarantee network isolation from outside the cluster. -- 💥[Improvement] Drop Python 3.5 compatibility. -- [Bugfix] Fix docker-compose project name in development on nightly branch. -- 💥[Bugfix] No longer track the Tutor version number in resource labels (and label selectors, which breaks the update of Deployment resources), but instead do so in resource annotations. -- [Bugfix] Make it possible for plugins to implement the "caddyfile" patch without relying on the "port" local variable. -- 💥[Improvement] Move the Open edX forum to a [dedicated plugin](https://github.com/overhangio/tutor-forum/) (#450). -- 💥[Improvement] Get rid of the "tutor-openedx" package, which is no longer supported. -- [Bugfix] Fix running Caddy container in k8s, which should always be the case even if `ENABLE_WEB_PROXY` is false. -- 💥[Improvement] Run all services as unprivileged containers, for better security. This has multiple consequences: - - The "openedx-dev" image is now built with `tutor dev dc build lms`. - - The "smtp" service now runs the "devture/exim-relay" Docker image, which is unprivileged. Also, the default SMTP port is now 8025. -- 💥[Feature] Get rid of the nginx container and service, which is now replaced by Caddy. this has the following consequences: - - Patches "nginx-cms", "nginx-lms", "nginx-extra", "local-docker-compose-nginx-aliases" are replaced by "caddyfile-cms", "caddyfile-lms", "caddyfile", " local-docker-compose-caddy-aliases". - - Patches "k8s-deployments-nginx-volume-mounts", "k8s-deployments-nginx-volumes" were obsolete and are removed. - - The `NGINX_HTTP_PORT` setting is renamed to `CADDY_HTTP_PORT`. From 18d3b57748c89c3914137b1e2a940bfb89e9382a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Mon, 28 Mar 2022 10:54:03 +0200 Subject: [PATCH 32/56] fix: remove references to edX from bulk emails Bulk course emails used to contain many references to edX. This is resolved here by switching to edx-ace for sending emails. See: https://github.com/openedx/build-test-release-wg/issues/100 https://edx.readthedocs.io/projects/edx-platform-technical/en/latest/featuretoggles.html#featuretoggle-BULK_EMAIL_SEND_USING_EDX_ACE https://github.com/openedx/edx-platform/pull/29900 --- CHANGELOG-nightly.md | 1 + tutor/templates/apps/openedx/settings/partials/common_lms.py | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGELOG-nightly.md b/CHANGELOG-nightly.md index 6fcfda7..a3ee6f3 100644 --- a/CHANGELOG-nightly.md +++ b/CHANGELOG-nightly.md @@ -2,6 +2,7 @@ Note: Breaking changes between versions are indicated by "💥". +- [Bugfix] Remove edX references from bulk emails ([issue](https://github.com/openedx/build-test-release-wg/issues/100)). - [Bugfix] Update ``celery`` invocations for lms-worker and cms-worker to be compatible with Celery 5 CLI. - [Improvement] Point CMS at its config file using ``CMS_CFG`` environment variable instead of deprecated ``STUDIO_CFG``. - [Bugfix] Start MongoDB when running migrations, because a new data migration fails if MongoDB is not running diff --git a/tutor/templates/apps/openedx/settings/partials/common_lms.py b/tutor/templates/apps/openedx/settings/partials/common_lms.py index 994cab8..ca49c50 100644 --- a/tutor/templates/apps/openedx/settings/partials/common_lms.py +++ b/tutor/templates/apps/openedx/settings/partials/common_lms.py @@ -20,6 +20,7 @@ OAUTH_ENFORCE_SECURE = False # Email settings DEFAULT_EMAIL_LOGO_URL = LMS_ROOT_URL + "/theming/asset/images/logo.png" +BULK_EMAIL_SEND_USING_EDX_ACE = True # Create folders if necessary for folder in [DATA_DIR, LOG_DIR, MEDIA_ROOT, STATIC_ROOT_BASE, ORA2_FILEUPLOAD_ROOT]: From 698f49854d6fa417f3c94db8112500c66d38618e Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Wed, 6 Apr 2022 10:29:09 -0400 Subject: [PATCH 33/56] build: NIGHTLY ONLY: install nightly branches of official plugins For Tutor Nightly (and only Nightly), official plugins are now installed from their nightly branches on GitHub instead of a version range on PyPI. This will allow Nightly users to install all official plugins by running: pip install -e ".[full]" Notes: * We use the syntax `EGG @ git+REPO@nightly` because the more common syntax of `git+REPO@nightly#egg=EGG` does not work when supplied to setup.py's extras_require. * Unlike other plugins, tutor-license is still installed from PyPI, but without any version constraint. This is because tutor-license is a simple, closed-source plugin which activates Wizard edition for subscribers. It should be available in Nightly but doesn't need to be installed from its own bleeding-edge branch. * Unlike most nightly commits, this commit should NOT ever be reflected on master. When it comes time to merge nightly into master during the release of Nutmeg, this commit will need to be manually reverted from master. * Documentation updates have been made separately so that they can be merged into master. --- CHANGELOG-nightly.md | 1 + requirements/plugins.txt | 25 +++++++++++++------------ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/CHANGELOG-nightly.md b/CHANGELOG-nightly.md index a3ee6f3..a24447f 100644 --- a/CHANGELOG-nightly.md +++ b/CHANGELOG-nightly.md @@ -2,6 +2,7 @@ Note: Breaking changes between versions are indicated by "💥". +- [Improvement] For Tutor Nightly (and only Nightly), official plugins are now installed from their nightly branches on GitHub instead of a version range on PyPI. This will allow Nightly users to install all official plugins by running ``pip install -e ".[full]"``. - [Bugfix] Remove edX references from bulk emails ([issue](https://github.com/openedx/build-test-release-wg/issues/100)). - [Bugfix] Update ``celery`` invocations for lms-worker and cms-worker to be compatible with Celery 5 CLI. - [Improvement] Point CMS at its config file using ``CMS_CFG`` environment variable instead of deprecated ``STUDIO_CFG``. diff --git a/requirements/plugins.txt b/requirements/plugins.txt index 5107ef7..492a6f6 100644 --- a/requirements/plugins.txt +++ b/requirements/plugins.txt @@ -1,12 +1,13 @@ -# change version ranges when upgrading from maple -tutor-android>=13.0.0,<14.0.0 -tutor-discovery>=13.0.0,<14.0.0 -tutor-ecommerce>=13.0.0,<14.0.0 -tutor-forum>=13.0.0,<14.0.0 -tutor-license>=13.0.0,<14.0.0 -tutor-mfe>=13.0.0,<14.0.0 -tutor-minio>=13.0.0,<14.0.0 -tutor-notes>=13.0.0,<14.0.0 -tutor-richie>=13.0.0,<14.0.0 -tutor-webui>=13.0.0,<14.0.0 -tutor-xqueue>=13.0.0,<14.0.0 +# For Tutor Nightly, we install plugins from their nightly branches instead of from PyPI +# (except tutor-license, for which we just want the latest version from PyPI). +tutor-android @ git+https://github.com/overhangio/tutor-android@nightly +tutor-discovery @ git+https://github.com/overhangio/tutor-discovery@nightly +tutor-ecommerce @ git+https://github.com/overhangio/tutor-ecommerce@nightly +tutor-forum @ git+https://github.com/overhangio/tutor-forum@nightly +tutor-license +tutor-mfe @ git+https://github.com/overhangio/tutor-mfe@nightly +tutor-minio @ git+https://github.com/overhangio/tutor-minio@nightly +tutor-notes @ git+https://github.com/overhangio/tutor-notes@nightly +tutor-richie @ git+https://github.com/overhangio/tutor-richie@nightly +tutor-webui @ git+https://github.com/overhangio/tutor-webui@nightly +tutor-xqueue @ git+https://github.com/overhangio/tutor-xqueue@nightly From 0fdf8d83189d030d3bfba8275b9ac6269d773d60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Sun, 24 Apr 2022 11:30:34 +0200 Subject: [PATCH 34/56] v13.2.0 (2022-04-24) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - [Improvement] Add the `COMPOSE_PROJECT_STARTED` action and run `dev stop` on `local start` (and vice versa). - [Feature] Introduce `local/dev copyfrom` command to copy contents from a container. - [Bugfix] Fix a race condition that could prevent a newly provisioned LMS container from starting due to a `FileExistsError` when creating data folders. - [Deprecation] Mark `tutor dev runserver` as deprecated in favor of `tutor dev start`. Since `start` now supports bind-mounting and breakpoint debugging, `runserver` is redundant and will be removed in a future release. - [Improvement] Allow breakpoint debugging when attached to a service via `tutor dev start SERVICE`. - [Security] Apply rate limiting security fix (see [commit](https://github.com/overhangio/edx-platform/commit/b5723e416e628cac4fa84392ca13e1b72817674f)). - [Feature] Introduce the ``-m/--mount`` option in ``local`` and ``dev`` commands to auto-magically bind-mount folders from the host. - [Feature] Add `tutor dev quickstart` command, which is similar to `tutor local quickstart`, except that it uses dev containers instead of local production ones and includes some other small differences for the convience of Open edX developers. This should remove some friction from the Open edX development setup process, which previously required that users provision using local producation containers (`tutor local quickstart`) but then stop them and switch to dev containers (`tutor local stop && tutor dev start -d`). - 💥[Improvement] Make it possible to run `tutor k8s exec ` (#636). As a consequence, it is no longer possible to run quoted commands: `tutor k8s exec ""`. Instead, you should remove the quotes: `tutor k8s exec `. - 💥[Deprecation] Drop support for the `TUTOR_EDX_PLATFORM_SETTINGS` environment variable. It is now recommended to create a plugin instead. - 💥[Improvement] Complete overhaul of the plugin extension mechanism. Tutor now has a hook-based Python API: actions can be triggered at different points of the application life cycle and data can be modified thanks to custom filters. The v0 plugin API is still supported, for backward compatibility, but plugin developers are encouraged to migrate their plugins to the new API. See the new plugin tutorial for more information. - [Improvement] Improved the output of `tutor plugins list`. - [Feature] Add `tutor [dev|local|k8s] status` command, which provides basic information about the platform's status. --- CHANGELOG.md | 2 ++ tutor/__about__.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 376da6f..d6a887c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Note: Breaking changes between versions are indicated by "💥". ## Unreleased +## v13.2.0 (2022-04-24) + - [Improvement] Add the `COMPOSE_PROJECT_STARTED` action and run `dev stop` on `local start` (and vice versa). - [Feature] Introduce `local/dev copyfrom` command to copy contents from a container. - [Bugfix] Fix a race condition that could prevent a newly provisioned LMS container from starting due to a `FileExistsError` when creating data folders. diff --git a/tutor/__about__.py b/tutor/__about__.py index 5b878fc..1ec297d 100644 --- a/tutor/__about__.py +++ b/tutor/__about__.py @@ -2,7 +2,7 @@ import os # Increment this version number to trigger a new release. See # docs/tutor.html#versioning for information on the versioning scheme. -__version__ = "13.1.11" +__version__ = "13.2.0" # The version suffix will be appended to the actual version, separated by a # dash. Use this suffix to differentiate between the actual released version and From 13382c889be8dde767ea18d526cf2cef835381c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Sun, 24 Apr 2022 12:50:34 +0200 Subject: [PATCH 35/56] ci: attempt to fix github release CI Github release CI was running on ubuntu 18.04 withh python 3.6. Installing tomli==2.0.1, which is required in dev, triggers a failure in python 3.6 because it is no longer available. --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f68978b..8c0b3e2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,7 +11,7 @@ jobs: strategy: matrix: include: - - os: ubuntu-18.04 + - os: ubuntu-latest locale: C.UTF-8 - os: macos-10.15 locale: en_US.UTF-8 @@ -23,7 +23,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: 3.6 + python-version: 3.7 - name: Upgrade pip run: python -m pip install --upgrade pip setuptools==44.0.0 - name: Print info about the current python installation From 65c58c18f559c53c2e8375b14a46defe85812397 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Fri, 3 Jun 2022 14:58:32 +0200 Subject: [PATCH 36/56] security: apply logout redirect security patch --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d57dc9..4170d26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ Every user-facing change should have an entry in this changelog. Please respect ## Unreleased +- [Security] Apply logout redirect url security fix. (by @regisb) - [Feature] Make it possible to force the rendering of a given template, even when the template path matches an ignore pattern. (by @regisb) - 💥[Fix] Get rid of the `tutor config render` command, which is useless now that themes can be implemented as plugins. (by @regisb) From 8a990c2bf7a7e14344603dd3a9b27726ec5a3f29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Fri, 3 Jun 2022 15:00:03 +0200 Subject: [PATCH 37/56] v13.3.0 (2022-06-03) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - [Security] Apply logout redirect url security fix. (by @regisb) - [Feature] Make it possible to force the rendering of a given template, even when the template path matches an ignore pattern. (by @regisb) - 💥[Fix] Get rid of the `tutor config render` command, which is useless now that themes can be implemented as plugins. (by @regisb) --- CHANGELOG.md | 2 ++ tutor/__about__.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4170d26..9fd5f66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,8 @@ Every user-facing change should have an entry in this changelog. Please respect ## Unreleased +## v13.3.0 (2022-06-03) + - [Security] Apply logout redirect url security fix. (by @regisb) - [Feature] Make it possible to force the rendering of a given template, even when the template path matches an ignore pattern. (by @regisb) - 💥[Fix] Get rid of the `tutor config render` command, which is useless now that themes can be implemented as plugins. (by @regisb) diff --git a/tutor/__about__.py b/tutor/__about__.py index cdba19b..18406ff 100644 --- a/tutor/__about__.py +++ b/tutor/__about__.py @@ -2,7 +2,7 @@ import os # Increment this version number to trigger a new release. See # docs/tutor.html#versioning for information on the versioning scheme. -__version__ = "13.2.3" +__version__ = "13.3.0" # The version suffix will be appended to the actual version, separated by a # dash. Use this suffix to differentiate between the actual released version and From 33c5c3269cc4e117c8e12fee3cfbc013395c8c7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Fri, 3 Jun 2022 18:38:59 +0200 Subject: [PATCH 38/56] chore: upgrade to node 16 in openedx Docker image --- tutor/templates/build/openedx/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index 9ec44a9..913c381 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -104,7 +104,7 @@ ENV PATH /openedx/nodeenv/bin:/openedx/venv/bin:${PATH} # Install nodeenv with the version provided by edx-platform RUN pip install nodeenv==1.6.0 -RUN nodeenv /openedx/nodeenv --node=12.13.0 --prebuilt +RUN nodeenv /openedx/nodeenv --node=16.14.0 --prebuilt # Install nodejs requirements ARG NPM_REGISTRY={{ NPM_REGISTRY }} From 5faf03e3b3eaf86efeb90d53ffd4dbcfe3183d2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Fri, 3 Jun 2022 15:36:11 +0200 Subject: [PATCH 39/56] fix: revert back to "npm install" "npm ci" is broken in master because of the node 16 upgrade. See discussion here: https://github.com/openedx/edx-platform/pull/30459#issuecomment-1136888560 We will revert to "npm ci" once the node 16 upgrade is complete. --- tutor/templates/build/openedx/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index 913c381..eb06564 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -111,7 +111,7 @@ ARG NPM_REGISTRY={{ NPM_REGISTRY }} COPY --from=code /openedx/edx-platform/package.json /openedx/edx-platform/package.json COPY --from=code /openedx/edx-platform/package-lock.json /openedx/edx-platform/package-lock.json WORKDIR /openedx/edx-platform -RUN npm clean-install --verbose --registry=$NPM_REGISTRY +RUN npm install --verbose --registry=$NPM_REGISTRY ###### Production image with system and python requirements FROM minimal as production From 6ac1c0d732275505290cef935eee4393d213b1df Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Thu, 30 Jun 2022 11:23:01 -0400 Subject: [PATCH 40/56] fix: replica set error from pymongo 3.10 -> 3.12 upgrade The pymongo dependency for edx-platform was updated (3.10.1 to 3.12.3) in https://github.com/openedx/edx-platform/pull/30569 This caused the following error when running the edx-platform database migration split_modulestore_django.0002_data_migration as part of `tutor dev quickstart`: pymongo.errors.ServerSelectionTimeoutError: client is configured to connect to a replica set named '' but this node belongs to a set named 'None', Timeout: 30s, Topology Description: ]> This commit explicitly sets replicaSet to None to indicate that it's a standalone MongoDB instance. I also had to remove the CONTENTSTORE entry from auth.yml because edx-platform's devstack.py assumes it has a non-null value (set in common.py), and devstack.py executes before tutor's development.py can set this replicaSet value. --- CHANGELOG-nightly.md | 1 + tutor/templates/apps/openedx/config/partials/auth.yml | 1 - tutor/templates/apps/openedx/settings/partials/common_all.py | 1 + 3 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG-nightly.md b/CHANGELOG-nightly.md index 6070b30..092f962 100644 --- a/CHANGELOG-nightly.md +++ b/CHANGELOG-nightly.md @@ -9,6 +9,7 @@ When backporting changes to master, we should keep only the entries that corresp facing changes. --> +- [Bugfix] Fix MongoDB replica set connection error resulting from edx-platform's pymongo (3.10.1 -> 3.12.3) upgrade ([edx-platform#30569](https://github.com/openedx/edx-platform/pull/30569)). (by @ormsbee) - [Improvement] For Tutor Nightly (and only Nightly), official plugins are now installed from their nightly branches on GitHub instead of a version range on PyPI. This will allow Nightly users to install all official plugins by running ``pip install -e ".[full]"``. - [Bugfix] Remove edX references from bulk emails ([issue](https://github.com/openedx/build-test-release-wg/issues/100)). - [Bugfix] Update ``celery`` invocations for lms-worker and cms-worker to be compatible with Celery 5 CLI. diff --git a/tutor/templates/apps/openedx/config/partials/auth.yml b/tutor/templates/apps/openedx/config/partials/auth.yml index 2ec6448..3e72058 100644 --- a/tutor/templates/apps/openedx/config/partials/auth.yml +++ b/tutor/templates/apps/openedx/config/partials/auth.yml @@ -1,7 +1,6 @@ SECRET_KEY: "{{ OPENEDX_SECRET_KEY }}" AWS_ACCESS_KEY_ID: "{{ OPENEDX_AWS_ACCESS_KEY }}" AWS_SECRET_ACCESS_KEY: "{{ OPENEDX_AWS_SECRET_ACCESS_KEY }}" -CONTENTSTORE: null DOC_STORE_CONFIG: null {{ patch("openedx-auth") }} XQUEUE_INTERFACE: diff --git a/tutor/templates/apps/openedx/settings/partials/common_all.py b/tutor/templates/apps/openedx/settings/partials/common_all.py index d6a9291..8a74c88 100644 --- a/tutor/templates/apps/openedx/settings/partials/common_all.py +++ b/tutor/templates/apps/openedx/settings/partials/common_all.py @@ -16,6 +16,7 @@ mongodb_parameters = { "password": None, {% endif %} "db": "{{ MONGODB_DATABASE }}", + "replicaSet": None, } DOC_STORE_CONFIG = mongodb_parameters CONTENTSTORE = { From 9d1ce4717bfd1df522edd508017ea93a4abc191a Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Wed, 13 Jul 2022 08:53:55 -0400 Subject: [PATCH 41/56] docs: update test-running guide with new top-level xmodule package edx-platform's ./common/lib/xmodule/xmodule folder has been moved to ./xmodule. The test-running instructions needed to be updated in order to account for this new top-level folder. For context on the edx-platform change, see: https://discuss.openedx.org/t/breaking-apart-edx-platforms-common-lib-folder --- docs/dev.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/dev.rst b/docs/dev.rst index 9c6d1e8..071badc 100644 --- a/docs/dev.rst +++ b/docs/dev.rst @@ -283,6 +283,7 @@ Then, run unit tests with ``pytest`` commands:: export EDXAPP_TEST_MONGO_HOST=mongodb pytest common pytest openedx + pytest xmodule # Run tests on LMS export DJANGO_SETTINGS_MODULE=lms.envs.tutor.test From b879a9d17bc5812ee4a3742c71c16519019bf14d Mon Sep 17 00:00:00 2001 From: Carlos Muniz Date: Thu, 21 Jul 2022 14:22:22 -0400 Subject: [PATCH 42/56] feat: remove the implementation of dev runserver --- tutor/commands/dev.py | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/tutor/commands/dev.py b/tutor/commands/dev.py index 5d2caa6..4194f56 100644 --- a/tutor/commands/dev.py +++ b/tutor/commands/dev.py @@ -120,37 +120,6 @@ Your Open edX platform is ready and can be accessed at the following urls: ) -@click.command( - help="DEPRECATED: Use 'tutor dev start ...' instead!", - context_settings={"ignore_unknown_options": True}, -) -@compose.mount_option -@click.argument("options", nargs=-1, required=False) -@click.argument("service") -@click.pass_context -def runserver( - context: click.Context, - mounts: t.Tuple[t.List[compose.MountParam.MountType]], - options: t.List[str], - service: str, -) -> None: - depr_warning = "'runserver' is deprecated and will be removed in a future release. Use 'start' instead." - for option in options: - if option.startswith("-v") or option.startswith("--volume"): - depr_warning += " Bind-mounts can be specified using '-m/--mount'." - break - fmt.echo_alert(depr_warning) - config = tutor_config.load(context.obj.root) - if service in ["lms", "cms"]: - port = 8000 if service == "lms" else 8001 - host = config["LMS_HOST"] if service == "lms" else config["CMS_HOST"] - fmt.echo_info( - f"The {service} service will be available at http://{host}:{port}" - ) - args = ["--service-ports", *options, service] - context.invoke(compose.run, mounts=mounts, args=args) - - @hooks.Actions.COMPOSE_PROJECT_STARTED.add() def _stop_on_local_start(root: str, config: Config, project_name: str) -> None: """ @@ -163,5 +132,4 @@ def _stop_on_local_start(root: str, config: Config, project_name: str) -> None: dev.add_command(quickstart) -dev.add_command(runserver) compose.add_commands(dev) From 72c417da629132cf7794bbda6a62a24089a220e4 Mon Sep 17 00:00:00 2001 From: Carlos Muniz Date: Thu, 21 Jul 2022 15:51:42 -0400 Subject: [PATCH 43/56] docs: add entry to CHANGELOG-nightly.md --- CHANGELOG-nightly.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG-nightly.md b/CHANGELOG-nightly.md index 092f962..a96d48b 100644 --- a/CHANGELOG-nightly.md +++ b/CHANGELOG-nightly.md @@ -9,6 +9,7 @@ When backporting changes to master, we should keep only the entries that corresp facing changes. --> +- 💥[Improvement] Remove the implementation of tutor dev runserver. (by @Carlos-Muniz) - [Bugfix] Fix MongoDB replica set connection error resulting from edx-platform's pymongo (3.10.1 -> 3.12.3) upgrade ([edx-platform#30569](https://github.com/openedx/edx-platform/pull/30569)). (by @ormsbee) - [Improvement] For Tutor Nightly (and only Nightly), official plugins are now installed from their nightly branches on GitHub instead of a version range on PyPI. This will allow Nightly users to install all official plugins by running ``pip install -e ".[full]"``. - [Bugfix] Remove edX references from bulk emails ([issue](https://github.com/openedx/build-test-release-wg/issues/100)). From d9314b750741fde14d5611d14444e4afdc87e786 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Tue, 30 Aug 2022 14:22:52 -0400 Subject: [PATCH 44/56] build: prepare Dockerfile for common/lib removal Soon, running: pip install -r ./requirements/edx/base.txt in edx-platform will no longer install the local project (that is, `-e .`). To prepare for that change, we add the line: pip install -e . to the Dockerfile. This is backwards-compatible. More details: https://openedx.atlassian.net/browse/BOM-2575?focusedCommentId=613181 --- tutor/templates/build/openedx/Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index d88b42b..6a4cc9a 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -74,8 +74,9 @@ ENV VIRTUAL_ENV /openedx/venv/ RUN apt update && apt install -y software-properties-common libmysqlclient-dev libxmlsec1-dev libgeos-dev # Note that this means that we need to reinstall all requirements whenever there is a -# change in edx-platform, which sucks. But there is no obvious alternative, as we need -# to install some packages from edx-platform. +# change in edx-platform, which sucks. Yet, we must do it, because edx-platform installs some +# Python projects from within the edx-platform repo itself. This is being fixed upstream. +# TODO: https://github.com/overhangio/2u-tutor-adoption/issues/86 COPY --from=code /openedx/edx-platform /openedx/edx-platform WORKDIR /openedx/edx-platform @@ -87,6 +88,7 @@ RUN pip install setuptools==62.1.0 pip==22.0.4 wheel==0.37.1 # Install base requirements RUN pip install -r ./requirements/edx/base.txt +RUN pip install -e . # Install django-redis for using redis as a django cache # https://pypi.org/project/django-redis/ From fe901ab9de707ff076a2294e3d1e974eeb40456c Mon Sep 17 00:00:00 2001 From: Carlos Muniz Date: Fri, 30 Sep 2022 06:00:24 -0400 Subject: [PATCH 45/56] feat: deprecate "quickstart" and rename to "launch" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `quickstart` is being renamed to `launch` and deprecated in favor of using `launch`. The `quickstart` function temporarily aliases to `launch`. Further mentions of `quickstart` have been changed to reference `launch` instead. We are indicating that this change is breaking 💥 to encourage people to migrate their scripts right away! --- CHANGELOG-nightly.md | 2 +- README.rst | 7 ++++--- docs/configuration.rst | 6 +++--- docs/dev.rst | 10 +++++----- docs/gettingstarted.rst | 2 +- docs/index.rst | 3 ++- docs/install.rst | 16 ++++++++-------- docs/intro.rst | 4 ++-- docs/k8s.rst | 6 +++--- docs/local.rst | 6 +++--- docs/plugins/intro.rst | 2 +- docs/plugins/v0/gettingstarted.rst | 2 +- docs/quickstart.rst | 8 ++++---- docs/troubleshooting.rst | 8 ++++---- docs/tutorials/google-smtp.rst | 2 +- docs/tutorials/nightly.rst | 2 +- docs/tutorials/oldreleases.rst | 6 +++--- docs/tutorials/plugin.rst | 2 +- docs/whatnext.rst | 2 +- tests/commands/test_local.py | 4 ++-- tutor/commands/dev.py | 25 ++++++++++++++++++++++++- tutor/commands/k8s.py | 20 +++++++++++++++++--- tutor/commands/local.py | 29 ++++++++++++++++++++++++++--- tutor/config.py | 2 +- 24 files changed, 119 insertions(+), 57 deletions(-) diff --git a/CHANGELOG-nightly.md b/CHANGELOG-nightly.md index a96d48b..bf85f74 100644 --- a/CHANGELOG-nightly.md +++ b/CHANGELOG-nightly.md @@ -8,7 +8,7 @@ will be backported to the master branch at every major release. When backporting changes to master, we should keep only the entries that correspond to user- facing changes. --> - +- 💥[Improvement] Rename the implementation of tutor quickstart to tutor launch. (by @Carlos-Muniz) - 💥[Improvement] Remove the implementation of tutor dev runserver. (by @Carlos-Muniz) - [Bugfix] Fix MongoDB replica set connection error resulting from edx-platform's pymongo (3.10.1 -> 3.12.3) upgrade ([edx-platform#30569](https://github.com/openedx/edx-platform/pull/30569)). (by @ormsbee) - [Improvement] For Tutor Nightly (and only Nightly), official plugins are now installed from their nightly branches on GitHub instead of a version range on PyPI. This will allow Nightly users to install all official plugins by running ``pip install -e ".[full]"``. diff --git a/README.rst b/README.rst index 0d39c6d..39ee708 100644 --- a/README.rst +++ b/README.rst @@ -55,16 +55,17 @@ Features * No technical skill required with the `zero-click Tutor AWS image `__ .. _readme_intro_end: - +.. + TODO: replace image + alt with tutor local launch .. image:: ./docs/img/quickstart.gif :alt: Tutor local quickstart :target: https://terminalizer.com/view/91b0bfdd557 -Quickstart +Launch ---------- 1. Install the `latest stable release `_ of Tutor -2. Run ``tutor local quickstart`` +2. Run ``tutor local launch`` 3. You're done! Documentation diff --git a/docs/configuration.rst b/docs/configuration.rst index fea8142..3c03396 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -15,7 +15,7 @@ This section does not cover :ref:`plugin development `. For simple chan Configuration ------------- -With Tutor, all Open edX deployment parameters are stored in a single ``config.yml`` file. This is the file that is generated when you run ``tutor local quickstart`` or ``tutor config save``. To view the content of this file, run:: +With Tutor, all Open edX deployment parameters are stored in a single ``config.yml`` file. This is the file that is generated when you run ``tutor local launch`` or ``tutor config save``. To view the content of this file, run:: cat "$(tutor config printroot)/config.yml" @@ -324,7 +324,7 @@ The following sections describe how to modify various aspects of the docker imag tutor local stop -The custom image will be used the next time you run ``tutor local quickstart`` or ``tutor local start``. Do not attempt to run ``tutor local restart``! Restarting will not pick up the new image and will continue to use the old image. +The custom image will be used the next time you run ``tutor local launch`` or ``tutor local start``. Do not attempt to run ``tutor local restart``! Restarting will not pick up the new image and will continue to use the old image. openedx Docker Image build arguments ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -480,7 +480,7 @@ By default, Tutor runs the `overhangio/openedx `.) -The customised Docker image tag value will then be used by Tutor to run the platform, for instance when running ``tutor local quickstart``. +The customised Docker image tag value will then be used by Tutor to run the platform, for instance when running ``tutor local launch``. Passing custom docker build options diff --git a/docs/dev.rst b/docs/dev.rst index 071badc..d39eefc 100644 --- a/docs/dev.rst +++ b/docs/dev.rst @@ -13,7 +13,7 @@ First, ensure you have already :ref:`installed Tutor ` (for development Then, launch the developer platform setup process:: - tutor dev quickstart + tutor dev launch This will perform several tasks for you. It will: @@ -51,14 +51,14 @@ To bring down your platform's containers, simply run:: Starting the platform back up ----------------------------- -Once you have used ``quickstart`` once, you can start the platform in the future with the lighter-weight ``start`` command, which brings up containers but does not perform any initialization tasks:: +Once you have used ``launch`` once, you can start the platform in the future with the lighter-weight ``start`` command, which brings up containers but does not perform any initialization tasks:: tutor dev start # Run platform in the same terminal ("attached") tutor dev start -d # Or, run platform the in the background ("detached") -Nonetheless, ``quickstart`` is idempotent, so it is always safe to run it again in the future without risk to your data. In fact, you may find it useful to use this command as a one-stop-shop for pulling images, running migrations, initializing new plugins you have enabled, and/or executing any new initialization steps that may have been introduced since you set up Tutor:: +Nonetheless, ``launch`` is idempotent, so it is always safe to run it again in the future without risk to your data. In fact, you may find it useful to use this command as a one-stop-shop for pulling images, running migrations, initializing new plugins you have enabled, and/or executing any new initialization steps that may have been introduced since you set up Tutor:: - tutor dev quickstart --pullimages + tutor dev launch --pullimages Running arbitrary commands @@ -112,7 +112,7 @@ It may sometimes be convenient to mount container directories on the host, for i Bind-mount volumes with ``--mount`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The ``quickstart``, ``run``, ``init`` and ``start`` subcommands of ``tutor dev`` and ``tutor local`` support the ``-m/--mount`` option (see :option:`tutor dev start -m`) which can take two different forms. The first is explicit:: +The ``launch``, ``run``, ``init`` and ``start`` subcommands of ``tutor dev`` and ``tutor local`` support the ``-m/--mount`` option (see :option:`tutor dev start -m`) which can take two different forms. The first is explicit:: tutor dev start --mount=lms:/path/to/edx-platform:/openedx/edx-platform lms diff --git a/docs/gettingstarted.rst b/docs/gettingstarted.rst index 3341091..b40792b 100644 --- a/docs/gettingstarted.rst +++ b/docs/gettingstarted.rst @@ -8,5 +8,5 @@ Getting started install intro - quickstart + quickstart whatnext diff --git a/docs/index.rst b/docs/index.rst index a92299c..4a2b1f4 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,7 +1,8 @@ .. include:: ../README.rst :start-after: _readme_intro_start: :end-before: _readme_intro_end: - +.. + TODO replace quickstart.gif + alt with 'launch' .. image:: ./img/quickstart.gif :alt: Tutor local quickstart :target: https://terminalizer.com/view/91b0bfdd557 diff --git a/docs/install.rst b/docs/install.rst index b9f8aa9..56f677e 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -91,20 +91,20 @@ To upgrade Open edX or benefit from the latest features and bug fixes, you shoul pip install --upgrade "tutor[full]" -Then run the ``quickstart`` command again. Depending on your deployment target, run one of:: +Then run the ``launch`` command again. Depending on your deployment target, run one of:: - tutor local quickstart # for local installations - tutor dev quickstart # for local development installations - tutor k8s quickstart # for Kubernetes installation + tutor local launch # for local installations + tutor dev launch # for local development installations + tutor k8s launch # for Kubernetes installation Upgrading with custom Docker images ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -If you run :ref:`customised ` Docker images, you need to rebuild them before running ``quickstart``:: +If you run :ref:`customised ` Docker images, you need to rebuild them before running ``launch``:: tutor config save tutor images build all # specify here the images that you need to build - tutor local quickstart + tutor local launch Upgrading to a new Open edX release ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -121,12 +121,12 @@ Major Open edX releases are published twice a year, in June and December, by the 4. Test the new release in a sandboxed environment. 5. If you are running edx-platform, or some other repository from a custom branch, then you should rebase (and test) your changes on top of the latest release tag (see :ref:`edx_platform_fork`). -The process for upgrading from one major release to the next works similarly to any other upgrade, with the ``quickstart`` command (see above). The single difference is that if the ``quickstart`` command detects that your tutor environment was generated with an older release, it will perform a few release-specific upgrade steps. These extra upgrade steps will be performed just once. But they will be ignored if you updated your local environment (for instance: with ``tutor config save``) before running ``quickstart``. This situation typically occurs if you need to re-build some Docker images (see above). In such a case, you should make use of the ``upgrade`` command. For instance, to upgrade a local installation from Maple to Nutmeg and rebuild some Docker images, run:: +The process for upgrading from one major release to the next works similarly to any other upgrade, with the ``launch`` command (see above). The single difference is that if the ``launch`` command detects that your tutor environment was generated with an older release, it will perform a few release-specific upgrade steps. These extra upgrade steps will be performed just once. But they will be ignored if you updated your local environment (for instance: with ``tutor config save``) before running ``launch``. This situation typically occurs if you need to re-build some Docker images (see above). In such a case, you should make use of the ``upgrade`` command. For instance, to upgrade a local installation from Maple to Nutmeg and rebuild some Docker images, run:: tutor config save tutor images build all # list the images that should be rebuilt here tutor local upgrade --from=maple - tutor local quickstart + tutor local launch .. _autocomplete: diff --git a/docs/intro.rst b/docs/intro.rst index 3f25e28..bbef116 100644 --- a/docs/intro.rst +++ b/docs/intro.rst @@ -99,7 +99,7 @@ Because the Tutor environment is generated entirely from the values in ``config. You can now take advantage of the Tutor-powered CLI (item #3) to bootstrap your Open edX platform:: - tutor local quickstart + tutor local launch Under the hood, Tutor simply runs ``docker-compose`` and ``docker`` commands to launch your platform. These commands are printed in the standard output, such that you are free to replicate the same behaviour by simply copying/pasting the same commands. @@ -117,7 +117,7 @@ as well as command trees for each mode in which Tutor can run:: tutor k8s ... # Commands for managing a Kubernetes Open edX deployment. tutor dev ... # Commands for hacking on Open edX in development mode. -Within each mode, Tutor has subcommands for managing that type of Open edX instance. Many of them are common between modes, such as ``quickstart``, ``start``, ``stop``, ``exec``, and ``logs``. For example:: +Within each mode, Tutor has subcommands for managing that type of Open edX instance. Many of them are common between modes, such as ``launch``, ``start``, ``stop``, ``exec``, and ``logs``. For example:: tutor local logs # View logs of a local deployment. tutor k8s logs # View logs of a Kubernetes-managed deployment. diff --git a/docs/k8s.rst b/docs/k8s.rst index 649c614..b8e8ae3 100644 --- a/docs/k8s.rst +++ b/docs/k8s.rst @@ -85,19 +85,19 @@ The other benefit of ``kubectl apply`` is that it allows you to customise the Ku To learn more about "kustomizations", refer to the `official documentation `__. -Quickstart +Launch ---------- Launch the platform on Kubernetes in one command:: - tutor k8s quickstart + tutor k8s launch All Kubernetes resources are associated with the "openedx" namespace. If you don't see anything in the Kubernetes dashboard, you are probably looking at the wrong namespace... 😉 .. image:: img/k8s-dashboard.png :alt: Kubernetes dashboard ("openedx" namespace) -The same ``tutor k8s quickstart`` command can be used to upgrade the cluster to the latest version. +The same ``tutor k8s launch`` command can be used to upgrade the cluster to the latest version. Other commands -------------- diff --git a/docs/local.rst b/docs/local.rst index 4e64a01..36e7d4f 100644 --- a/docs/local.rst +++ b/docs/local.rst @@ -18,7 +18,7 @@ In the following, environment and data files will be generated in a user-specifi tutor run ... .. note:: - As of v10.0.0, a locally-running Open edX platform can no longer be accessed from http://localhost or http://studio.localhost. Instead, when running ``tutor local quickstart``, you must now decide whether you are running a platform that will be used in production. If not, the platform will be automatically be bound to http://local.overhang.io and http://studio.local.overhang.io, which are domain names that point to 127.0.0.1 (localhost). This change was made to facilitate internal communication between Docker containers. + As of v10.0.0, a locally-running Open edX platform can no longer be accessed from http://localhost or http://studio.localhost. Instead, when running ``tutor local launch``, you must now decide whether you are running a platform that will be used in production. If not, the platform will be automatically be bound to http://local.overhang.io and http://studio.local.overhang.io, which are domain names that point to 127.0.0.1 (localhost). This change was made to facilitate internal communication between Docker containers. Main commands ------------- @@ -32,9 +32,9 @@ All-in-one command A fully-functional platform can be configured and run in one command:: - tutor local quickstart + tutor local launch -But you may want to run commands one at a time: it's faster when you need to run only part of the local deployment process, and it helps you understand how your platform works. In the following, we decompose the ``quickstart`` command. +But you may want to run commands one at a time: it's faster when you need to run only part of the local deployment process, and it helps you understand how your platform works. In the following, we decompose the ``launch`` command. Configuration ~~~~~~~~~~~~~ diff --git a/docs/plugins/intro.rst b/docs/plugins/intro.rst index c848923..1001df5 100644 --- a/docs/plugins/intro.rst +++ b/docs/plugins/intro.rst @@ -11,7 +11,7 @@ Tutor comes with a plugin system that allows anyone to customise the deployment # 2) Enable the plugin tutor plugins enable myapp # 3) Reconfigure and restart the platform - tutor local quickstart + tutor local launch For simple changes, it may be extremely easy to create a Tutor plugin: even non-technical users may get started with our :ref:`plugin_development_tutorial` tutorial. We also provide a list of :ref:`simple example plugins `. diff --git a/docs/plugins/v0/gettingstarted.rst b/docs/plugins/v0/gettingstarted.rst index f9ec0de..8fdaaf4 100644 --- a/docs/plugins/v0/gettingstarted.rst +++ b/docs/plugins/v0/gettingstarted.rst @@ -51,7 +51,7 @@ You should be able to view your changes in every LMS and CMS settings file:: Now just restart your platform to start sending tracking events to Google Analytics:: - tutor local quickstart + tutor local launch That's it! And it's very easy to share your plugins. Just upload them to your Github repo and share the url with other users. They will be able to install your plugin by running:: diff --git a/docs/quickstart.rst b/docs/quickstart.rst index e321a29..5fb7f76 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -1,4 +1,4 @@ -.. _quickstart: +.. _launch: Quickstart (1-click install) ---------------------------- @@ -11,12 +11,12 @@ Or `download `_ the pre-compiled b .. include:: download/binary.rst -2. Run ``tutor local quickstart`` +2. Run ``tutor local launch`` 3. You're done! **That's it?** -Yes :) This is what happens when you run ``tutor local quickstart``: +Yes :) This is what happens when you run ``tutor local launch``: 1. You answer a few questions about the :ref:`configuration` of your Open edX platform. 2. Configuration files are generated from templates. @@ -26,4 +26,4 @@ Yes :) This is what happens when you run ``tutor local quickstart``: The whole procedure should require less than 10 minutes, on a server with good bandwidth. Note that your host environment will not be affected in any way, since everything runs inside docker containers. Root access is not even necessary. -There's a lot more to Tutor than that! To learn more about what you can do with Tutor and Open edX, check out the :ref:`whatnext` section. If the quickstart installation method above somehow didn't work for you, check out the :ref:`troubleshooting` guide. +There's a lot more to Tutor than that! To learn more about what you can do with Tutor and Open edX, check out the :ref:`whatnext` section. If the launch installation method above somehow didn't work for you, check out the :ref:`troubleshooting` guide. diff --git a/docs/troubleshooting.rst b/docs/troubleshooting.rst index 7661c55..13d605b 100644 --- a/docs/troubleshooting.rst +++ b/docs/troubleshooting.rst @@ -63,15 +63,15 @@ If the above command does not work, you should fix your Docker installation. Som "Running migrations... Killed!" / "Command failed with status 137: docker-compose" ---------------------------------------------------------------------------------- -Open edX requires at least 4 GB RAM, in particular, to run the SQL migrations. If the ``tutor local quickstart`` command dies after displaying "Running migrations", you most probably need to buy more memory or add swap to your machine. +Open edX requires at least 4 GB RAM, in particular, to run the SQL migrations. If the ``tutor local launch`` command dies after displaying "Running migrations", you most probably need to buy more memory or add swap to your machine. -On macOS, by default, Docker allocates at most 2 GB of RAM to containers. ``quickstart`` tries to check your current allocation and outputs a warning if it can't find a value of at least 4 GB. You should follow `these instructions from the official Docker documentation `__ to allocate at least 4-5 GB to the Docker daemon. +On macOS, by default, Docker allocates at most 2 GB of RAM to containers. ``launch`` tries to check your current allocation and outputs a warning if it can't find a value of at least 4 GB. You should follow `these instructions from the official Docker documentation `__ to allocate at least 4-5 GB to the Docker daemon. -If migrations were killed halfway, there is a good chance that the MySQL database is in a state that is hard to recover from. The easiest way to recover is simply to delete all the MySQL data and restart the quickstart process. After you have allocated more memory to the Docker daemon, run:: +If migrations were killed halfway, there is a good chance that the MySQL database is in a state that is hard to recover from. The easiest way to recover is simply to delete all the MySQL data and restart the launch process. After you have allocated more memory to the Docker daemon, run:: tutor local stop sudo rm -rf "$(tutor config printroot)/data/mysql" - tutor local quickstart + tutor local launch .. warning:: THIS WILL ERASE ALL YOUR DATA! Do not run this on a production instance. This solution is only viable for new Open edX installations. diff --git a/docs/tutorials/google-smtp.rst b/docs/tutorials/google-smtp.rst index 7a27ede..2352d71 100644 --- a/docs/tutorials/google-smtp.rst +++ b/docs/tutorials/google-smtp.rst @@ -34,7 +34,7 @@ Don't forget to replace your email address and password in the prompt above. If Then, restart your platform:: - $ tutor local quickstart + $ tutor local launch That's it! You can send a test email with the following command:: diff --git a/docs/tutorials/nightly.rst b/docs/tutorials/nightly.rst index d963518..0418a9f 100644 --- a/docs/tutorials/nightly.rst +++ b/docs/tutorials/nightly.rst @@ -22,7 +22,7 @@ In addition to installing Tutor Nightly itself, this will install automatically Once Tutor Nightly is installed, you can run the usual ``tutor`` commands:: - tutor dev quickstart + tutor dev launch tutor dev run lms bash # ... and so on diff --git a/docs/tutorials/oldreleases.rst b/docs/tutorials/oldreleases.rst index 8bb14ac..b7344f2 100644 --- a/docs/tutorials/oldreleases.rst +++ b/docs/tutorials/oldreleases.rst @@ -4,9 +4,9 @@ Upgrading from older releases Upgrading from v3+ ~~~~~~~~~~~~~~~~~~ -Just upgrade Tutor using your :ref:`favorite installation method ` and run quickstart again:: +Just upgrade Tutor using your :ref:`favorite installation method ` and run launch again:: - tutor local quickstart + tutor local launch Upgrading from v1 or v2 ~~~~~~~~~~~~~~~~~~~~~~~ @@ -22,4 +22,4 @@ Then, install Tutor using one of the :ref:`installation methods `. Then Finally, launch your platform with:: - tutor local quickstart + tutor local launch diff --git a/docs/tutorials/plugin.rst b/docs/tutorials/plugin.rst index 158d2e6..c4ca5b0 100644 --- a/docs/tutorials/plugin.rst +++ b/docs/tutorials/plugin.rst @@ -218,7 +218,7 @@ You can now run the "myservice" container which will execute the ``CMD`` stateme Declaring initialisation tasks ------------------------------ -Services often need to run specific tasks before they can be started. For instance, the LMS and the CMS need to apply database migrations. These commands are written in shell scripts that are executed whenever we run ``quickstart``. We call these scripts "init tasks". To add a new local init task, we must first add the corresponding service to the ``docker-compose-jobs.yml`` file by implementing the :patch:`local-docker-compose-jobs-services` patch:: +Services often need to run specific tasks before they can be started. For instance, the LMS and the CMS need to apply database migrations. These commands are written in shell scripts that are executed whenever we run ``launch``. We call these scripts "init tasks". To add a new local init task, we must first add the corresponding service to the ``docker-compose-jobs.yml`` file by implementing the :patch:`local-docker-compose-jobs-services` patch:: hooks.Filters.ENV_PATCHES.add_item( ( diff --git a/docs/whatnext.rst b/docs/whatnext.rst index f185605..37ce521 100644 --- a/docs/whatnext.rst +++ b/docs/whatnext.rst @@ -3,7 +3,7 @@ What next? ========== -You have gone through the :ref:`Quickstart installation `: at this point, you should have a running Open edX platform. If you don't, please follow the instructions from the :ref:`Troubleshooting ` section. +You have gone through the :ref:`Launch installation `: at this point, you should have a running Open edX platform. If you don't, please follow the instructions from the :ref:`Troubleshooting ` section. Logging-in as administrator --------------------------- diff --git a/tests/commands/test_local.py b/tests/commands/test_local.py index abb8443..97082a4 100644 --- a/tests/commands/test_local.py +++ b/tests/commands/test_local.py @@ -14,8 +14,8 @@ class LocalTests(unittest.TestCase, TestCommandMixin): self.assertIsNone(result.exception) self.assertEqual(0, result.exit_code) - def test_local_quickstart_help(self) -> None: - result = self.invoke(["local", "quickstart", "--help"]) + def test_local_launch_help(self) -> None: + result = self.invoke(["local", "launch", "--help"]) self.assertIsNone(result.exception) self.assertEqual(0, result.exit_code) diff --git a/tutor/commands/dev.py b/tutor/commands/dev.py index 4194f56..ddb2b67 100644 --- a/tutor/commands/dev.py +++ b/tutor/commands/dev.py @@ -66,7 +66,7 @@ def dev(context: click.Context) -> None: @click.option("-p", "--pullimages", is_flag=True, help="Update docker images") @compose.mount_option @click.pass_context -def quickstart( +def launch( context: click.Context, non_interactive: bool, pullimages: bool, @@ -120,6 +120,28 @@ Your Open edX platform is ready and can be accessed at the following urls: ) +@click.command(help="Configure and run Open edX from scratch, for development") +@click.option("-I", "--non-interactive", is_flag=True, help="Run non-interactively") +@click.option("-p", "--pullimages", is_flag=True, help="Update docker images") +@compose.mount_option +@click.pass_context +def quickstart( + context: click.Context, + non_interactive: bool, + pullimages: bool, + mounts: t.Tuple[t.List[compose.MountParam.MountType]], +) -> None: + """ + This command has been renamed to 'launch'. + """ + fmt.echo_alert( + "The 'quickstart' command is deprecated and will be removed in a later release. Use 'launch' instead." + ) + context.invoke( + launch, non_interactive=non_interactive, pullimages=pullimages, mounts=mounts + ) + + @hooks.Actions.COMPOSE_PROJECT_STARTED.add() def _stop_on_local_start(root: str, config: Config, project_name: str) -> None: """ @@ -131,5 +153,6 @@ def _stop_on_local_start(root: str, config: Config, project_name: str) -> None: runner.docker_compose("stop") +dev.add_command(launch) dev.add_command(quickstart) compose.add_commands(dev) diff --git a/tutor/commands/k8s.py b/tutor/commands/k8s.py index 0a855dc..19a1721 100644 --- a/tutor/commands/k8s.py +++ b/tutor/commands/k8s.py @@ -163,7 +163,7 @@ def k8s(context: click.Context) -> None: @click.command(help="Configure and run Open edX from scratch") @click.option("-I", "--non-interactive", is_flag=True, help="Run non-interactively") @click.pass_context -def quickstart(context: click.Context, non_interactive: bool) -> None: +def launch(context: click.Context, non_interactive: bool) -> None: run_upgrade_from_release = tutor_env.should_upgrade_from_release(context.obj.root) if run_upgrade_from_release is not None: click.echo(fmt.title("Upgrading from an older release")) @@ -214,6 +214,19 @@ Press enter when you are ready to continue""" ) +@click.command(help="Configure and run Open edX from scratch") +@click.option("-I", "--non-interactive", is_flag=True, help="Run non-interactively") +@click.pass_context +def quickstart(context: click.Context, non_interactive: bool) -> None: + """ + This command has been renamed to 'launch'. + """ + fmt.echo_alert( + "The 'quickstart' command is deprecated and will be removed in a later release. Use 'launch' instead." + ) + context.invoke(launch, non_interactive=non_interactive) + + @click.command( short_help="Run all configured Open edX resources", help=( @@ -463,7 +476,7 @@ def wait(context: K8sContext, name: str) -> None: @click.command( short_help="Perform release-specific upgrade tasks", - help="Perform release-specific upgrade tasks. To perform a full upgrade remember to run `quickstart`.", + help="Perform release-specific upgrade tasks. To perform a full upgrade remember to run `launch`.", ) @click.option( "--from", @@ -479,7 +492,7 @@ def upgrade(context: click.Context, from_release: Optional[str]) -> None: else: fmt.echo_alert( "This command only performs a partial upgrade of your Open edX platform. " - "To perform a full upgrade, you should run `tutor k8s quickstart`." + "To perform a full upgrade, you should run `tutor k8s launch`." ) upgrade_from(context.obj, from_release) # We update the environment to update the version @@ -569,6 +582,7 @@ def k8s_namespace(config: Config) -> str: return get_typed(config, "K8S_NAMESPACE", str) +k8s.add_command(launch) k8s.add_command(quickstart) k8s.add_command(start) k8s.add_command(stop) diff --git a/tutor/commands/local.py b/tutor/commands/local.py index e6aae8b..9a0877f 100644 --- a/tutor/commands/local.py +++ b/tutor/commands/local.py @@ -67,7 +67,7 @@ def local(context: click.Context) -> None: @click.option("-I", "--non-interactive", is_flag=True, help="Run non-interactively") @click.option("-p", "--pullimages", is_flag=True, help="Update docker images") @click.pass_context -def quickstart( +def launch( context: click.Context, mounts: t.Tuple[t.List[compose.MountParam.MountType]], non_interactive: bool, @@ -159,9 +159,31 @@ Your Open edX platform is ready and can be accessed at the following urls: ) +@click.command(help="Configure and run Open edX from scratch") +@compose.mount_option +@click.option("-I", "--non-interactive", is_flag=True, help="Run non-interactively") +@click.option("-p", "--pullimages", is_flag=True, help="Update docker images") +@click.pass_context +def quickstart( + context: click.Context, + mounts: t.Tuple[t.List[compose.MountParam.MountType]], + non_interactive: bool, + pullimages: bool, +) -> None: + """ + This command has been renamed to 'launch'. + """ + fmt.echo_alert( + "The 'quickstart' command is deprecated and will be removed in a later release. Use 'launch' instead." + ) + context.invoke( + launch, non_interactive=non_interactive, pullimages=pullimages, mounts=mounts + ) + + @click.command( short_help="Perform release-specific upgrade tasks", - help="Perform release-specific upgrade tasks. To perform a full upgrade remember to run `quickstart`.", + help="Perform release-specific upgrade tasks. To perform a full upgrade remember to run `launch`.", ) @click.option( "--from", @@ -172,7 +194,7 @@ Your Open edX platform is ready and can be accessed at the following urls: def upgrade(context: click.Context, from_release: t.Optional[str]) -> None: fmt.echo_alert( "This command only performs a partial upgrade of your Open edX platform. " - "To perform a full upgrade, you should run `tutor local quickstart`." + "To perform a full upgrade, you should run `tutor local launch`." ) if from_release is None: from_release = tutor_env.get_env_release(context.obj.root) @@ -195,6 +217,7 @@ def _stop_on_dev_start(root: str, config: Config, project_name: str) -> None: runner.docker_compose("stop") +local.add_command(launch) local.add_command(quickstart) local.add_command(upgrade) compose.add_commands(local) diff --git a/tutor/config.py b/tutor/config.py index 89d545a..d9c94d8 100644 --- a/tutor/config.py +++ b/tutor/config.py @@ -19,7 +19,7 @@ def load(root: str) -> Config: raise exceptions.TutorError( "Project root does not exist. Make sure to generate the initial " "configuration with `tutor config save --interactive` or `tutor local " - "quickstart` prior to running other commands." + "launch` prior to running other commands." ) env.check_is_up_to_date(root) return load_full(root) From 2f442d7db304ee53097c56df68a2f8d2b398a541 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Thu, 29 Sep 2022 18:18:36 +0200 Subject: [PATCH 46/56] fix: edx-platform requirement installation The local requirements files does not exist since local requirements were all removed from the edx-platform repo. As a consequence, the nightly build was broken. --- tutor/templates/build/openedx/Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index 4c2dc54..0a5ccbd 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -148,9 +148,6 @@ ENV PATH /openedx/venv/bin:./node_modules/.bin:/openedx/nodeenv/bin:${PATH} ENV VIRTUAL_ENV /openedx/venv/ WORKDIR /openedx/edx-platform -# Re-install local requirements, otherwise egg-info folders are missing -RUN pip install -r requirements/edx/local.in - # Create folder that will store lms/cms.env.yml files, as well as # the tutor-specific settings files. RUN mkdir -p /openedx/config ./lms/envs/tutor ./cms/envs/tutor From a6c6fde86767ab315a6ed3cddc6f88d6e600bf2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Mon, 3 Oct 2022 12:00:11 +0200 Subject: [PATCH 47/56] fix: installation of local requirements The `compilejsi18n` command was failing during image building because the Open-edX package was not installed properly. The reason for that was an earlier change where we got rid of the `pip install -r requirements/edx/local.in` command. Installing the Open-edX package was part of this requirement file. The local.in requirements file no longer exists, but we still need to `pip install -e .` the edx-platform repo. To run this command we need both the edx-platform repo and the virtualenv. The good news is that there are no more local requirements in the base.txt requirements file. This means that we no longer have to COPY the edx-platform repo in the requirements installation step. Thus, changes in edx-platform will no longer trigger a rebuild of the pip requirements; this means that re-builds will be much faster when making changes to edx-platform. Note that plugins that implemented the "openedx-dockerfile-post-python-requirements" patch and that needed access to the edx-platform repo will no longer work. Instead, these plugins should implement the "openedx-dockerfile-pre-assets" patch. This scenario should be very rare, though. Close #726 --- CHANGELOG-nightly.md | 1 + tutor/templates/build/openedx/Dockerfile | 15 ++++++--------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/CHANGELOG-nightly.md b/CHANGELOG-nightly.md index bf85f74..5f89eb7 100644 --- a/CHANGELOG-nightly.md +++ b/CHANGELOG-nightly.md @@ -8,6 +8,7 @@ will be backported to the master branch at every major release. When backporting changes to master, we should keep only the entries that correspond to user- facing changes. --> +- 💥[Bugfix] Fix local installation requirements. Plugins that implemented the "openedx-dockerfile-post-python-requirements" patch and that needed access to the edx-platform repo will no longer work. Instead, these plugins should implement the "openedx-dockerfile-pre-assets" patch. This scenario should be very rare, though. (by @regisb) - 💥[Improvement] Rename the implementation of tutor quickstart to tutor launch. (by @Carlos-Muniz) - 💥[Improvement] Remove the implementation of tutor dev runserver. (by @Carlos-Muniz) - [Bugfix] Fix MongoDB replica set connection error resulting from edx-platform's pymongo (3.10.1 -> 3.12.3) upgrade ([edx-platform#30569](https://github.com/openedx/edx-platform/pull/30569)). (by @ormsbee) diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index 0a5ccbd..4e8dd8b 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -69,13 +69,6 @@ ENV VIRTUAL_ENV /openedx/venv/ RUN apt update && apt install -y software-properties-common libmysqlclient-dev libxmlsec1-dev libgeos-dev -# Note that this means that we need to reinstall all requirements whenever there is a -# change in edx-platform, which sucks. Yet, we must do it, because edx-platform installs some -# Python projects from within the edx-platform repo itself. This is being fixed upstream. -# TODO: https://github.com/overhangio/2u-tutor-adoption/issues/86 -COPY --from=code /openedx/edx-platform /openedx/edx-platform -WORKDIR /openedx/edx-platform - # Install the right version of pip/setuptools # https://pypi.org/project/setuptools/ # https://pypi.org/project/pip/ @@ -83,8 +76,8 @@ WORKDIR /openedx/edx-platform RUN pip install setuptools==62.1.0 pip==22.0.4 wheel==0.37.1 # Install base requirements -RUN pip install -r ./requirements/edx/base.txt -RUN pip install -e . +COPY --from=code /openedx/edx-platform/requirements/edx/base.txt /tmp/base.txt +RUN pip install -r /tmp/base.txt # Install django-redis for using redis as a django cache # https://pypi.org/project/django-redis/ @@ -148,6 +141,10 @@ ENV PATH /openedx/venv/bin:./node_modules/.bin:/openedx/nodeenv/bin:${PATH} ENV VIRTUAL_ENV /openedx/venv/ WORKDIR /openedx/edx-platform +# We install edx-platform here because it creates an egg-info folder in the current +# repo. We need both the source code and the virtualenv to run this command. +RUN pip install -e . + # Create folder that will store lms/cms.env.yml files, as well as # the tutor-specific settings files. RUN mkdir -p /openedx/config ./lms/envs/tutor ./cms/envs/tutor From 3de28377a17b1e218c66fb7e860803d581784db4 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Mon, 3 Oct 2022 15:47:59 -0400 Subject: [PATCH 48/56] docs: in quickstart's helptext, note that it's renamed to 'launch' --- tutor/commands/dev.py | 2 +- tutor/commands/local.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tutor/commands/dev.py b/tutor/commands/dev.py index ddb2b67..46dfb05 100644 --- a/tutor/commands/dev.py +++ b/tutor/commands/dev.py @@ -120,7 +120,7 @@ Your Open edX platform is ready and can be accessed at the following urls: ) -@click.command(help="Configure and run Open edX from scratch, for development") +@click.command(help="Deprecated alias to 'launch'") @click.option("-I", "--non-interactive", is_flag=True, help="Run non-interactively") @click.option("-p", "--pullimages", is_flag=True, help="Update docker images") @compose.mount_option diff --git a/tutor/commands/local.py b/tutor/commands/local.py index 9a0877f..e4614c3 100644 --- a/tutor/commands/local.py +++ b/tutor/commands/local.py @@ -159,7 +159,7 @@ Your Open edX platform is ready and can be accessed at the following urls: ) -@click.command(help="Configure and run Open edX from scratch") +@click.command(help="Deprecated alias to 'launch'") @compose.mount_option @click.option("-I", "--non-interactive", is_flag=True, help="Run non-interactively") @click.option("-p", "--pullimages", is_flag=True, help="Update docker images") From e56918bf4796c821d240eb9314758c77690a6566 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Tue, 18 Oct 2022 14:40:32 +0200 Subject: [PATCH 49/56] depr: get rid of the `local/dev bindmount` commands This command has always been clunky. It is now removed in favour of the `-m/--mount` option. Close https://github.com/overhangio/2u-tutor-adoption/issues/88 Close https://github.com/overhangio/2u-tutor-adoption/issues/89 --- CHANGELOG-nightly.md | 1 + docs/dev.rst | 18 ------------------ tests/commands/test_dev.py | 5 ----- tests/test_bindmounts.py | 36 ----------------------------------- tutor/bindmounts.py | 22 --------------------- tutor/commands/compose.py | 39 +------------------------------------- 6 files changed, 2 insertions(+), 119 deletions(-) delete mode 100644 tests/test_bindmounts.py diff --git a/CHANGELOG-nightly.md b/CHANGELOG-nightly.md index 5f89eb7..ee725c9 100644 --- a/CHANGELOG-nightly.md +++ b/CHANGELOG-nightly.md @@ -8,6 +8,7 @@ will be backported to the master branch at every major release. When backporting changes to master, we should keep only the entries that correspond to user- facing changes. --> +- 💥[Improvement] Remove the `local/dev bindmount` commands, which have been marked as deprecated for some time. The `--mount` option should be used instead. - 💥[Bugfix] Fix local installation requirements. Plugins that implemented the "openedx-dockerfile-post-python-requirements" patch and that needed access to the edx-platform repo will no longer work. Instead, these plugins should implement the "openedx-dockerfile-pre-assets" patch. This scenario should be very rare, though. (by @regisb) - 💥[Improvement] Rename the implementation of tutor quickstart to tutor launch. (by @Carlos-Muniz) - 💥[Improvement] Remove the implementation of tutor dev runserver. (by @Carlos-Muniz) diff --git a/docs/dev.rst b/docs/dev.rst index d39eefc..d52457f 100644 --- a/docs/dev.rst +++ b/docs/dev.rst @@ -152,24 +152,6 @@ Then, bind-mount that folder back in the container with the ``--mount`` option ( You can then edit the files in ``~/venv`` on your local filesystem and see the changes live in your container. -Bind-mount from the "volumes/" directory -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. warning:: Bind-mounting volumes with the ``bindmount`` command is no longer the default, recommended way of bind-mounting volumes from the host. Instead, see the :ref:`mount option ` and the ``tutor dev/local copyfrom`` commands. - -Tutor makes it easy to create a bind-mount from an existing container. First, copy the contents of a container directory with the ``bindmount`` command. For instance, to copy the virtual environment of the "lms" container:: - - tutor dev bindmount lms /openedx/venv - -This command recursively copies the contents of the ``/opendedx/venv`` directory to ``$(tutor config printroot)/volumes/venv``. The code of any Python dependency can then be edited -- for instance, you can then add a ``breakpoint()`` statement for step-by-step debugging, or implement a custom feature. - -Then, bind-mount the directory back in the container with the ``--mount`` option:: - - tutor dev start --mount=lms:$(tutor config printroot)/volumes/venv:/openedx/venv lms - -.. note:: - The ``bindmount`` command and the ``--mount=...`` option syntax are available both for the ``tutor local`` and ``tutor dev`` commands. - Manual bind-mount to any directory ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/tests/commands/test_dev.py b/tests/commands/test_dev.py index 2f911f5..0b962d7 100644 --- a/tests/commands/test_dev.py +++ b/tests/commands/test_dev.py @@ -8,8 +8,3 @@ class DevTests(unittest.TestCase, TestCommandMixin): result = self.invoke(["dev", "--help"]) self.assertEqual(0, result.exit_code) self.assertIsNone(result.exception) - - def test_dev_bindmount(self) -> None: - result = self.invoke(["dev", "bindmount", "--help"]) - self.assertEqual(0, result.exit_code) - self.assertIsNone(result.exception) diff --git a/tests/test_bindmounts.py b/tests/test_bindmounts.py deleted file mode 100644 index dbc4b4b..0000000 --- a/tests/test_bindmounts.py +++ /dev/null @@ -1,36 +0,0 @@ -import unittest - -from tutor import bindmounts -from tutor.exceptions import TutorError - - -class BindMountsTests(unittest.TestCase): - def test_get_name(self) -> None: - self.assertEqual("venv", bindmounts.get_name("/openedx/venv")) - self.assertEqual("venv", bindmounts.get_name("/openedx/venv/")) - - def test_get_name_root_folder(self) -> None: - with self.assertRaises(TutorError): - bindmounts.get_name("/") - with self.assertRaises(TutorError): - bindmounts.get_name("") - - def test_parse_volumes(self) -> None: - volume_args, non_volume_args = bindmounts.parse_volumes( - [ - "run", - "--volume=/openedx/venv", - "-v", - "/tmp/openedx:/openedx", - "lms", - "echo", - "boom", - ] - ) - self.assertEqual(("/openedx/venv", "/tmp/openedx:/openedx"), volume_args) - self.assertEqual(("run", "lms", "echo", "boom"), non_volume_args) - - def test_parse_volumes_empty_list(self) -> None: - volume_args, non_volume_args = bindmounts.parse_volumes([]) - self.assertEqual((), volume_args) - self.assertEqual((), non_volume_args) diff --git a/tutor/bindmounts.py b/tutor/bindmounts.py index 874507d..4bc156e 100644 --- a/tutor/bindmounts.py +++ b/tutor/bindmounts.py @@ -1,7 +1,4 @@ import os -from typing import List, Tuple - -import click from .exceptions import TutorError from .jobs import BaseComposeJobRunner @@ -62,22 +59,3 @@ def get_name(container_bind_path: str) -> str: def get_root_path(root: str) -> str: return os.path.join(root, "volumes") - - -def parse_volumes(docker_compose_args: List[str]) -> Tuple[List[str], List[str]]: - """ - Parse `-v/--volume` options from an arbitrary list of arguments. - """ - - @click.command(context_settings={"ignore_unknown_options": True}) - @click.option("-v", "--volume", "volumes", multiple=True) - @click.argument("args", nargs=-1) - def custom_docker_compose( - volumes: List[str], args: List[str] # pylint: disable=unused-argument - ) -> None: - pass - - if isinstance(docker_compose_args, tuple): - docker_compose_args = list(docker_compose_args) - context = custom_docker_compose.make_context("custom", docker_compose_args) - return context.params["volumes"], context.params["args"] diff --git a/tutor/commands/compose.py b/tutor/commands/compose.py index 31d1910..6b62b6f 100644 --- a/tutor/commands/compose.py +++ b/tutor/commands/compose.py @@ -5,7 +5,6 @@ from copy import deepcopy import click -from tutor import bindmounts from tutor import config as tutor_config from tutor import env as tutor_env from tutor import fmt, hooks, jobs, serialize, utils @@ -392,28 +391,6 @@ def run( context.invoke(dc_command, mounts=mounts, command="run", args=[*extra_args, *args]) -@click.command( - name="bindmount", - help="Copy the contents of a container directory to a ready-to-bind-mount host directory", -) -@click.argument("service") -@click.argument("path") -@click.pass_obj -def bindmount_command(context: BaseComposeContext, service: str, path: str) -> None: - """ - This command is made obsolete by the --mount arguments. - """ - fmt.echo_alert( - "The 'bindmount' command is deprecated and will be removed in a later release. Use 'copyfrom' instead." - ) - config = tutor_config.load(context.root) - host_path = bindmounts.create(context.job_runner(config), service, path) - fmt.echo_info( - f"Bind-mount volume created at {host_path}. You can now use it in all `local` and `dev` " - f"commands with the `--volume={path}` option." - ) - - @click.command( name="copyfrom", help="Copy files/folders from a container directory to the local filesystem.", @@ -520,20 +497,7 @@ def dc_command( ) -> None: mount_tmp_volumes(mounts, context) config = tutor_config.load(context.root) - volumes, non_volume_args = bindmounts.parse_volumes(args) - volume_args = [] - for volume_arg in volumes: - if ":" not in volume_arg: - # This is a bind-mounted volume from the "volumes/" folder. - host_bind_path = bindmounts.get_path(context.root, volume_arg) - if not os.path.exists(host_bind_path): - raise TutorError( - f"Bind-mount volume directory {host_bind_path} does not exist. It must first be created " - f"with the '{bindmount_command.name}' command." - ) - volume_arg = f"{host_bind_path}:{volume_arg}" - volume_args += ["--volume", volume_arg] - context.job_runner(config).docker_compose(command, *volume_args, *non_volume_args) + context.job_runner(config).docker_compose(command, *args) @hooks.Filters.COMPOSE_MOUNTS.add() @@ -569,7 +533,6 @@ def add_commands(command_group: click.Group) -> None: command_group.add_command(dc_command) command_group.add_command(run) command_group.add_command(copyfrom) - command_group.add_command(bindmount_command) command_group.add_command(execute) command_group.add_command(logs) command_group.add_command(status) From e734f52f07a5626eb006def31dedb092b8068850 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Tue, 8 Nov 2022 16:09:16 +0100 Subject: [PATCH 50/56] feat: filter priorities Nothing revolutionary here, we just implement the same priority queue that existed in actions. It will be necessary to trigger init tasks in the right order. --- CHANGELOG-nightly.md | 1 + tutor/hooks/actions.py | 22 ++++++-------------- tutor/hooks/filters.py | 43 ++++++++++++++++++++++++--------------- tutor/hooks/priorities.py | 25 +++++++++++++++++++++++ 4 files changed, 59 insertions(+), 32 deletions(-) create mode 100644 tutor/hooks/priorities.py diff --git a/CHANGELOG-nightly.md b/CHANGELOG-nightly.md index ee725c9..26a7343 100644 --- a/CHANGELOG-nightly.md +++ b/CHANGELOG-nightly.md @@ -8,6 +8,7 @@ will be backported to the master branch at every major release. When backporting changes to master, we should keep only the entries that correspond to user- facing changes. --> +- [Feature] Implement hook filter priorities, which work like action priorities. (by @regisb) - 💥[Improvement] Remove the `local/dev bindmount` commands, which have been marked as deprecated for some time. The `--mount` option should be used instead. - 💥[Bugfix] Fix local installation requirements. Plugins that implemented the "openedx-dockerfile-post-python-requirements" patch and that needed access to the edx-platform repo will no longer work. Instead, these plugins should implement the "openedx-dockerfile-pre-assets" patch. This scenario should be very rare, though. (by @regisb) - 💥[Improvement] Rename the implementation of tutor quickstart to tutor launch. (by @Carlos-Muniz) diff --git a/tutor/hooks/actions.py b/tutor/hooks/actions.py index 83d73fe..2473ba7 100644 --- a/tutor/hooks/actions.py +++ b/tutor/hooks/actions.py @@ -4,14 +4,13 @@ __license__ = "Apache 2.0" import sys import typing as t +from . import priorities from .contexts import Contextualized # Similarly to CallableFilter, it should be possible to refine the definition of # CallableAction in the future. CallableAction = t.Callable[..., None] -DEFAULT_PRIORITY = 10 - class ActionCallback(Contextualized): def __init__( @@ -21,7 +20,7 @@ class ActionCallback(Contextualized): ): super().__init__() self.func = func - self.priority = priority or DEFAULT_PRIORITY + self.priority = priority or priorities.DEFAULT def do( self, *args: t.Any, context: t.Optional[str] = None, **kwargs: t.Any @@ -57,15 +56,7 @@ class Action: ) -> t.Callable[[CallableAction], CallableAction]: def inner(func: CallableAction) -> CallableAction: callback = ActionCallback(func, priority=priority) - # I wish we could use bisect.insort_right here but the `key=` parameter - # is unsupported in Python 3.9 - position = 0 - while ( - position < len(self.callbacks) - and self.callbacks[position].priority <= callback.priority - ): - position += 1 - self.callbacks.insert(position, callback) + priorities.insert_callback(callback, self.callbacks) return func return inner @@ -128,8 +119,7 @@ def get_template(name: str) -> ActionTemplate: def add( - name: str, - priority: t.Optional[int] = None, + name: str, priority: t.Optional[int] = None ) -> t.Callable[[CallableAction], CallableAction]: """ Decorator to add a callback action associated to a name. @@ -139,8 +129,8 @@ def add( :py:class:`tutor.hooks.Actions` instead. :param priority: optional order in which the action callbacks are performed. Higher values mean that they will be performed later. The default value is - ``DEFAULT_PRIORITY`` (10). Actions that should be performed last should - have a priority of 100. + ``priorities.DEFAULT`` (10). Actions that should be performed last should have a + priority of 100. Usage:: diff --git a/tutor/hooks/filters.py b/tutor/hooks/filters.py index a0b1d8a..a15df0b 100644 --- a/tutor/hooks/filters.py +++ b/tutor/hooks/filters.py @@ -4,7 +4,7 @@ __license__ = "Apache 2.0" import sys import typing as t -from . import contexts +from . import contexts, priorities # For now, this signature is not very restrictive. In the future, we could improve it by writing: # @@ -22,9 +22,10 @@ CallableFilter = t.Callable[..., t.Any] class FilterCallback(contexts.Contextualized): - def __init__(self, func: CallableFilter): + def __init__(self, func: CallableFilter, priority: t.Optional[int] = None): super().__init__() self.func = func + self.priority = priority or priorities.DEFAULT def apply( self, value: T, *args: t.Any, context: t.Optional[str] = None, **kwargs: t.Any @@ -36,7 +37,7 @@ class FilterCallback(contexts.Contextualized): class Filter: """ - Each filter is associated to a name and a list of callbacks. + Each filter is associated to a name and a list of callbacks, sorted by priority. """ INDEX: t.Dict[str, "Filter"] = {} @@ -55,18 +56,21 @@ class Filter: """ return cls.INDEX.setdefault(name, cls(name)) - def add(self) -> t.Callable[[CallableFilter], CallableFilter]: + def add( + self, priority: t.Optional[int] = None + ) -> t.Callable[[CallableFilter], CallableFilter]: def inner(func: CallableFilter) -> CallableFilter: - self.callbacks.append(FilterCallback(func)) + callback = FilterCallback(func, priority=priority) + priorities.insert_callback(callback, self.callbacks) return func return inner - def add_item(self, item: T) -> None: - self.add_items([item]) + def add_item(self, item: T, priority: t.Optional[int] = None) -> None: + self.add_items([item], priority=priority) - def add_items(self, items: t.List[T]) -> None: - @self.add() + def add_items(self, items: t.List[T], priority: t.Optional[int] = None) -> None: + @self.add(priority=priority) def callback(value: t.List[T], *_args: t.Any, **_kwargs: t.Any) -> t.List[T]: return value + items @@ -153,11 +157,17 @@ def get_template(name: str) -> FilterTemplate: return FilterTemplate(name) -def add(name: str) -> t.Callable[[CallableFilter], CallableFilter]: +def add( + name: str, priority: t.Optional[int] = None +) -> t.Callable[[CallableFilter], CallableFilter]: """ Decorator for functions that will be applied to a single named filter. - :param name: name of the filter to which the decorated function should be added. + :param str name: name of the filter to which the decorated function should be added. + :param int priority: optional order in which the filter callbacks are called. Higher + values mean that they will be performed later. The default value is + ``priorities.DEFAULT`` (10). Filters that should be called last should have a + priority of 100. The return value of each filter function callback will be passed as the first argument to the next one. @@ -174,15 +184,16 @@ def add(name: str) -> t.Callable[[CallableFilter], CallableFilter]: # After filters have been created, the result of calling all filter callbacks is obtained by running: hooks.filters.apply("my-filter", initial_value, some_other_argument_value) """ - return Filter.get(name).add() + return Filter.get(name).add(priority=priority) -def add_item(name: str, item: T) -> None: +def add_item(name: str, item: T, priority: t.Optional[int] = None) -> None: """ Convenience function to add a single item to a filter that returns a list of items. :param name: filter name. :param object item: item that will be appended to the resulting list. + :param int priority: see :py:data:`add`. Usage:: @@ -193,10 +204,10 @@ def add_item(name: str, item: T) -> None: assert ["item1", "item2"] == hooks.filters.apply("my-filter", []) """ - get(name).add_item(item) + get(name).add_item(item, priority=priority) -def add_items(name: str, items: t.List[T]) -> None: +def add_items(name: str, items: t.List[T], priority: t.Optional[int] = None) -> None: """ Convenience function to add multiple item to a filter that returns a list of items. @@ -211,7 +222,7 @@ def add_items(name: str, items: t.List[T]) -> None: assert ["item1", "item2"] == hooks.filters.apply("my-filter", []) """ - get(name).add_items(items) + get(name).add_items(items, priority=priority) def iterate( diff --git a/tutor/hooks/priorities.py b/tutor/hooks/priorities.py new file mode 100644 index 0000000..3c43d53 --- /dev/null +++ b/tutor/hooks/priorities.py @@ -0,0 +1,25 @@ +import typing as t + +from typing_extensions import Protocol + +HIGH = 5 +DEFAULT = 10 +LOW = 50 + + +class PrioritizedCallback(Protocol): + priority: int + + +TPrioritized = t.TypeVar("TPrioritized", bound=PrioritizedCallback) + + +def insert_callback(callback: TPrioritized, callbacks: t.List[TPrioritized]) -> None: + # I wish we could use bisect.insort_right here but the `key=` parameter + # is unsupported in Python 3.9 + position = 0 + while ( + position < len(callbacks) and callbacks[position].priority <= callback.priority + ): + position += 1 + callbacks.insert(position, callback) From b6dc65cc6488c9c68ed4a9ef4b5327a0128625e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Tue, 18 Oct 2022 16:57:07 +0200 Subject: [PATCH 51/56] refactor: deduplicate jobs code createuser, importdemocourse and settheme were 100% duplicated code between k8s.py and compose.py. --- tests/{ => commands}/test_jobs.py | 25 +-- tutor/commands/compose.py | 68 +------- tutor/commands/jobs.py | 187 +++++++++++++++++++++ tutor/commands/k8s.py | 68 +------- tutor/config.py | 3 +- tutor/env.py | 3 +- tutor/jobs.py | 137 ++------------- tutor/templates/hooks/cms/importdemocourse | 8 - 8 files changed, 216 insertions(+), 283 deletions(-) rename tests/{ => commands}/test_jobs.py (76%) create mode 100644 tutor/commands/jobs.py delete mode 100644 tutor/templates/hooks/cms/importdemocourse diff --git a/tests/test_jobs.py b/tests/commands/test_jobs.py similarity index 76% rename from tests/test_jobs.py rename to tests/commands/test_jobs.py index 7b6ea5d..687fc24 100644 --- a/tests/test_jobs.py +++ b/tests/commands/test_jobs.py @@ -5,7 +5,7 @@ from unittest.mock import patch from tests.helpers import TestContext, temporary_root from tutor import config as tutor_config -from tutor import jobs +from tutor.commands import jobs class JobsTests(unittest.TestCase): @@ -21,26 +21,21 @@ class JobsTests(unittest.TestCase): self.assertTrue(output.endswith("All services initialised.")) def test_create_user_command_without_staff(self) -> None: - command = jobs.create_user_command("superuser", False, "username", "email") + command = jobs.create_user_template("superuser", False, "username", "email", "p4ssw0rd") self.assertNotIn("--staff", command) + self.assertIn("set_password", command) def test_create_user_command_with_staff(self) -> None: - command = jobs.create_user_command("superuser", True, "username", "email") + command = jobs.create_user_template("superuser", True, "username", "email", "p4ssw0rd") self.assertIn("--staff", command) - def test_create_user_command_with_staff_with_password(self) -> None: - command = jobs.create_user_command( - "superuser", True, "username", "email", "command" - ) - self.assertIn("set_password", command) - @patch("sys.stdout", new_callable=StringIO) def test_import_demo_course(self, mock_stdout: StringIO) -> None: with temporary_root() as root: context = TestContext(root) config = tutor_config.load_full(root) runner = context.job_runner(config) - jobs.import_demo_course(runner) + runner.run_job_from_str("cms", jobs.import_demo_course_template()) output = mock_stdout.getvalue() service = re.search(r"Service: (\w*)", output) @@ -60,7 +55,8 @@ class JobsTests(unittest.TestCase): context = TestContext(root) config = tutor_config.load_full(root) runner = context.job_runner(config) - jobs.set_theme("sample_theme", ["domain1", "domain2"], runner) + command = jobs.set_theme_template("sample_theme", ["domain1", "domain2"]) + runner.run_job_from_str("lms", command) output = mock_stdout.getvalue() service = re.search(r"Service: (\w*)", output) @@ -73,10 +69,3 @@ class JobsTests(unittest.TestCase): .strip() .startswith('echo "Loading settings $DJANGO_SETTINGS_MODULE"') ) - - def test_get_all_openedx_domains(self) -> None: - with temporary_root() as root: - config = tutor_config.load_full(root) - domains = jobs.get_all_openedx_domains(config) - self.assertTrue(domains) - self.assertEqual(6, len(domains)) diff --git a/tutor/commands/compose.py b/tutor/commands/compose.py index 6b62b6f..f8000ff 100644 --- a/tutor/commands/compose.py +++ b/tutor/commands/compose.py @@ -7,13 +7,15 @@ import click from tutor import config as tutor_config from tutor import env as tutor_env -from tutor import fmt, hooks, jobs, serialize, utils +from tutor import fmt, hooks, serialize, utils +from tutor.commands import jobs from tutor.commands.context import BaseJobContext from tutor.exceptions import TutorError +from tutor.jobs import BaseComposeJobRunner from tutor.types import Config -class ComposeJobRunner(jobs.BaseComposeJobRunner): +class ComposeJobRunner(BaseComposeJobRunner): def __init__(self, root: str, config: Config): super().__init__(root, config) self.project_name = "" @@ -310,64 +312,6 @@ def init( jobs.initialise(runner, limit_to=limit) -@click.command(help="Create an Open edX user and interactively set their password") -@click.option("--superuser", is_flag=True, help="Make superuser") -@click.option("--staff", is_flag=True, help="Make staff user") -@click.option( - "-p", - "--password", - help="Specify password from the command line. If undefined, you will be prompted to input a password", -) -@click.argument("name") -@click.argument("email") -@click.pass_obj -def createuser( - context: BaseComposeContext, - superuser: str, - staff: bool, - password: str, - name: str, - email: str, -) -> None: - config = tutor_config.load(context.root) - runner = context.job_runner(config) - command = jobs.create_user_command(superuser, staff, name, email, password=password) - runner.run_job("lms", command) - - -@click.command( - help="Assign a theme to the LMS and the CMS. To reset to the default theme , use 'default' as the theme name." -) -@click.option( - "-d", - "--domain", - "domains", - multiple=True, - help=( - "Limit the theme to these domain names. By default, the theme is " - "applied to the LMS and the CMS, both in development and production mode" - ), -) -@click.argument("theme_name") -@click.pass_obj -def settheme( - context: BaseComposeContext, domains: t.List[str], theme_name: str -) -> None: - config = tutor_config.load(context.root) - runner = context.job_runner(config) - domains = domains or jobs.get_all_openedx_domains(config) - jobs.set_theme(theme_name, domains, runner) - - -@click.command(help="Import the demo course") -@click.pass_obj -def importdemocourse(context: BaseComposeContext) -> None: - config = tutor_config.load(context.root) - runner = context.job_runner(config) - fmt.echo_info("Importing demo course") - jobs.import_demo_course(runner) - - @click.command( short_help="Run a command in a new container", help=( @@ -527,12 +471,10 @@ def add_commands(command_group: click.Group) -> None: command_group.add_command(restart) command_group.add_command(reboot) command_group.add_command(init) - command_group.add_command(createuser) - command_group.add_command(importdemocourse) - command_group.add_command(settheme) command_group.add_command(dc_command) command_group.add_command(run) command_group.add_command(copyfrom) command_group.add_command(execute) command_group.add_command(logs) command_group.add_command(status) + jobs.add_commands(command_group) diff --git a/tutor/commands/jobs.py b/tutor/commands/jobs.py new file mode 100644 index 0000000..eef81c3 --- /dev/null +++ b/tutor/commands/jobs.py @@ -0,0 +1,187 @@ +""" +Common jobs that must be added both to local, dev and k8s commands. +""" + +import typing as t + +import click + +from tutor import config as tutor_config +from tutor import fmt, hooks, jobs + +from .context import BaseJobContext + +BASE_OPENEDX_COMMAND = """ +echo "Loading settings $DJANGO_SETTINGS_MODULE" +""" + + +@hooks.Actions.CORE_READY.add() +def _add_core_init_tasks() -> None: + """ + Declare core init scripts at runtime. + + The context is important, because it allows us to select the init scripts based on + the --limit argument. + """ + with hooks.Contexts.APP("mysql").enter(): + hooks.Filters.COMMANDS_INIT.add_item(("mysql", ("hooks", "mysql", "init"))) + with hooks.Contexts.APP("lms").enter(): + hooks.Filters.COMMANDS_INIT.add_item(("lms", ("hooks", "lms", "init"))) + with hooks.Contexts.APP("cms").enter(): + hooks.Filters.COMMANDS_INIT.add_item(("cms", ("hooks", "cms", "init"))) + + +def initialise(runner: jobs.BaseJobRunner, limit_to: t.Optional[str] = None) -> None: + fmt.echo_info("Initialising all services...") + filter_context = hooks.Contexts.APP(limit_to).name if limit_to else None + + # Pre-init tasks + iter_pre_init_tasks: t.Iterator[ + t.Tuple[str, t.Iterable[str]] + ] = hooks.Filters.COMMANDS_PRE_INIT.iterate(context=filter_context) + for service, path in iter_pre_init_tasks: + fmt.echo_info(f"Running pre-init task: {'/'.join(path)}") + runner.run_job_from_template(service, *path) + + # Init tasks + iter_init_tasks: t.Iterator[ + t.Tuple[str, t.Iterable[str]] + ] = hooks.Filters.COMMANDS_INIT.iterate(context=filter_context) + for service, path in iter_init_tasks: + fmt.echo_info(f"Running init task: {'/'.join(path)}") + runner.run_job_from_template(service, *path) + + fmt.echo_info("All services initialised.") + + +@click.command(help="Create an Open edX user and interactively set their password") +@click.option("--superuser", is_flag=True, help="Make superuser") +@click.option("--staff", is_flag=True, help="Make staff user") +@click.option( + "-p", + "--password", + help="Specify password from the command line. If undefined, you will be prompted to input a password", + prompt=True, + hide_input=True, +) +@click.argument("name") +@click.argument("email") +@click.pass_obj +def createuser( + context: BaseJobContext, + superuser: str, + staff: bool, + password: str, + name: str, + email: str, +) -> None: + run_job( + context, "lms", create_user_template(superuser, staff, name, email, password) + ) + + +@click.command(help="Import the demo course") +@click.pass_obj +def importdemocourse(context: BaseJobContext) -> None: + run_job(context, "cms", import_demo_course_template()) + + +@click.command( + help="Assign a theme to the LMS and the CMS. To reset to the default theme , use 'default' as the theme name." +) +@click.option( + "-d", + "--domain", + "domains", + multiple=True, + help=( + "Limit the theme to these domain names. By default, the theme is " + "applied to the LMS and the CMS, both in development and production mode" + ), +) +@click.argument("theme_name") +@click.pass_obj +def settheme(context: BaseJobContext, domains: t.List[str], theme_name: str) -> None: + run_job(context, "lms", set_theme_template(theme_name, domains)) + + +def run_job(context: BaseJobContext, service: str, command: str) -> None: + config = tutor_config.load(context.root) + runner = context.job_runner(config) + runner.run_job_from_str(service, command) + + +def create_user_template( + superuser: str, staff: bool, username: str, email: str, password: str +) -> str: + opts = "" + if superuser: + opts += " --superuser" + if staff: + opts += " --staff" + return ( + BASE_OPENEDX_COMMAND + + f""" +./manage.py lms manage_user {opts} {username} {email} +./manage.py lms shell -c " +from django.contrib.auth import get_user_model +u = get_user_model().objects.get(username='{username}') +u.set_password('{password}') +u.save()" +""" + ) + + +def import_demo_course_template() -> str: + return ( + BASE_OPENEDX_COMMAND + + """ +# Import demo course +git clone https://github.com/openedx/edx-demo-course --branch {{ OPENEDX_COMMON_VERSION }} --depth 1 ../edx-demo-course +python ./manage.py cms import ../data ../edx-demo-course + +# Re-index courses +./manage.py cms reindex_course --all --setup""" + ) + + +def set_theme_template(theme_name: str, domain_names: t.List[str]) -> str: + """ + For each domain, get or create a Site object and assign the selected theme. + """ + # Note that there are no double quotes " in this piece of code + python_command = """ +import sys +from django.contrib.sites.models import Site +def assign_theme(name, domain): + print('Assigning theme', name, 'to', domain) + if len(domain) > 50: + sys.stderr.write( + 'Assigning a theme to a site with a long (> 50 characters) domain name.' + ' The displayed site name will be truncated to 50 characters.\\n' + ) + site, _ = Site.objects.get_or_create(domain=domain) + if not site.name: + name_max_length = Site._meta.get_field('name').max_length + site.name = domain[:name_max_length] + site.save() + site.themes.all().delete() + site.themes.create(theme_dir_name=name) +""" + domain_names = domain_names or [ + "{{ LMS_HOST }}", + "{{ LMS_HOST }}:8000", + "{{ CMS_HOST }}", + "{{ CMS_HOST }}:8001", + "{{ PREVIEW_LMS_HOST }}", + "{{ PREVIEW_LMS_HOST }}:8000", + ] + for domain_name in domain_names: + python_command += f"assign_theme('{theme_name}', '{domain_name}')\n" + return BASE_OPENEDX_COMMAND + f'./manage.py lms shell -c "{python_command}"' + + +def add_commands(command_group: click.Group) -> None: + for job_command in [createuser, importdemocourse, settheme]: + command_group.add_command(job_command) diff --git a/tutor/commands/k8s.py b/tutor/commands/k8s.py index 19a1721..9b3e85d 100644 --- a/tutor/commands/k8s.py +++ b/tutor/commands/k8s.py @@ -8,10 +8,12 @@ from tutor import config as tutor_config from tutor import env as tutor_env from tutor import exceptions, fmt from tutor import interactive as interactive_config -from tutor import jobs, serialize, utils +from tutor import serialize, utils +from tutor.commands import jobs from tutor.commands.config import save as config_save_command from tutor.commands.context import BaseJobContext from tutor.commands.upgrade.k8s import upgrade_from +from tutor.jobs import BaseJobRunner from tutor.types import Config, get_typed @@ -46,7 +48,7 @@ class K8sClients: return self._core_api -class K8sJobRunner(jobs.BaseJobRunner): +class K8sJobRunner(BaseJobRunner): def load_job(self, name: str) -> Any: all_jobs = self.render("k8s", "jobs.yml") for job in serialize.load_all(all_jobs): @@ -370,64 +372,6 @@ def scale(context: K8sContext, deployment: str, replicas: int) -> None: ) -@click.command(help="Create an Open edX user and interactively set their password") -@click.option("--superuser", is_flag=True, help="Make superuser") -@click.option("--staff", is_flag=True, help="Make staff user") -@click.option( - "-p", - "--password", - help="Specify password from the command line. If undefined, you will be prompted to input a password", - prompt=True, - hide_input=True, -) -@click.argument("name") -@click.argument("email") -@click.pass_obj -def createuser( - context: K8sContext, - superuser: str, - staff: bool, - password: str, - name: str, - email: str, -) -> None: - config = tutor_config.load(context.root) - command = jobs.create_user_command(superuser, staff, name, email, password=password) - runner = context.job_runner(config) - runner.run_job("lms", command) - - -@click.command(help="Import the demo course") -@click.pass_obj -def importdemocourse(context: K8sContext) -> None: - fmt.echo_info("Importing demo course") - config = tutor_config.load(context.root) - runner = context.job_runner(config) - jobs.import_demo_course(runner) - - -@click.command( - help="Assign a theme to the LMS and the CMS. To reset to the default theme , use 'default' as the theme name." -) -@click.option( - "-d", - "--domain", - "domains", - multiple=True, - help=( - "Limit the theme to these domain names. By default, the theme is " - "applied to the LMS and the CMS, both in development and production mode" - ), -) -@click.argument("theme_name") -@click.pass_obj -def settheme(context: K8sContext, domains: List[str], theme_name: str) -> None: - config = tutor_config.load(context.root) - runner = context.job_runner(config) - domains = domains or jobs.get_all_openedx_domains(config) - jobs.set_theme(theme_name, domains, runner) - - @click.command( name="exec", help="Execute a command in a pod of the given application", @@ -590,12 +534,10 @@ k8s.add_command(reboot) k8s.add_command(delete) k8s.add_command(init) k8s.add_command(scale) -k8s.add_command(createuser) -k8s.add_command(importdemocourse) -k8s.add_command(settheme) k8s.add_command(exec_command) k8s.add_command(logs) k8s.add_command(wait) k8s.add_command(upgrade) k8s.add_command(apply_command) k8s.add_command(status) +jobs.add_commands(k8s) diff --git a/tutor/config.py b/tutor/config.py index d9c94d8..5624400 100644 --- a/tutor/config.py +++ b/tutor/config.py @@ -223,10 +223,9 @@ def upgrade_obsolete(config: Config) -> None: ]: if name in config: config[name.replace("ACTIVATE_", "RUN_")] = config.pop(name) - # Replace RUN_CADDY by ENABLE_WEB_PROXY + # Replace nginx by caddy if "RUN_CADDY" in config: config["ENABLE_WEB_PROXY"] = config.pop("RUN_CADDY") - # Replace RUN_CADDY by ENABLE_WEB_PROXY if "NGINX_HTTP_PORT" in config: config["CADDY_HTTP_PORT"] = config.pop("NGINX_HTTP_PORT") diff --git a/tutor/env.py b/tutor/env.py index 5d74368..345bd93 100644 --- a/tutor/env.py +++ b/tutor/env.py @@ -69,8 +69,7 @@ class JinjaEnvironment(jinja2.Environment): class Renderer: def __init__(self, config: t.Optional[Config] = None): - config = config or {} - self.config = deepcopy(config) + self.config = deepcopy(config or {}) self.template_roots = hooks.Filters.ENV_TEMPLATE_ROOTS.apply([TEMPLATES_ROOT]) # Create environment with extra filters and globals diff --git a/tutor/jobs.py b/tutor/jobs.py index e90aaa2..6f145e9 100644 --- a/tutor/jobs.py +++ b/tutor/jobs.py @@ -1,16 +1,13 @@ -import typing as t - -from tutor import env, fmt, hooks -from tutor.types import Config, get_typed - -BASE_OPENEDX_COMMAND = """ -echo "Loading settings $DJANGO_SETTINGS_MODULE" -""" +from tutor import env +from tutor.types import Config class BaseJobRunner: """ - A job runner is responsible for getting a certain task to complete. + A job runner is responsible for running bash commands in the right context. + + Commands may be loaded from string or template files. The `run_job` method must be + implemented by child classes. """ def __init__(self, root: str, config: Config): @@ -21,6 +18,10 @@ class BaseJobRunner: command = self.render(*path) self.run_job(service, command) + def run_job_from_str(self, service: str, command: str) -> None: + rendered = env.render_str(self.config, command).strip() + self.run_job(service, rendered) + def render(self, *path: str) -> str: rendered = env.render_file(self.config, *path).strip() if isinstance(rendered, bytes): @@ -39,121 +40,3 @@ class BaseJobRunner: class BaseComposeJobRunner(BaseJobRunner): def docker_compose(self, *command: str) -> int: raise NotImplementedError - - -@hooks.Actions.CORE_READY.add() -def _add_core_init_tasks() -> None: - """ - Declare core init scripts at runtime. - - The context is important, because it allows us to select the init scripts based on - the --limit argument. - """ - with hooks.Contexts.APP("mysql").enter(): - hooks.Filters.COMMANDS_INIT.add_item(("mysql", ("hooks", "mysql", "init"))) - with hooks.Contexts.APP("lms").enter(): - hooks.Filters.COMMANDS_INIT.add_item(("lms", ("hooks", "lms", "init"))) - with hooks.Contexts.APP("cms").enter(): - hooks.Filters.COMMANDS_INIT.add_item(("cms", ("hooks", "cms", "init"))) - - -def initialise(runner: BaseJobRunner, limit_to: t.Optional[str] = None) -> None: - fmt.echo_info("Initialising all services...") - filter_context = hooks.Contexts.APP(limit_to).name if limit_to else None - - # Pre-init tasks - iter_pre_init_tasks: t.Iterator[ - t.Tuple[str, t.Iterable[str]] - ] = hooks.Filters.COMMANDS_PRE_INIT.iterate(context=filter_context) - for service, path in iter_pre_init_tasks: - fmt.echo_info(f"Running pre-init task: {'/'.join(path)}") - runner.run_job_from_template(service, *path) - - # Init tasks - iter_init_tasks: t.Iterator[ - t.Tuple[str, t.Iterable[str]] - ] = hooks.Filters.COMMANDS_INIT.iterate(context=filter_context) - for service, path in iter_init_tasks: - fmt.echo_info(f"Running init task: {'/'.join(path)}") - runner.run_job_from_template(service, *path) - - fmt.echo_info("All services initialised.") - - -def create_user_command( - superuser: str, - staff: bool, - username: str, - email: str, - password: t.Optional[str] = None, -) -> str: - command = BASE_OPENEDX_COMMAND - - opts = "" - if superuser: - opts += " --superuser" - if staff: - opts += " --staff" - command += """ -./manage.py lms manage_user {opts} {username} {email} -""" - if password: - command += """ -./manage.py lms shell -c "from django.contrib.auth import get_user_model -u = get_user_model().objects.get(username='{username}') -u.set_password('{password}') -u.save()" -""" - else: - command += """ -./manage.py lms changepassword {username} -""" - - return command.format(opts=opts, username=username, email=email, password=password) - - -def import_demo_course(runner: BaseJobRunner) -> None: - runner.run_job_from_template("cms", "hooks", "cms", "importdemocourse") - - -def set_theme( - theme_name: str, domain_names: t.List[str], runner: BaseJobRunner -) -> None: - """ - For each domain, get or create a Site object and assign the selected theme. - """ - if not domain_names: - return - python_code = "from django.contrib.sites.models import Site" - for domain_name in domain_names: - if len(domain_name) > 50: - fmt.echo_alert( - "Assigning a theme to a site with a long (> 50 characters) domain name." - " The displayed site name will be truncated to 50 characters." - ) - python_code += """ -print('Assigning theme {theme_name} to {domain_name}...') -site, _ = Site.objects.get_or_create(domain='{domain_name}') -if not site.name: - name_max_length = Site._meta.get_field('name').max_length - name = '{domain_name}'[:name_max_length] - site.name = name - site.save() -site.themes.all().delete() -site.themes.create(theme_dir_name='{theme_name}') -""".format( - theme_name=theme_name, domain_name=domain_name - ) - command = BASE_OPENEDX_COMMAND + f'./manage.py lms shell -c "{python_code}"' - runner.run_job("lms", command) - - -def get_all_openedx_domains(config: Config) -> t.List[str]: - return [ - get_typed(config, "LMS_HOST", str), - get_typed(config, "LMS_HOST", str) + ":8000", - get_typed(config, "CMS_HOST", str), - get_typed(config, "CMS_HOST", str) + ":8001", - get_typed(config, "PREVIEW_LMS_HOST", str), - get_typed(config, "PREVIEW_LMS_HOST", str) + ":8000", - ] diff --git a/tutor/templates/hooks/cms/importdemocourse b/tutor/templates/hooks/cms/importdemocourse deleted file mode 100644 index 236d7b1..0000000 --- a/tutor/templates/hooks/cms/importdemocourse +++ /dev/null @@ -1,8 +0,0 @@ -echo "Loading settings $DJANGO_SETTINGS_MODULE" - -# Import demo course -git clone https://github.com/openedx/edx-demo-course --branch {{ OPENEDX_COMMON_VERSION }} --depth 1 ../edx-demo-course -python ./manage.py cms import ../data ../edx-demo-course - -# Re-index courses -./manage.py cms reindex_course --all --setup From 16e6131f96ad2a64de813ee438f8fb9871f68c59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Wed, 19 Oct 2022 17:46:31 +0200 Subject: [PATCH 52/56] feat: pluggable `local/dev/k8s do ` commands We introduce a new filter to implement custom commands in arbitrary containers. It becomes easy to write convenient ad-hoc commands that users will then be able to run either on Kubernetes or locally using a documented CLI. Pluggable jobs are declared as Click commands and are responsible for parsing their own arguments. See the new CLI_DO_COMMANDS filter. Close https://github.com/overhangio/2u-tutor-adoption/issues/75 --- CHANGELOG-nightly.md | 3 + docs/local.rst | 8 +- docs/tutorials/arm64.rst | 4 +- docs/tutorials/plugin.rst | 30 ++- docs/tutorials/theming.rst | 2 +- tests/commands/base.py | 8 +- tests/commands/test_context.py | 4 +- tests/commands/test_jobs.py | 109 ++++---- tests/helpers.py | 14 +- tests/test_env.py | 2 +- tests/test_plugins_v0.py | 13 +- tutor/bindmounts.py | 8 +- tutor/commands/compose.py | 42 +-- tutor/commands/context.py | 8 +- tutor/commands/dev.py | 10 +- tutor/commands/jobs.py | 247 ++++++++++++------ tutor/commands/k8s.py | 124 ++++++--- tutor/commands/local.py | 10 +- tutor/hooks/__init__.py | 2 +- tutor/hooks/consts.py | 101 ++++--- tutor/plugins/v0.py | 16 +- tutor/{jobs.py => tasks.py} | 18 +- .../{hooks/cms/init => jobs/init/cms.sh} | 0 .../{hooks/lms/init => jobs/init/lms.sh} | 0 .../{hooks/mysql/init => jobs/init/mysql.sh} | 0 tutor/utils.py | 4 + 26 files changed, 493 insertions(+), 294 deletions(-) rename tutor/{jobs.py => tasks.py} (67%) rename tutor/templates/{hooks/cms/init => jobs/init/cms.sh} (100%) rename tutor/templates/{hooks/lms/init => jobs/init/lms.sh} (100%) rename tutor/templates/{hooks/mysql/init => jobs/init/mysql.sh} (100%) diff --git a/CHANGELOG-nightly.md b/CHANGELOG-nightly.md index 26a7343..99daf2b 100644 --- a/CHANGELOG-nightly.md +++ b/CHANGELOG-nightly.md @@ -8,6 +8,9 @@ will be backported to the master branch at every major release. When backporting changes to master, we should keep only the entries that correspond to user- facing changes. --> +- 💥[Feature] Add an extensible `local/dev/k8s do ...` command to trigger custom job commands. These commands are used to run a series of bash scripts in designated containers. Any plugin can add custom jobs thanks to the `CLI_DO_COMMANDS` filter. This causes the following breaking changes: + - The "init", "createuser", "settheme", "importdemocourse" commands were all migrated to this new interface. For instance, `tutor local init` was replaced by `tutor local do init`. + - Plugin developers are encouraged to replace calls to the `COMMANDS_INIT` and `COMMANDS_PRE_INIT` filters by `CLI_DO_INIT_TASKS`. - [Feature] Implement hook filter priorities, which work like action priorities. (by @regisb) - 💥[Improvement] Remove the `local/dev bindmount` commands, which have been marked as deprecated for some time. The `--mount` option should be used instead. - 💥[Bugfix] Fix local installation requirements. Plugins that implemented the "openedx-dockerfile-post-python-requirements" patch and that needed access to the edx-platform repo will no longer work. Instead, these plugins should implement the "openedx-dockerfile-pre-assets" patch. This scenario should be very rare, though. (by @regisb) diff --git a/docs/local.rst b/docs/local.rst index 590692f..d9c4e13 100644 --- a/docs/local.rst +++ b/docs/local.rst @@ -80,7 +80,7 @@ Service initialisation :: - tutor local init + tutor local do init This command should be run just once. It will initialise all applications in a running platform. In particular, this will create the required databases tables and apply database migrations for all applications. @@ -120,7 +120,7 @@ Creating a new user with staff and admin rights You will most certainly need to create a user to administer the platform. Just run:: - tutor local createuser --staff --superuser yourusername user@email.com + tutor local do createuser --staff --superuser yourusername user@email.com You will be asked to set the user password interactively. @@ -131,7 +131,7 @@ Importing the demo course After a fresh installation, your platform will not have a single course. To import the `Open edX demo course `_, run:: - tutor local importdemocourse + tutor local do importdemocourse .. _settheme: @@ -140,7 +140,7 @@ Setting a new theme The default Open edX theme is rather bland, so Tutor makes it easy to switch to a different theme:: - tutor local settheme mytheme + tutor local do settheme mytheme Out of the box, only the default "open-edx" theme is available. We also developed `Indigo, a beautiful, customizable theme `__ which is easy to install with Tutor. diff --git a/docs/tutorials/arm64.rst b/docs/tutorials/arm64.rst index aa172c6..5978b9b 100644 --- a/docs/tutorials/arm64.rst +++ b/docs/tutorials/arm64.rst @@ -48,9 +48,9 @@ Finish setup and start Tutor From this point on, use Tutor as normal. For example, start Open edX and run migrations with:: tutor local start -d - tutor local init + tutor local do init Or for a development environment:: tutor dev start -d - tutor dev init + tutor dev do init diff --git a/docs/tutorials/plugin.rst b/docs/tutorials/plugin.rst index c4ca5b0..f522f61 100644 --- a/docs/tutorials/plugin.rst +++ b/docs/tutorials/plugin.rst @@ -218,7 +218,7 @@ You can now run the "myservice" container which will execute the ``CMD`` stateme Declaring initialisation tasks ------------------------------ -Services often need to run specific tasks before they can be started. For instance, the LMS and the CMS need to apply database migrations. These commands are written in shell scripts that are executed whenever we run ``launch``. We call these scripts "init tasks". To add a new local init task, we must first add the corresponding service to the ``docker-compose-jobs.yml`` file by implementing the :patch:`local-docker-compose-jobs-services` patch:: +Services often need to run specific tasks before they can be started. For instance, the LMS and the CMS need to apply database migrations. These commands are written in shell scripts that are executed whenever we run ``launch``. We call these scripts "init tasks". To add a new local initialisation task, we must first add the corresponding service to the ``docker-compose-jobs.yml`` file by implementing the :patch:`local-docker-compose-jobs-services` patch:: hooks.Filters.ENV_PATCHES.add_item( ( @@ -234,24 +234,22 @@ The patch above defined the "myservice-job" container which will run our initial $ tutor config save -Next, we create the folder which will contain our init task script:: +Next, we create an initialisation task by adding an item to the :py:data:`tutor.hooks.Filters.CLI_DO_INIT_TASKS` filter:: - $ mkdir "$(tutor plugins printroot)/templates/myplugin/tasks" - -Edit ``$(tutor plugins printroot)/templates/myplugin/tasks/init.sh``:: + hooks.Filters.CLI_DO_INIT_TASKS.add_item( + ( + "myservice", + """ echo "++++++ initialising my plugin..." echo "++++++ done!" - -Add our init task script to the :py:data:`tutor.hooks.Filters.COMMANDS_INIT` filter:: - - hooks.Filters.COMMANDS_INIT.add_item( - ("myservice", ("myplugin", "tasks", "init.sh")), + """ + ) ) Run this initialisation task with:: - $ tutor local init --limit=myplugin + $ tutor local do init --limit=myplugin ... Running init task: myplugin/tasks/init.sh ... @@ -354,8 +352,14 @@ Eventually, our plugin is composed of the following files, all stored within the ) hooks.Filters.IMAGES_PUSH.add_item(("myservice", "myservice:latest")) hooks.Filters.IMAGES_PULL.add_item(("myservice", "myservice:latest")) - hooks.Filters.COMMANDS_INIT.add_item( - ("myservice", ("myplugin", "tasks", "init.sh")), + hooks.Filters.CLI_DO_INIT_TASKS.add_item( + ( + "myservice", + """ + echo "++++++ initialising my plugin..." + echo "++++++ done!" + """ + ) ) ``templates/myplugin/build/myservice/Dockerfile`` diff --git a/docs/tutorials/theming.rst b/docs/tutorials/theming.rst index de2c7a2..224a559 100644 --- a/docs/tutorials/theming.rst +++ b/docs/tutorials/theming.rst @@ -48,7 +48,7 @@ Then, run a local webserver:: The LMS can then be accessed at http://local.overhang.io:8000. You will then have to :ref:`enable that theme `:: - tutor dev settheme mythemename + tutor dev do settheme mythemename Watch the themes folders for changes (in a different terminal):: diff --git a/tests/commands/base.py b/tests/commands/base.py index a39d934..e2f4ed6 100644 --- a/tests/commands/base.py +++ b/tests/commands/base.py @@ -17,7 +17,9 @@ class TestCommandMixin: return TestCommandMixin.invoke_in_root(root, args) @staticmethod - def invoke_in_root(root: str, args: t.List[str]) -> click.testing.Result: + def invoke_in_root( + root: str, args: t.List[str], catch_exceptions: bool = True + ) -> click.testing.Result: """ Use this method for commands that all need to run in the same root: @@ -32,4 +34,6 @@ class TestCommandMixin: "TUTOR_IGNORE_DICT_PLUGINS": "1", } ) - return runner.invoke(cli, args, obj=TestContext(root)) + return runner.invoke( + cli, args, obj=TestContext(root), catch_exceptions=catch_exceptions + ) diff --git a/tests/commands/test_context.py b/tests/commands/test_context.py index dd76bde..cf2a1f0 100644 --- a/tests/commands/test_context.py +++ b/tests/commands/test_context.py @@ -1,7 +1,7 @@ import os import unittest -from tests.helpers import TestContext, TestJobRunner, temporary_root +from tests.helpers import TestContext, TestTaskRunner, temporary_root from tutor import config as tutor_config @@ -15,4 +15,4 @@ class TestContextTests(unittest.TestCase): self.assertFalse( os.path.exists(os.path.join(context.root, tutor_config.CONFIG_FILENAME)) ) - self.assertTrue(isinstance(runner, TestJobRunner)) + self.assertTrue(isinstance(runner, TestTaskRunner)) diff --git a/tests/commands/test_jobs.py b/tests/commands/test_jobs.py index 687fc24..f5f8bb3 100644 --- a/tests/commands/test_jobs.py +++ b/tests/commands/test_jobs.py @@ -1,71 +1,74 @@ -import re -import unittest -from io import StringIO from unittest.mock import patch -from tests.helpers import TestContext, temporary_root -from tutor import config as tutor_config +from tests.helpers import PluginsTestCase, temporary_root from tutor.commands import jobs +from .base import TestCommandMixin -class JobsTests(unittest.TestCase): - @patch("sys.stdout", new_callable=StringIO) - def test_initialise(self, mock_stdout: StringIO) -> None: + +class JobsTests(PluginsTestCase, TestCommandMixin): + def test_initialise(self) -> None: with temporary_root() as root: - context = TestContext(root) - config = tutor_config.load_full(root) - runner = context.job_runner(config) - jobs.initialise(runner) - output = mock_stdout.getvalue().strip() - self.assertTrue(output.startswith("Initialising all services...")) - self.assertTrue(output.endswith("All services initialised.")) + self.invoke_in_root(root, ["config", "save"]) + result = self.invoke_in_root(root, ["local", "do", "init"]) + self.assertIsNone(result.exception) + self.assertEqual(0, result.exit_code) + self.assertIn("All services initialised.", result.output) - def test_create_user_command_without_staff(self) -> None: - command = jobs.create_user_template("superuser", False, "username", "email", "p4ssw0rd") + def test_create_user_template_without_staff(self) -> None: + command = jobs.create_user_template( + "superuser", False, "username", "email", "p4ssw0rd" + ) self.assertNotIn("--staff", command) self.assertIn("set_password", command) - def test_create_user_command_with_staff(self) -> None: - command = jobs.create_user_template("superuser", True, "username", "email", "p4ssw0rd") + def test_create_user_template_with_staff(self) -> None: + command = jobs.create_user_template( + "superuser", True, "username", "email", "p4ssw0rd" + ) self.assertIn("--staff", command) - @patch("sys.stdout", new_callable=StringIO) - def test_import_demo_course(self, mock_stdout: StringIO) -> None: + def test_import_demo_course(self) -> None: with temporary_root() as root: - context = TestContext(root) - config = tutor_config.load_full(root) - runner = context.job_runner(config) - runner.run_job_from_str("cms", jobs.import_demo_course_template()) + self.invoke_in_root(root, ["config", "save"]) + with patch("tutor.utils.docker_compose") as mock_docker_compose: + result = self.invoke_in_root(root, ["local", "do", "importdemocourse"]) + dc_args, _dc_kwargs = mock_docker_compose.call_args + self.assertIsNone(result.exception) + self.assertEqual(0, result.exit_code) + self.assertIn("cms-job", dc_args) + self.assertTrue( + dc_args[-1] + .strip() + .startswith('echo "Loading settings $DJANGO_SETTINGS_MODULE"') + ) - output = mock_stdout.getvalue() - service = re.search(r"Service: (\w*)", output) - commands = re.search(r"(-----)([\S\s]+)(-----)", output) - assert service is not None - assert commands is not None - self.assertEqual(service.group(1), "cms") + def test_set_theme(self) -> None: + with temporary_root() as root: + self.invoke_in_root(root, ["config", "save"]) + with patch("tutor.utils.docker_compose") as mock_docker_compose: + result = self.invoke_in_root( + root, + [ + "local", + "do", + "settheme", + "--domain", + "domain1", + "--domain", + "domain2", + "beautiful", + ], + ) + dc_args, _dc_kwargs = mock_docker_compose.call_args + + self.assertIsNone(result.exception) + self.assertEqual(0, result.exit_code) + self.assertIn("lms-job", dc_args) self.assertTrue( - commands.group(2) - .strip() - .startswith('echo "Loading settings $DJANGO_SETTINGS_MODULE"') - ) - - @patch("sys.stdout", new_callable=StringIO) - def test_set_theme(self, mock_stdout: StringIO) -> None: - with temporary_root() as root: - context = TestContext(root) - config = tutor_config.load_full(root) - runner = context.job_runner(config) - command = jobs.set_theme_template("sample_theme", ["domain1", "domain2"]) - runner.run_job_from_str("lms", command) - - output = mock_stdout.getvalue() - service = re.search(r"Service: (\w*)", output) - commands = re.search(r"(-----)([\S\s]+)(-----)", output) - assert service is not None - assert commands is not None - self.assertEqual(service.group(1), "lms") - self.assertTrue( - commands.group(2) + dc_args[-1] .strip() .startswith('echo "Loading settings $DJANGO_SETTINGS_MODULE"') ) + self.assertIn("assign_theme('beautiful', 'domain1')", dc_args[-1]) + self.assertIn("assign_theme('beautiful', 'domain2')", dc_args[-1]) diff --git a/tests/helpers.py b/tests/helpers.py index 82a412c..fef3203 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -5,12 +5,12 @@ import unittest import unittest.result from tutor import hooks -from tutor.commands.context import BaseJobContext -from tutor.jobs import BaseJobRunner +from tutor.commands.context import BaseTaskContext +from tutor.tasks import BaseTaskRunner from tutor.types import Config -class TestJobRunner(BaseJobRunner): +class TestTaskRunner(BaseTaskRunner): """ Mock job runner for unit testing. @@ -18,7 +18,7 @@ class TestJobRunner(BaseJobRunner): separated by dashes. """ - def run_job(self, service: str, command: str) -> int: + def run_task(self, service: str, command: str) -> int: print(os.linesep.join([f"Service: {service}", "-----", command, "----- "])) return 0 @@ -36,13 +36,13 @@ def temporary_root() -> "tempfile.TemporaryDirectory[str]": return tempfile.TemporaryDirectory(prefix="tutor-test-root-") -class TestContext(BaseJobContext): +class TestContext(BaseTaskContext): """ Click context that will use only test job runners. """ - def job_runner(self, config: Config) -> TestJobRunner: - return TestJobRunner(self.root, config) + def job_runner(self, config: Config) -> TestTaskRunner: + return TestTaskRunner(self.root, config) class PluginsTestCase(unittest.TestCase): diff --git a/tests/test_env.py b/tests/test_env.py index f0de174..a752c69 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -83,7 +83,7 @@ class EnvTests(PluginsTestCase): tutor_config.render_full(config) config["MYSQL_ROOT_PASSWORD"] = "testpassword" - rendered = env.render_file(config, "hooks", "mysql", "init") + rendered = env.render_file(config, "jobs", "init", "mysql.sh") self.assertIn("testpassword", rendered) @patch.object(tutor_config.fmt, "echo") diff --git a/tests/test_plugins_v0.py b/tests/test_plugins_v0.py index 67f975c..85dddb9 100644 --- a/tests/test_plugins_v0.py +++ b/tests/test_plugins_v0.py @@ -167,10 +167,17 @@ class PluginsTests(PluginsTestCase): def test_init_tasks(self) -> None: plugins_v0.DictPlugin({"name": "plugin1", "hooks": {"init": ["myclient"]}}) - plugins.load("plugin1") + with patch.object( + plugins_v0.env, "read_template_file", return_value="echo hello" + ) as mock_read_template: + plugins.load("plugin1") + mock_read_template.assert_called_once_with( + "plugin1", "hooks", "myclient", "init" + ) + self.assertIn( - ("myclient", ("plugin1", "hooks", "myclient", "init")), - list(hooks.Filters.COMMANDS_INIT.iterate()), + ("myclient", "echo hello"), + list(hooks.Filters.CLI_DO_INIT_TASKS.iterate()), ) def test_plugins_are_updated_on_config_change(self) -> None: diff --git a/tutor/bindmounts.py b/tutor/bindmounts.py index 4bc156e..d807358 100644 --- a/tutor/bindmounts.py +++ b/tutor/bindmounts.py @@ -1,12 +1,12 @@ import os -from .exceptions import TutorError -from .jobs import BaseComposeJobRunner -from .utils import get_user_id +from tutor.exceptions import TutorError +from tutor.tasks import BaseComposeTaskRunner +from tutor.utils import get_user_id def create( - runner: BaseComposeJobRunner, + runner: BaseComposeTaskRunner, service: str, path: str, ) -> str: diff --git a/tutor/commands/compose.py b/tutor/commands/compose.py index f8000ff..85a95b1 100644 --- a/tutor/commands/compose.py +++ b/tutor/commands/compose.py @@ -9,13 +9,13 @@ from tutor import config as tutor_config from tutor import env as tutor_env from tutor import fmt, hooks, serialize, utils from tutor.commands import jobs -from tutor.commands.context import BaseJobContext +from tutor.commands.context import BaseTaskContext from tutor.exceptions import TutorError -from tutor.jobs import BaseComposeJobRunner +from tutor.tasks import BaseComposeTaskRunner from tutor.types import Config -class ComposeJobRunner(BaseComposeJobRunner): +class ComposeTaskRunner(BaseComposeTaskRunner): def __init__(self, root: str, config: Config): super().__init__(root, config) self.project_name = "" @@ -81,7 +81,7 @@ class ComposeJobRunner(BaseComposeJobRunner): docker_compose_jobs_tmp_path, ) - def run_job(self, service: str, command: str) -> int: + def run_task(self, service: str, command: str) -> int: """ Run the "{{ service }}-job" service from local/docker-compose.jobs.yml with the specified command. @@ -105,11 +105,11 @@ class ComposeJobRunner(BaseComposeJobRunner): ) -class BaseComposeContext(BaseJobContext): +class BaseComposeContext(BaseTaskContext): COMPOSE_TMP_FILTER: hooks.filters.Filter = NotImplemented COMPOSE_JOBS_TMP_FILTER: hooks.filters.Filter = NotImplemented - def job_runner(self, config: Config) -> ComposeJobRunner: + def job_runner(self, config: Config) -> ComposeTaskRunner: raise NotImplementedError @@ -297,19 +297,22 @@ def restart(context: BaseComposeContext, services: t.List[str]) -> None: context.job_runner(config).docker_compose(*command) -@click.command(help="Initialise all applications") -@click.option("-l", "--limit", help="Limit initialisation to this service or plugin") +@jobs.do_group @mount_option @click.pass_obj -def init( - context: BaseComposeContext, - limit: str, - mounts: t.Tuple[t.List[MountParam.MountType]], +def do( + context: BaseComposeContext, mounts: t.Tuple[t.List[MountParam.MountType]] ) -> None: - mount_tmp_volumes(mounts, context) - config = tutor_config.load(context.root) - runner = context.job_runner(config) - jobs.initialise(runner, limit_to=limit) + """ + Run a custom job in the right container(s). + """ + @hooks.Actions.DO_JOB.add() + def _mount_tmp_volumes(_job_name: str, *_args: t.Any, **_kwargs: t.Any) -> None: + """ + We add this logic to an action callback because we do not want to trigger it + whenever we run `tutor local do --help`. + """ + mount_tmp_volumes(mounts, context) @click.command( @@ -470,11 +473,14 @@ def add_commands(command_group: click.Group) -> None: command_group.add_command(stop) command_group.add_command(restart) command_group.add_command(reboot) - command_group.add_command(init) command_group.add_command(dc_command) command_group.add_command(run) command_group.add_command(copyfrom) command_group.add_command(execute) command_group.add_command(logs) command_group.add_command(status) - jobs.add_commands(command_group) + + @hooks.Actions.PLUGINS_LOADED.add() + def _add_do_commands() -> None: + jobs.add_job_commands(do) + command_group.add_command(do) diff --git a/tutor/commands/context.py b/tutor/commands/context.py index 3e859e0..abd4ea0 100644 --- a/tutor/commands/context.py +++ b/tutor/commands/context.py @@ -1,5 +1,5 @@ -from ..jobs import BaseJobRunner -from ..types import Config +from tutor.tasks import BaseTaskRunner +from tutor.types import Config class Context: @@ -16,14 +16,14 @@ class Context: self.root = root -class BaseJobContext(Context): +class BaseTaskContext(Context): """ Specialized context that subcommands may use. For instance `dev`, `local` and `k8s` define custom runners to run jobs. """ - def job_runner(self, config: Config) -> BaseJobRunner: + def job_runner(self, config: Config) -> BaseTaskRunner: """ Return a runner capable of running docker-compose/kubectl commands. """ diff --git a/tutor/commands/dev.py b/tutor/commands/dev.py index 46dfb05..e7d4cf4 100644 --- a/tutor/commands/dev.py +++ b/tutor/commands/dev.py @@ -11,7 +11,7 @@ from tutor.commands import compose from tutor.types import Config, get_typed -class DevJobRunner(compose.ComposeJobRunner): +class DevTaskRunner(compose.ComposeTaskRunner): def __init__(self, root: str, config: Config): """ Load docker-compose files from dev/ and local/ @@ -51,8 +51,8 @@ class DevContext(compose.BaseComposeContext): COMPOSE_TMP_FILTER = hooks.Filters.COMPOSE_DEV_TMP COMPOSE_JOBS_TMP_FILTER = hooks.Filters.COMPOSE_DEV_JOBS_TMP - def job_runner(self, config: Config) -> DevJobRunner: - return DevJobRunner(self.root, config) + def job_runner(self, config: Config) -> DevTaskRunner: + return DevTaskRunner(self.root, config) @click.group(help="Run Open edX locally with development settings") @@ -105,7 +105,7 @@ Tutor may not work if Docker is configured with < 4 GB RAM. Please follow instru context.invoke(compose.start, detach=True) click.echo(fmt.title("Database creation and migrations")) - context.invoke(compose.init) + context.invoke(compose.do.commands["init"]) fmt.echo_info( """The Open edX platform is now running in detached mode @@ -148,7 +148,7 @@ def _stop_on_local_start(root: str, config: Config, project_name: str) -> None: Stop the dev platform as soon as a platform with a different project name is started. """ - runner = DevJobRunner(root, config) + runner = DevTaskRunner(root, config) if project_name != runner.project_name: runner.docker_compose("stop") diff --git a/tutor/commands/jobs.py b/tutor/commands/jobs.py index eef81c3..9f8ac6a 100644 --- a/tutor/commands/jobs.py +++ b/tutor/commands/jobs.py @@ -1,19 +1,38 @@ """ Common jobs that must be added both to local, dev and k8s commands. """ - +import functools import typing as t import click +from typing_extensions import ParamSpec from tutor import config as tutor_config -from tutor import fmt, hooks, jobs +from tutor import env, fmt, hooks -from .context import BaseJobContext -BASE_OPENEDX_COMMAND = """ -echo "Loading settings $DJANGO_SETTINGS_MODULE" -""" +class DoGroup(click.Group): + """ + A Click group that prints subcommands under 'Jobs' instead of 'Commands' when we run + `.. do --help`. Hackish but it works. + """ + + def get_help(self, ctx: click.Context) -> str: + return super().get_help(ctx).replace("Commands:\n", "Jobs:\n") + + +# A convenient easy-to-use decorator for creating `do` commands. +do_group = click.group(cls=DoGroup, subcommand_metavar="JOB [ARGS]...") + + +def add_job_commands(do_command_group: click.Group) -> None: + """ + This is meant to be called with the `local/dev/k8s do` group commands, to add the + different `do` subcommands. + """ + subcommands: t.Iterator[click.Command] = hooks.Filters.CLI_DO_COMMANDS.iterate() + for subcommand in subcommands: + do_command_group.add_command(subcommand) @hooks.Actions.CORE_READY.add() @@ -25,32 +44,52 @@ def _add_core_init_tasks() -> None: the --limit argument. """ with hooks.Contexts.APP("mysql").enter(): - hooks.Filters.COMMANDS_INIT.add_item(("mysql", ("hooks", "mysql", "init"))) + hooks.Filters.CLI_DO_INIT_TASKS.add_item( + ("mysql", env.read_template_file("jobs", "init", "mysql.sh")) + ) with hooks.Contexts.APP("lms").enter(): - hooks.Filters.COMMANDS_INIT.add_item(("lms", ("hooks", "lms", "init"))) + hooks.Filters.CLI_DO_INIT_TASKS.add_item( + ("lms", env.read_template_file("jobs", "init", "lms.sh")) + ) with hooks.Contexts.APP("cms").enter(): - hooks.Filters.COMMANDS_INIT.add_item(("cms", ("hooks", "cms", "init"))) + hooks.Filters.CLI_DO_INIT_TASKS.add_item( + ("cms", env.read_template_file("jobs", "init", "cms.sh")) + ) -def initialise(runner: jobs.BaseJobRunner, limit_to: t.Optional[str] = None) -> None: +@click.command("init", help="Initialise all applications") +@click.option("-l", "--limit", help="Limit initialisation to this service or plugin") +def initialise(limit: t.Optional[str]) -> t.Iterator[t.Tuple[str, str]]: fmt.echo_info("Initialising all services...") - filter_context = hooks.Contexts.APP(limit_to).name if limit_to else None + filter_context = hooks.Contexts.APP(limit).name if limit else None - # Pre-init tasks - iter_pre_init_tasks: t.Iterator[ + # Deprecated pre-init tasks + depr_iter_pre_init_tasks: t.Iterator[ t.Tuple[str, t.Iterable[str]] ] = hooks.Filters.COMMANDS_PRE_INIT.iterate(context=filter_context) - for service, path in iter_pre_init_tasks: - fmt.echo_info(f"Running pre-init task: {'/'.join(path)}") - runner.run_job_from_template(service, *path) + for service, path in depr_iter_pre_init_tasks: + fmt.echo_alert( + f"Running deprecated pre-init task: {'/'.join(path)}. Init tasks should no longer be added to the COMMANDS_PRE_INIT filter. Plugin developers should use the CLI_DO_INIT_TASKS filter instead, with a high priority." + ) + yield service, env.read_template_file(*path) # Init tasks iter_init_tasks: t.Iterator[ + t.Tuple[str, str] + ] = hooks.Filters.CLI_DO_INIT_TASKS.iterate(context=filter_context) + for service, task in iter_init_tasks: + fmt.echo_info(f"Running init task in {service}") + yield service, task + + # Deprecated init tasks + depr_iter_init_tasks: t.Iterator[ t.Tuple[str, t.Iterable[str]] ] = hooks.Filters.COMMANDS_INIT.iterate(context=filter_context) - for service, path in iter_init_tasks: - fmt.echo_info(f"Running init task: {'/'.join(path)}") - runner.run_job_from_template(service, *path) + for service, path in depr_iter_init_tasks: + fmt.echo_alert( + f"Running deprecated init task: {'/'.join(path)}. Init tasks should no longer be added to the COMMANDS_INIT filter. Plugin developers should use the CLI_DO_INIT_TASKS filter instead." + ) + yield service, env.read_template_file(*path) fmt.echo_info("All services initialised.") @@ -67,29 +106,52 @@ def initialise(runner: jobs.BaseJobRunner, limit_to: t.Optional[str] = None) -> ) @click.argument("name") @click.argument("email") -@click.pass_obj def createuser( - context: BaseJobContext, superuser: str, staff: bool, password: str, name: str, email: str, -) -> None: - run_job( - context, "lms", create_user_template(superuser, staff, name, email, password) - ) +) -> t.Iterable[t.Tuple[str, str]]: + """ + Create an Open edX user + + Password can be passed as an option or will be set interactively. + """ + yield ("lms", create_user_template(superuser, staff, name, email, password)) + + +def create_user_template( + superuser: str, staff: bool, username: str, email: str, password: str +) -> str: + opts = "" + if superuser: + opts += " --superuser" + if staff: + opts += " --staff" + return f""" +./manage.py lms manage_user {opts} {username} {email} +./manage.py lms shell -c " +from django.contrib.auth import get_user_model +u = get_user_model().objects.get(username='{username}') +u.set_password('{password}') +u.save()" +""" @click.command(help="Import the demo course") -@click.pass_obj -def importdemocourse(context: BaseJobContext) -> None: - run_job(context, "cms", import_demo_course_template()) +def importdemocourse() -> t.Iterable[t.Tuple[str, str]]: + template = """ +# Import demo course +git clone https://github.com/openedx/edx-demo-course --branch {{ OPENEDX_COMMON_VERSION }} --depth 1 ../edx-demo-course +python ./manage.py cms import ../data ../edx-demo-course + +# Re-index courses +./manage.py cms reindex_course --all --setup""" + yield ("cms", template) -@click.command( - help="Assign a theme to the LMS and the CMS. To reset to the default theme , use 'default' as the theme name." -) +@click.command() @click.option( "-d", "--domain", @@ -101,49 +163,13 @@ def importdemocourse(context: BaseJobContext) -> None: ), ) @click.argument("theme_name") -@click.pass_obj -def settheme(context: BaseJobContext, domains: t.List[str], theme_name: str) -> None: - run_job(context, "lms", set_theme_template(theme_name, domains)) +def settheme(domains: t.List[str], theme_name: str) -> t.Iterable[t.Tuple[str, str]]: + """ + Assign a theme to the LMS and the CMS. - -def run_job(context: BaseJobContext, service: str, command: str) -> None: - config = tutor_config.load(context.root) - runner = context.job_runner(config) - runner.run_job_from_str(service, command) - - -def create_user_template( - superuser: str, staff: bool, username: str, email: str, password: str -) -> str: - opts = "" - if superuser: - opts += " --superuser" - if staff: - opts += " --staff" - return ( - BASE_OPENEDX_COMMAND - + f""" -./manage.py lms manage_user {opts} {username} {email} -./manage.py lms shell -c " -from django.contrib.auth import get_user_model -u = get_user_model().objects.get(username='{username}') -u.set_password('{password}') -u.save()" -""" - ) - - -def import_demo_course_template() -> str: - return ( - BASE_OPENEDX_COMMAND - + """ -# Import demo course -git clone https://github.com/openedx/edx-demo-course --branch {{ OPENEDX_COMMON_VERSION }} --depth 1 ../edx-demo-course -python ./manage.py cms import ../data ../edx-demo-course - -# Re-index courses -./manage.py cms reindex_course --all --setup""" - ) + To reset to the default theme , use 'default' as the theme name. + """ + yield ("lms", set_theme_template(theme_name, domains)) def set_theme_template(theme_name: str, domain_names: t.List[str]) -> str: @@ -179,9 +205,76 @@ def assign_theme(name, domain): ] for domain_name in domain_names: python_command += f"assign_theme('{theme_name}', '{domain_name}')\n" - return BASE_OPENEDX_COMMAND + f'./manage.py lms shell -c "{python_command}"' + return f'./manage.py lms shell -c "{python_command}"' -def add_commands(command_group: click.Group) -> None: - for job_command in [createuser, importdemocourse, settheme]: - command_group.add_command(job_command) +hooks.Filters.CLI_DO_COMMANDS.add_items( + [ + createuser, + importdemocourse, + initialise, + settheme, + ] +) + + +def do_callback(service_commands: t.Iterable[t.Tuple[str, str]]) -> None: + """ + This function must be added as a callback to all `do` subcommands. + + `do` subcommands don't actually run any task. They just yield tuples of (service + name, unrendered script string). This function is responsible for actually running + the scripts. It does the following: + + - Prefix the script with a base command + - Render the script string + - Run a job in the right container + + In order to be added as a callback to the do subcommands, the + `_patch_do_commands_callbacks` must be called. + """ + context = click.get_current_context().obj + config = tutor_config.load(context.root) + runner = context.job_runner(config) + base_openedx_command = """ +echo "Loading settings $DJANGO_SETTINGS_MODULE" +""" + for service, command in service_commands: + runner.run_task_from_str(service, base_openedx_command + command) + + +@hooks.Actions.PLUGINS_LOADED.add() +def _patch_do_commands_callbacks() -> None: + """ + After plugins have been loaded, patch `do` subcommands such that their output is + forwarded to `do_callback`. + """ + subcommands: t.Iterator[click.Command] = hooks.Filters.CLI_DO_COMMANDS.iterate() + for subcommand in subcommands: + # Modify the subcommand callback such that job results are processed by do_callback + if subcommand.callback is None: + raise ValueError("Cannot patch None callback") + if subcommand.name is None: + raise ValueError("Defined job with None name") + subcommand.callback = _patch_callback(subcommand.name, subcommand.callback) + + +P = ParamSpec("P") + + +def _patch_callback( + job_name: str, + func: t.Callable[P, t.Iterable[t.Tuple[str, str]]] +) -> t.Callable[P, None]: + """ + Modify a subcommand callback function such that its results are processed by `do_callback`. + """ + + def new_callback(*args: P.args, **kwargs: P.kwargs) -> None: + hooks.Actions.DO_JOB.do(job_name, *args, context=None, **kwargs) + do_callback(func(*args, **kwargs)) + + # Make the new callback behave like the old one + functools.update_wrapper(new_callback, func) + + return new_callback diff --git a/tutor/commands/k8s.py b/tutor/commands/k8s.py index 9b3e85d..2313e28 100644 --- a/tutor/commands/k8s.py +++ b/tutor/commands/k8s.py @@ -6,14 +6,14 @@ import click from tutor import config as tutor_config from tutor import env as tutor_env -from tutor import exceptions, fmt +from tutor import exceptions, fmt, hooks from tutor import interactive as interactive_config from tutor import serialize, utils from tutor.commands import jobs from tutor.commands.config import save as config_save_command -from tutor.commands.context import BaseJobContext +from tutor.commands.context import BaseTaskContext from tutor.commands.upgrade.k8s import upgrade_from -from tutor.jobs import BaseJobRunner +from tutor.tasks import BaseTaskRunner from tutor.types import Config, get_typed @@ -22,7 +22,8 @@ class K8sClients: def __init__(self) -> None: # Loading the kubernetes module here to avoid import overhead - from kubernetes import client, config # pylint: disable=import-outside-toplevel + # pylint: disable=import-outside-toplevel + from kubernetes import client, config config.load_kube_config() self._batch_api = None @@ -48,33 +49,20 @@ class K8sClients: return self._core_api -class K8sJobRunner(BaseJobRunner): - def load_job(self, name: str) -> Any: - all_jobs = self.render("k8s", "jobs.yml") - for job in serialize.load_all(all_jobs): - job_name = job["metadata"]["name"] - if not isinstance(job_name, str): - raise exceptions.TutorError( - f"Invalid job name: '{job_name}'. Expected str." - ) - if job_name == name: - return job - raise exceptions.TutorError(f"Could not find job '{name}'") +class K8sTaskRunner(BaseTaskRunner): + """ + Run tasks (bash commands) in Kubernetes-managed services. - def active_job_names(self) -> List[str]: - """ - Return a list of active job names - Docs: - https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#list-job-v1-batch - """ - api = K8sClients.instance().batch_api - return [ - job.metadata.name - for job in api.list_namespaced_job(k8s_namespace(self.config)).items - if job.status.active - ] + Note: a single Tutor "task" correspond to a Kubernetes "job": + https://kubernetes.io/docs/concepts/workloads/controllers/job/ + A Tutor "job" is composed of multiple Tutor tasks run in different services. - def run_job(self, service: str, command: str) -> int: + In Kubernetes, each task that is expected to run in a "myservice" container will + trigger the "myservice-job" Kubernetes job. This job definition must be present in + the "k8s/jobs.yml" template. + """ + + def run_task(self, service: str, command: str) -> int: job_name = f"{service}-job" job = self.load_job(job_name) # Create a unique job name to make it deduplicate jobs and make it easier to @@ -150,10 +138,41 @@ class K8sJobRunner(BaseJobRunner): sleep(5) return 0 + def load_job(self, name: str) -> Any: + """ + Find a given job definition in the rendered k8s/jobs.yml template. + """ + all_jobs = self.render("k8s", "jobs.yml") + for job in serialize.load_all(all_jobs): + job_name = job["metadata"]["name"] + if not isinstance(job_name, str): + raise exceptions.TutorError( + f"Invalid job name: '{job_name}'. Expected str." + ) + if job_name == name: + return job + raise exceptions.TutorError(f"Could not find job '{name}'") -class K8sContext(BaseJobContext): - def job_runner(self, config: Config) -> K8sJobRunner: - return K8sJobRunner(self.root, config) + def active_job_names(self) -> List[str]: + """ + Return a list of active job names + Docs: + https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#list-job-v1-batch + + This is necessary to make sure that we don't run the same job multiple times at + the same time. + """ + api = K8sClients.instance().batch_api + return [ + job.metadata.name + for job in api.list_namespaced_job(k8s_namespace(self.config)).items + if job.status.active + ] + + +class K8sContext(BaseTaskContext): + def job_runner(self, config: Config) -> K8sTaskRunner: + return K8sTaskRunner(self.root, config) @click.group(help="Run Open edX on Kubernetes") @@ -341,17 +360,33 @@ def delete(context: K8sContext, yes: bool) -> None: ) +@jobs.do_group +@click.pass_obj +def do(context: K8sContext) -> None: + """ + Run a custom job in the right container(s). + + We make sure that some essential containers (databases, proxy) are up before we + launch the jobs. + """ + @hooks.Actions.DO_JOB.add() + def _start_base_deployments(_job_name: str, *_args: Any, **_kwargs: Any) -> None: + """ + We add this logic to an action callback because we do not want to trigger it + whenever we run `tutor k8s do --help`. + """ + config = tutor_config.load(context.root) + wait_for_deployment_ready(config, "caddy") + for name in ["elasticsearch", "mysql", "mongodb"]: + if tutor_config.is_service_activated(config, name): + wait_for_deployment_ready(config, name) + + @click.command(help="Initialise all applications") @click.option("-l", "--limit", help="Limit initialisation to this service or plugin") -@click.pass_obj -def init(context: K8sContext, limit: Optional[str]) -> None: - config = tutor_config.load(context.root) - runner = context.job_runner(config) - wait_for_deployment_ready(config, "caddy") - for name in ["elasticsearch", "mysql", "mongodb"]: - if tutor_config.is_service_activated(config, name): - wait_for_deployment_ready(config, name) - jobs.initialise(runner, limit_to=limit) +@click.pass_context +def init(context: click.Context, limit: Optional[str]) -> None: + context.invoke(do.commands["init"], limit=limit) @click.command(help="Scale the number of replicas of a given deployment") @@ -540,4 +575,9 @@ k8s.add_command(wait) k8s.add_command(upgrade) k8s.add_command(apply_command) k8s.add_command(status) -jobs.add_commands(k8s) + + +@hooks.Actions.PLUGINS_LOADED.add() +def _add_k8s_do_commands() -> None: + jobs.add_job_commands(do) + k8s.add_command(do) diff --git a/tutor/commands/local.py b/tutor/commands/local.py index e4614c3..258cb54 100644 --- a/tutor/commands/local.py +++ b/tutor/commands/local.py @@ -13,7 +13,7 @@ from tutor.commands.upgrade.local import upgrade_from from tutor.types import Config, get_typed -class LocalJobRunner(compose.ComposeJobRunner): +class LocalTaskRunner(compose.ComposeTaskRunner): def __init__(self, root: str, config: Config): """ Load docker-compose files from local/. @@ -52,8 +52,8 @@ class LocalContext(compose.BaseComposeContext): COMPOSE_TMP_FILTER = hooks.Filters.COMPOSE_LOCAL_TMP COMPOSE_JOBS_TMP_FILTER = hooks.Filters.COMPOSE_LOCAL_JOBS_TMP - def job_runner(self, config: Config) -> LocalJobRunner: - return LocalJobRunner(self.root, config) + def job_runner(self, config: Config) -> LocalTaskRunner: + return LocalTaskRunner(self.root, config) @click.group(help="Run Open edX locally with docker-compose") @@ -142,7 +142,7 @@ Press enter when you are ready to continue""" click.echo(fmt.title("Starting the platform in detached mode")) context.invoke(compose.start, detach=True) click.echo(fmt.title("Database creation and migrations")) - context.invoke(compose.init) + context.invoke(compose.do.commands["init"]) config = tutor_config.load(context.obj.root) fmt.echo_info( @@ -212,7 +212,7 @@ def _stop_on_dev_start(root: str, config: Config, project_name: str) -> None: Stop the local platform as soon as a platform with a different project name is started. """ - runner = LocalJobRunner(root, config) + runner = LocalTaskRunner(root, config) if project_name != runner.project_name: runner.docker_compose("stop") diff --git a/tutor/hooks/__init__.py b/tutor/hooks/__init__.py index 156d6e6..3944f29 100644 --- a/tutor/hooks/__init__.py +++ b/tutor/hooks/__init__.py @@ -4,7 +4,7 @@ __license__ = "Apache 2.0" import typing as t # These imports are the hooks API -from . import actions, contexts, filters +from . import actions, contexts, filters, priorities from .consts import * diff --git a/tutor/hooks/consts.py b/tutor/hooks/consts.py index 9a09cb9..e863e97 100644 --- a/tutor/hooks/consts.py +++ b/tutor/hooks/consts.py @@ -47,6 +47,13 @@ class Actions: #: This action does not have any parameter. CORE_READY = actions.get("core:ready") + #: Called just before triggering the job tasks of any `... do ` command. + #: + #: :parameter: str job: job name. + #: :parameter: args: job positional arguments. + #: :parameter: kwargs: job named arguments. + DO_JOB = actions.get("do:job") + #: Called as soon as we have access to the Tutor project root. #: #: :parameter str root: absolute path to the project root. @@ -104,6 +111,31 @@ class Filters: return items """ + #: List of command line interface (CLI) commands. + #: + #: :parameter list commands: commands are instances of ``click.Command``. They will + #: all be added as subcommands of the main ``tutor`` command. + CLI_COMMANDS = filters.get("cli:commands") + + #: List of `do ...` commands. + #: + #: :parameter list commands: see :py:data:`CLI_COMMANDS`. These commands will be + #: added as subcommands to the `local/dev/k8s do` commands. They must return a list of + #: ("service name", "service command") tuples. Each "service command" will be executed + #: in the "service" container, both in local, dev and k8s mode. + CLI_DO_COMMANDS = filters.get("cli:commands:do") + + #: List of initialization tasks (scripts) to be run in the `init` job. This job + #: includes all database migrations, setting up, etc. To run some tasks before or + #: after others, they should be assigned a different priority. + #: + #: :parameter list[tuple[str, str]] tasks: list of ``(service, task)`` tuples. Each + #: task is essentially a bash script to be run in the "service" container. Scripts + #: may contain Jinja markup, similar to templates. + CLI_DO_INIT_TASKS = filters.get("cli:commands:do:init") + + #: DEPRECATED use :py:data:`CLI_DO_INIT_TASKS` instead. + #: #: List of commands to be executed during initialization. These commands typically #: include database migrations, setting feature flags, etc. #: @@ -111,14 +143,17 @@ class Filters: #: #: - ``service`` is the name of the container in which the task will be executed. #: - ``path`` is a tuple that corresponds to a template relative path. - #: Example: ``("myplugin", "hooks", "myservice", "pre-init")`` (see:py:data:`IMAGES_BUILD`). + #: Example: ``("myplugin", "hooks", "myservice", "pre-init")`` (see :py:data:`IMAGES_BUILD`). #: The command to execute will be read from that template, after it is rendered. COMMANDS_INIT = filters.get("commands:init") + #: DEPRECATED use :py:data:`CLI_DO_INIT_TASKS` instead with a lower priority score. + #: #: List of commands to be executed prior to initialization. These commands are run even #: before the mysql databases are created and the migrations are applied. #: - #: :parameter list[tuple[str, tuple[str, ...]]] tasks: list of ``(service, path)`` tasks. (see :py:data:`COMMANDS_INIT`). + #: :parameter list[tuple[str, tuple[str, ...]]] tasks: list of ``(service, path)`` + #: tasks. (see :py:data:`COMMANDS_INIT`). COMMANDS_PRE_INIT = filters.get("commands:pre-init") #: Same as :py:data:`COMPOSE_LOCAL_TMP` but for the development environment. @@ -159,40 +194,6 @@ class Filters: #: Same as :py:data:`COMPOSE_LOCAL_TMP` but for jobs COMPOSE_LOCAL_JOBS_TMP = filters.get("compose:local-jobs:tmp") - #: List of images to be built when we run ``tutor images build ...``. - #: - #: :parameter list[tuple[str, tuple[str, ...], str, tuple[str, ...]]] tasks: list of ``(name, path, tag, args)`` tuples. - #: - #: - ``name`` is the name of the image, as in ``tutor images build myimage``. - #: - ``path`` is the relative path to the folder that contains the Dockerfile. - #: For instance ``("myplugin", "build", "myservice")`` indicates that the template will be read from - #: ``myplugin/build/myservice/Dockerfile`` - #: - ``tag`` is the Docker tag that will be applied to the image. It will be - #: rendered at runtime with the user configuration. Thus, the image tag could - #: be ``"{{ DOCKER_REGISTRY }}/myimage:{{ TUTOR_VERSION }}"``. - #: - ``args`` is a list of arguments that will be passed to ``docker build ...``. - #: :parameter dict config: user configuration. - IMAGES_BUILD = filters.get("images:build") - - #: List of images to be pulled when we run ``tutor images pull ...``. - #: - #: :parameter list[tuple[str, str]] tasks: list of ``(name, tag)`` tuples. - #: - #: - ``name`` is the name of the image, as in ``tutor images pull myimage``. - #: - ``tag`` is the Docker tag that will be applied to the image. (see :py:data:`IMAGES_BUILD`). - #: :parameter dict config: user configuration. - IMAGES_PULL = filters.get("images:pull") - - #: List of images to be pushed when we run ``tutor images push ...``. - #: Parameters are the same as for :py:data:`IMAGES_PULL`. - IMAGES_PUSH = filters.get("images:push") - - #: List of command line interface (CLI) commands. - #: - #: :parameter list commands: commands are instances of ``click.Command``. They will - #: all be added as subcommands of the main ``tutor`` command. - CLI_COMMANDS = filters.get("cli:commands") - #: Declare new default configuration settings that don't necessarily have to be saved in the user #: ``config.yml`` file. Default settings may be overridden with ``tutor config save --set=...``, in which #: case they will automatically be added to ``config.yml``. @@ -298,6 +299,34 @@ class Filters: #: :parameter filters: list of (name, value) tuples. ENV_TEMPLATE_VARIABLES = filters.get("env:templates:variables") + #: List of images to be built when we run ``tutor images build ...``. + #: + #: :parameter list[tuple[str, tuple[str, ...], str, tuple[str, ...]]] tasks: list of ``(name, path, tag, args)`` tuples. + #: + #: - ``name`` is the name of the image, as in ``tutor images build myimage``. + #: - ``path`` is the relative path to the folder that contains the Dockerfile. + #: For instance ``("myplugin", "build", "myservice")`` indicates that the template will be read from + #: ``myplugin/build/myservice/Dockerfile`` + #: - ``tag`` is the Docker tag that will be applied to the image. It will be + #: rendered at runtime with the user configuration. Thus, the image tag could + #: be ``"{{ DOCKER_REGISTRY }}/myimage:{{ TUTOR_VERSION }}"``. + #: - ``args`` is a list of arguments that will be passed to ``docker build ...``. + #: :parameter dict config: user configuration. + IMAGES_BUILD = filters.get("images:build") + + #: List of images to be pulled when we run ``tutor images pull ...``. + #: + #: :parameter list[tuple[str, str]] tasks: list of ``(name, tag)`` tuples. + #: + #: - ``name`` is the name of the image, as in ``tutor images pull myimage``. + #: - ``tag`` is the Docker tag that will be applied to the image. (see :py:data:`IMAGES_BUILD`). + #: :parameter dict config: user configuration. + IMAGES_PULL = filters.get("images:pull") + + #: List of images to be pushed when we run ``tutor images push ...``. + #: Parameters are the same as for :py:data:`IMAGES_PULL`. + IMAGES_PUSH = filters.get("images:push") + #: List of installed plugins. In order to be added to this list, a plugin must first #: be discovered (see :py:data:`Actions.CORE_READY`). #: diff --git a/tutor/plugins/v0.py b/tutor/plugins/v0.py index 52e0c54..74d88cc 100644 --- a/tutor/plugins/v0.py +++ b/tutor/plugins/v0.py @@ -7,7 +7,7 @@ from glob import glob import click import pkg_resources -from tutor import exceptions, fmt, hooks, serialize +from tutor import env, exceptions, fmt, hooks, serialize from tutor.__about__ import __app__ from tutor.types import Config @@ -179,12 +179,18 @@ class BasePlugin: ) # Pre-init scripts: hooks = {"pre-init": ["myservice1", "myservice2"]} for service in pre_init_tasks: - path = (self.name, "hooks", service, "pre-init") - hooks.Filters.COMMANDS_PRE_INIT.add_item((service, path)) + hooks.Filters.CLI_DO_INIT_TASKS.add_item( + ( + service, + env.read_template_file(self.name, "hooks", service, "pre-init"), + ), + priority=hooks.priorities.HIGH, + ) # Init scripts: hooks = {"init": ["myservice1", "myservice2"]} for service in init_tasks: - path = (self.name, "hooks", service, "init") - hooks.Filters.COMMANDS_INIT.add_item((service, path)) + hooks.Filters.CLI_DO_INIT_TASKS.add_item( + (service, env.read_template_file(self.name, "hooks", service, "init")) + ) def _load_templates_root(self) -> None: templates_root = get_callable_attr(self.obj, "templates", default=None) diff --git a/tutor/jobs.py b/tutor/tasks.py similarity index 67% rename from tutor/jobs.py rename to tutor/tasks.py index 6f145e9..1e1de75 100644 --- a/tutor/jobs.py +++ b/tutor/tasks.py @@ -2,11 +2,11 @@ from tutor import env from tutor.types import Config -class BaseJobRunner: +class BaseTaskRunner: """ - A job runner is responsible for running bash commands in the right context. + A task runner is responsible for running bash commands in the right context. - Commands may be loaded from string or template files. The `run_job` method must be + Commands may be loaded from string or template files. The `run_task` method must be implemented by child classes. """ @@ -14,13 +14,13 @@ class BaseJobRunner: self.root = root self.config = config - def run_job_from_template(self, service: str, *path: str) -> None: + def run_task_from_template(self, service: str, *path: str) -> None: command = self.render(*path) - self.run_job(service, command) + self.run_task(service, command) - def run_job_from_str(self, service: str, command: str) -> None: + def run_task_from_str(self, service: str, command: str) -> None: rendered = env.render_str(self.config, command).strip() - self.run_job(service, rendered) + self.run_task(service, rendered) def render(self, *path: str) -> str: rendered = env.render_file(self.config, *path).strip() @@ -28,7 +28,7 @@ class BaseJobRunner: raise TypeError("Cannot load job from binary file") return rendered - def run_job(self, service: str, command: str) -> int: + def run_task(self, service: str, command: str) -> int: """ Given a (potentially large) string command, run it with the corresponding service. Implementations will differ depending on the @@ -37,6 +37,6 @@ class BaseJobRunner: raise NotImplementedError -class BaseComposeJobRunner(BaseJobRunner): +class BaseComposeTaskRunner(BaseTaskRunner): def docker_compose(self, *command: str) -> int: raise NotImplementedError diff --git a/tutor/templates/hooks/cms/init b/tutor/templates/jobs/init/cms.sh similarity index 100% rename from tutor/templates/hooks/cms/init rename to tutor/templates/jobs/init/cms.sh diff --git a/tutor/templates/hooks/lms/init b/tutor/templates/jobs/init/lms.sh similarity index 100% rename from tutor/templates/hooks/lms/init rename to tutor/templates/jobs/init/lms.sh diff --git a/tutor/templates/hooks/mysql/init b/tutor/templates/jobs/init/mysql.sh similarity index 100% rename from tutor/templates/hooks/mysql/init rename to tutor/templates/jobs/init/mysql.sh diff --git a/tutor/utils.py b/tutor/utils.py index 122a9b3..f2d6b80 100644 --- a/tutor/utils.py +++ b/tutor/utils.py @@ -203,6 +203,10 @@ def is_a_tty() -> bool: def execute(*command: str) -> int: click.echo(fmt.command(_shlex_join(*command))) + return execute_silent(*command) + + +def execute_silent(*command: str) -> int: with subprocess.Popen(command) as p: try: result = p.wait(timeout=None) From 0453b0c0023101aaceb31944c90f6fd018c7cd44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Fri, 11 Nov 2022 14:13:36 +0100 Subject: [PATCH 53/56] feat: add `-h` help option to all commands --- CHANGELOG-nightly.md | 1 + tutor/commands/cli.py | 1 + tutor/commands/compose.py | 1 + tutor/commands/jobs.py | 3 +-- tutor/commands/k8s.py | 1 + 5 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG-nightly.md b/CHANGELOG-nightly.md index 99daf2b..55903cf 100644 --- a/CHANGELOG-nightly.md +++ b/CHANGELOG-nightly.md @@ -8,6 +8,7 @@ will be backported to the master branch at every major release. When backporting changes to master, we should keep only the entries that correspond to user- facing changes. --> +- [Improvement] Add the `-h` help option to all commands and subcommands. Previously, we could only use `--help`, which is quite long for lazy fingers. (by @regisb) - 💥[Feature] Add an extensible `local/dev/k8s do ...` command to trigger custom job commands. These commands are used to run a series of bash scripts in designated containers. Any plugin can add custom jobs thanks to the `CLI_DO_COMMANDS` filter. This causes the following breaking changes: - The "init", "createuser", "settheme", "importdemocourse" commands were all migrated to this new interface. For instance, `tutor local init` was replaced by `tutor local do init`. - Plugin developers are encouraged to replace calls to the `COMMANDS_INIT` and `COMMANDS_PRE_INIT` filters by `CLI_DO_INIT_TASKS`. diff --git a/tutor/commands/cli.py b/tutor/commands/cli.py index a4d75e0..80c6475 100755 --- a/tutor/commands/cli.py +++ b/tutor/commands/cli.py @@ -117,6 +117,7 @@ def cli(context: click.Context, root: str, show_help: bool) -> None: "/install/linux/linux-postinstall/#manage-docker-as-a-non-root-user)" ) context.obj = Context(root) + context.help_option_names = ["-h", "--help"] if context.invoked_subcommand is None or show_help: click.echo(context.get_help()) diff --git a/tutor/commands/compose.py b/tutor/commands/compose.py index 85a95b1..7f43f19 100644 --- a/tutor/commands/compose.py +++ b/tutor/commands/compose.py @@ -306,6 +306,7 @@ def do( """ Run a custom job in the right container(s). """ + @hooks.Actions.DO_JOB.add() def _mount_tmp_volumes(_job_name: str, *_args: t.Any, **_kwargs: t.Any) -> None: """ diff --git a/tutor/commands/jobs.py b/tutor/commands/jobs.py index 9f8ac6a..db17a7d 100644 --- a/tutor/commands/jobs.py +++ b/tutor/commands/jobs.py @@ -263,8 +263,7 @@ P = ParamSpec("P") def _patch_callback( - job_name: str, - func: t.Callable[P, t.Iterable[t.Tuple[str, str]]] + job_name: str, func: t.Callable[P, t.Iterable[t.Tuple[str, str]]] ) -> t.Callable[P, None]: """ Modify a subcommand callback function such that its results are processed by `do_callback`. diff --git a/tutor/commands/k8s.py b/tutor/commands/k8s.py index 2313e28..243eb28 100644 --- a/tutor/commands/k8s.py +++ b/tutor/commands/k8s.py @@ -369,6 +369,7 @@ def do(context: K8sContext) -> None: We make sure that some essential containers (databases, proxy) are up before we launch the jobs. """ + @hooks.Actions.DO_JOB.add() def _start_base_deployments(_job_name: str, *_args: Any, **_kwargs: Any) -> None: """ From b4a8069cfd74ee256cc194684206725dabff4567 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Mon, 21 Nov 2022 09:56:59 +0100 Subject: [PATCH 54/56] fix: init template paths Plugin template paths of init jobs could not be found because they were searched for in the Tutor template root only. --- tests/test_env.py | 4 ++-- tutor/commands/jobs.py | 6 ++--- tutor/config.py | 2 +- tutor/env.py | 53 ++++++++++++++++++++++++------------------ 4 files changed, 36 insertions(+), 29 deletions(-) diff --git a/tests/test_env.py b/tests/test_env.py index 9fd9c7b..e74475d 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -42,8 +42,8 @@ class EnvTests(PluginsTestCase): self.assertTrue(env.is_binary_file("/home/somefile.ico")) def test_find_os_path(self) -> None: - renderer = env.Renderer() - path = renderer.find_os_path("local/docker-compose.yml") + environment = env.JinjaEnvironment() + path = environment.find_os_path("local/docker-compose.yml") self.assertTrue(os.path.exists(path)) def test_pathjoin(self) -> None: diff --git a/tutor/commands/jobs.py b/tutor/commands/jobs.py index 559dd87..ce7a18b 100644 --- a/tutor/commands/jobs.py +++ b/tutor/commands/jobs.py @@ -45,15 +45,15 @@ def _add_core_init_tasks() -> None: """ with hooks.Contexts.APP("mysql").enter(): hooks.Filters.CLI_DO_INIT_TASKS.add_item( - ("mysql", env.read_template_file("jobs", "init", "mysql.sh")) + ("mysql", env.read_core_template_file("jobs", "init", "mysql.sh")) ) with hooks.Contexts.APP("lms").enter(): hooks.Filters.CLI_DO_INIT_TASKS.add_item( - ("lms", env.read_template_file("jobs", "init", "lms.sh")) + ("lms", env.read_core_template_file("jobs", "init", "lms.sh")) ) with hooks.Contexts.APP("cms").enter(): hooks.Filters.CLI_DO_INIT_TASKS.add_item( - ("cms", env.read_template_file("jobs", "init", "cms.sh")) + ("cms", env.read_core_template_file("jobs", "init", "cms.sh")) ) diff --git a/tutor/config.py b/tutor/config.py index d6a06fa..b77a494 100644 --- a/tutor/config.py +++ b/tutor/config.py @@ -139,7 +139,7 @@ def get_template(filename: str) -> Config: Entries in this configuration are unrendered. """ - config = serialize.load(env.read_template_file("config", filename)) + config = serialize.load(env.read_core_template_file("config", filename)) return cast_config(config) diff --git a/tutor/env.py b/tutor/env.py index f5d2fa3..6206092 100644 --- a/tutor/env.py +++ b/tutor/env.py @@ -60,20 +60,35 @@ _prepare_environment() class JinjaEnvironment(jinja2.Environment): - loader: jinja2.BaseLoader + loader: jinja2.FileSystemLoader - def __init__(self, template_roots: t.List[str]) -> None: + def __init__(self) -> None: + template_roots = hooks.Filters.ENV_TEMPLATE_ROOTS.apply([TEMPLATES_ROOT]) loader = jinja2.FileSystemLoader(template_roots) super().__init__(loader=loader, undefined=jinja2.StrictUndefined) + def read_str(self, template_name: str) -> str: + return self.read_bytes(template_name).decode() + + def read_bytes(self, template_name: str) -> bytes: + with open(self.find_os_path(template_name), "rb") as f: + return f.read() + + def find_os_path(self, template_name: str) -> str: + path = template_name.replace("/", os.sep) + for templates_root in self.loader.searchpath: + full_path = os.path.join(templates_root, path) + if os.path.exists(full_path): + return full_path + raise ValueError("Template path does not exist") + class Renderer: def __init__(self, config: t.Optional[Config] = None): self.config = deepcopy(config or {}) - self.template_roots = hooks.Filters.ENV_TEMPLATE_ROOTS.apply([TEMPLATES_ROOT]) # Create environment with extra filters and globals - self.environment = JinjaEnvironment(self.template_roots) + self.environment = JinjaEnvironment() # Filters plugin_filters = hooks.Filters.ENV_TEMPLATE_FILTERS.iterate() @@ -135,14 +150,6 @@ class Renderer: """ yield from self.iter_templates_in(subdir) - def find_os_path(self, template_name: str) -> str: - path = template_name.replace("/", os.sep) - for templates_root in self.template_roots: - full_path = os.path.join(templates_root, path) - if os.path.exists(full_path): - return full_path - raise ValueError("Template path does not exist") - def patch(self, name: str, separator: str = "\n", suffix: str = "") -> str: """ Render calls to {{ patch("...") }} in environment templates from plugin patches. @@ -173,8 +180,7 @@ class Renderer: """ if is_binary_file(template_name): # Don't try to render binary files - with open(self.find_os_path(template_name), "rb") as f: - return f.read() + return self.environment.read_bytes(template_name) try: template = self.environment.get_template(template_name) @@ -396,9 +402,17 @@ def current_version(root: str) -> t.Optional[str]: def read_template_file(*path: str) -> str: """ Read raw content of template located at `path`. + + The template may be located inside any of the template root folders. """ - src = template_path(*path) - with open(src, encoding="utf-8") as fi: + return JinjaEnvironment().read_str("/".join(path)) + + +def read_core_template_file(*path: str) -> str: + """ + Read raw content of template located in tutor core template directory. + """ + with open(os.path.join(TEMPLATES_ROOT, *path), encoding="utf-8") as fi: return fi.read() @@ -407,13 +421,6 @@ def is_binary_file(path: str) -> bool: return ext in BIN_FILE_EXTENSIONS -def template_path(*path: str, templates_root: str = TEMPLATES_ROOT) -> str: - """ - Return the template file's absolute path. - """ - return os.path.join(templates_root, *path) - - def data_path(root: str, *path: str) -> str: """ Return the file's absolute path inside the data directory. From 478d44c299996f225bf68353b47ab4c572ae0b34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Thu, 24 Nov 2022 18:20:23 +0100 Subject: [PATCH 55/56] fix: type DO_* filters and actions --- tutor/commands/jobs.py | 91 ++++++++++++++++++++++-------------------- tutor/hooks/consts.py | 12 ++++-- 2 files changed, 56 insertions(+), 47 deletions(-) diff --git a/tutor/commands/jobs.py b/tutor/commands/jobs.py index ce7a18b..b101cba 100644 --- a/tutor/commands/jobs.py +++ b/tutor/commands/jobs.py @@ -25,16 +25,6 @@ class DoGroup(click.Group): do_group = click.group(cls=DoGroup, subcommand_metavar="JOB [ARGS]...") -def add_job_commands(do_command_group: click.Group) -> None: - """ - This is meant to be called with the `local/dev/k8s do` group commands, to add the - different `do` subcommands. - """ - subcommands: t.Iterator[click.Command] = hooks.Filters.CLI_DO_COMMANDS.iterate() - for subcommand in subcommands: - do_command_group.add_command(subcommand) - - @hooks.Actions.CORE_READY.add() def _add_core_init_tasks() -> None: """ @@ -205,39 +195,14 @@ def assign_theme(name, domain): return f'./manage.py lms shell -c "{python_command}"' -hooks.Filters.CLI_DO_COMMANDS.add_items( - [ - createuser, - importdemocourse, - initialise, - settheme, - ] -) - - -def do_callback(service_commands: t.Iterable[t.Tuple[str, str]]) -> None: +def add_job_commands(do_command_group: click.Group) -> None: """ - This function must be added as a callback to all `do` subcommands. - - `do` subcommands don't actually run any task. They just yield tuples of (service - name, unrendered script string). This function is responsible for actually running - the scripts. It does the following: - - - Prefix the script with a base command - - Render the script string - - Run a job in the right container - - In order to be added as a callback to the do subcommands, the - `_patch_do_commands_callbacks` must be called. + This is meant to be called with the `local/dev/k8s do` group commands, to add the + different `do` subcommands. """ - context = click.get_current_context().obj - config = tutor_config.load(context.root) - runner = context.job_runner(config) - base_openedx_command = """ -echo "Loading settings $DJANGO_SETTINGS_MODULE" -""" - for service, command in service_commands: - runner.run_task_from_str(service, base_openedx_command + command) + for subcommand in hooks.Filters.CLI_DO_COMMANDS.iterate(): + assert isinstance(subcommand, click.Command) + do_command_group.add_command(subcommand) @hooks.Actions.PLUGINS_LOADED.add() @@ -245,9 +210,15 @@ def _patch_do_commands_callbacks() -> None: """ After plugins have been loaded, patch `do` subcommands such that their output is forwarded to `do_callback`. + + This function is not called as part of add_job_commands because subcommands must be + patched just once. """ - subcommands: t.Iterator[click.Command] = hooks.Filters.CLI_DO_COMMANDS.iterate() - for subcommand in subcommands: + for subcommand in hooks.Filters.CLI_DO_COMMANDS.iterate(): + if not isinstance(subcommand, click.Command): + raise ValueError( + f"Command {subcommand} which was added to the CLI_DO_COMMANDS filter must be an instance of click.Command" + ) # Modify the subcommand callback such that job results are processed by do_callback if subcommand.callback is None: raise ValueError("Cannot patch None callback") @@ -274,3 +245,37 @@ def _patch_callback( functools.update_wrapper(new_callback, func) return new_callback + + +def do_callback(service_commands: t.Iterable[t.Tuple[str, str]]) -> None: + """ + This function must be added as a callback to all `do` subcommands. + + `do` subcommands don't actually run any task. They just yield tuples of (service + name, unrendered script string). This function is responsible for actually running + the scripts. It does the following: + + - Prefix the script with a base command + - Render the script string + - Run a job in the right container + + This callback is added to the "do" subcommands by the `add_job_commands` function. + """ + context = click.get_current_context().obj + config = tutor_config.load(context.root) + runner = context.job_runner(config) + base_openedx_command = """ +echo "Loading settings $DJANGO_SETTINGS_MODULE" +""" + for service, command in service_commands: + runner.run_task_from_str(service, base_openedx_command + command) + + +hooks.Filters.CLI_DO_COMMANDS.add_items( + [ + createuser, + importdemocourse, + initialise, + settheme, + ] +) diff --git a/tutor/hooks/consts.py b/tutor/hooks/consts.py index 6f086f1..e27fd28 100644 --- a/tutor/hooks/consts.py +++ b/tutor/hooks/consts.py @@ -7,7 +7,7 @@ from __future__ import annotations # The Tutor plugin system is licensed under the terms of the Apache 2.0 license. __license__ = "Apache 2.0" -from typing import Any, Callable +from typing import Any, Callable, Iterable import click @@ -64,7 +64,7 @@ class Actions: #: :parameter: str job: job name. #: :parameter: args: job positional arguments. #: :parameter: kwargs: job named arguments. - DO_JOB = actions.get("do:job") + DO_JOB: Action[[str, Any]] = actions.get("do:job") #: Called as soon as we have access to the Tutor project root. #: @@ -135,7 +135,9 @@ class Filters: #: added as subcommands to the `local/dev/k8s do` commands. They must return a list of #: ("service name", "service command") tuples. Each "service command" will be executed #: in the "service" container, both in local, dev and k8s mode. - CLI_DO_COMMANDS = filters.get("cli:commands:do") + CLI_DO_COMMANDS: Filter[ + list[Callable[[Any], Iterable[tuple[str, str]]]], [] + ] = filters.get("cli:commands:do") #: List of initialization tasks (scripts) to be run in the `init` job. This job #: includes all database migrations, setting up, etc. To run some tasks before or @@ -144,7 +146,9 @@ class Filters: #: :parameter list[tuple[str, str]] tasks: list of ``(service, task)`` tuples. Each #: task is essentially a bash script to be run in the "service" container. Scripts #: may contain Jinja markup, similar to templates. - CLI_DO_INIT_TASKS = filters.get("cli:commands:do:init") + CLI_DO_INIT_TASKS: Filter[list[tuple[str, str]], []] = filters.get( + "cli:commands:do:init" + ) #: DEPRECATED use :py:data:`CLI_DO_INIT_TASKS` instead. #: From 85d868423a941bd04118331e6b7dcf4ae09842e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Thu, 24 Nov 2022 18:26:00 +0100 Subject: [PATCH 56/56] fix: do not prepend DJANGO settings info to all jobs The DJANGO_SETTINGS_MODULE is far from being relevant in all containers. --- tests/commands/test_jobs.py | 11 ++--------- tutor/commands/jobs.py | 5 +---- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/tests/commands/test_jobs.py b/tests/commands/test_jobs.py index f5f8bb3..793931e 100644 --- a/tests/commands/test_jobs.py +++ b/tests/commands/test_jobs.py @@ -37,10 +37,8 @@ class JobsTests(PluginsTestCase, TestCommandMixin): self.assertIsNone(result.exception) self.assertEqual(0, result.exit_code) self.assertIn("cms-job", dc_args) - self.assertTrue( - dc_args[-1] - .strip() - .startswith('echo "Loading settings $DJANGO_SETTINGS_MODULE"') + self.assertIn( + "git clone https://github.com/openedx/edx-demo-course", dc_args[-1] ) def test_set_theme(self) -> None: @@ -65,10 +63,5 @@ class JobsTests(PluginsTestCase, TestCommandMixin): self.assertIsNone(result.exception) self.assertEqual(0, result.exit_code) self.assertIn("lms-job", dc_args) - self.assertTrue( - dc_args[-1] - .strip() - .startswith('echo "Loading settings $DJANGO_SETTINGS_MODULE"') - ) self.assertIn("assign_theme('beautiful', 'domain1')", dc_args[-1]) self.assertIn("assign_theme('beautiful', 'domain2')", dc_args[-1]) diff --git a/tutor/commands/jobs.py b/tutor/commands/jobs.py index b101cba..9840307 100644 --- a/tutor/commands/jobs.py +++ b/tutor/commands/jobs.py @@ -264,11 +264,8 @@ def do_callback(service_commands: t.Iterable[t.Tuple[str, str]]) -> None: context = click.get_current_context().obj config = tutor_config.load(context.root) runner = context.job_runner(config) - base_openedx_command = """ -echo "Loading settings $DJANGO_SETTINGS_MODULE" -""" for service, command in service_commands: - runner.run_task_from_str(service, base_openedx_command + command) + runner.run_task_from_str(service, command) hooks.Filters.CLI_DO_COMMANDS.add_items(