From b8ab829c11f66f2747559324901d87d291d5ee6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Mon, 25 Oct 2021 19:32:13 +0200 Subject: [PATCH] feat: move all forum-related code to a dedicated plugin Forum is an optional feature, and as such it deserves its own plugin. Starting from Maple, users will be able to install the forum from https://github.com/overhangio/tutor-forum/ Close #450. --- CHANGELOG-nightly.md | 1 + Makefile | 2 +- bin/main.py | 1 + docs/configuration.rst | 11 +---- docs/plugins/api.rst | 2 +- requirements/plugins.txt | 1 + tutor/commands/images.py | 2 +- tutor/config.py | 1 - tutor/jobs.py | 2 +- .../apps/openedx/config/lms.env.json | 2 - .../apps/openedx/settings/lms/development.py | 1 - tutor/templates/build/forum/Dockerfile | 48 ------------------- .../build/forum/bin/docker-entrypoint.sh | 16 ------- tutor/templates/config.yml | 4 -- tutor/templates/k8s/deployments.yml | 39 --------------- tutor/templates/k8s/jobs.yml | 27 ----------- tutor/templates/k8s/services.yml | 14 ------ tutor/templates/local/docker-compose.jobs.yml | 10 ---- tutor/templates/local/docker-compose.yml | 16 ------- 19 files changed, 8 insertions(+), 192 deletions(-) delete mode 100644 tutor/templates/build/forum/Dockerfile delete mode 100755 tutor/templates/build/forum/bin/docker-entrypoint.sh diff --git a/CHANGELOG-nightly.md b/CHANGELOG-nightly.md index 7a7077d..4ae842e 100644 --- a/CHANGELOG-nightly.md +++ b/CHANGELOG-nightly.md @@ -2,6 +2,7 @@ Note: Breaking changes between versions are indicated by "💥". +- 💥[Improvement] Move the Open edX forum to a [dedicated plugin](https://github.com/overhangio/tutor-forum/) (#450). - 💥[Improvement] Get rid of the "tutor-openedx" package, which is no longer supported. - [Bugfix] Fix running Caddy container in k8s, which should always be the case even if `ENABLE_WEB_PROXY` is false. - 💥[Improvement] Run all services as unprivileged containers, for better security. This has multiple consequences: diff --git a/Makefile b/Makefile index 42eef3a..910f93c 100644 --- a/Makefile +++ b/Makefile @@ -92,7 +92,7 @@ ci-test-bundle: ## Run basic tests on bundle yes "" | ./dist/tutor config save --interactive ./dist/tutor config save ./dist/tutor plugins list - ./dist/tutor plugins enable android discovery ecommerce license mfe minio notes richie webui xqueue + ./dist/tutor plugins enable android discovery ecommerce forum license mfe minio notes richie webui xqueue ./dist/tutor plugins list ./dist/tutor license --help diff --git a/bin/main.py b/bin/main.py index 0d118c5..20bc050 100755 --- a/bin/main.py +++ b/bin/main.py @@ -6,6 +6,7 @@ for plugin_name in [ "android", "discovery", "ecommerce", + "forum", "license", "mfe", "minio", diff --git a/docs/configuration.rst b/docs/configuration.rst index b681016..abbd235 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -42,7 +42,6 @@ Individual service activation - ``RUN_LMS`` (default: ``true``) - ``RUN_CMS`` (default: ``true``) -- ``RUN_FORUM`` (default: ``true``) - ``RUN_ELASTICSEARCH`` (default: ``true``) - ``RUN_MONGODB`` (default: ``true``) - ``RUN_MYSQL`` (default: ``true``) @@ -61,9 +60,8 @@ Custom images ************* - ``DOCKER_IMAGE_OPENEDX`` (default: ``"{{ DOCKER_REGISTRY }}overhangio/openedx:{{ TUTOR_VERSION }}"``) -- ``DOCKER_IMAGE_FORUM`` (default: ``"{{ DOCKER_REGISTRY }}overhangio/openedx-forum:{{ TUTOR_VERSION }}"``) -These configuration parameters define which image to run for each service. By default, the docker image tag matches the Tutor version it was built with. +This configuration parameter defines the name of the Docker image to run for the lms and cms containers. By default, the Docker image tag matches the Tutor version it was built with. Custom registry *************** @@ -171,13 +169,6 @@ SMTP Note that the SMTP server shipped with Tutor by default does not implement TLS. With external servers, only one of SSL or TLS should be enabled, at most. -Forum -***** - -- ``RUN_FORUM`` (default: ``true``) -- ``FORUM_HOST`` (default: ``"forum"``) -- ``FORUM_MONGODB_DATABASE`` (default: ``"cs_comments_service"``) - SSL/TLS certificates for HTTPS access ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/plugins/api.rst b/docs/plugins/api.rst index 903eb51..ca91c78 100644 --- a/docs/plugins/api.rst +++ b/docs/plugins/api.rst @@ -83,7 +83,7 @@ Example:: During initialisation, "myservice1" and "myservice2" will be run in sequence with the commands defined in the templates ``myplugin/hooks/myservice1/init`` and ``myplugin/hooks/myservice2/init``. -To initialise a "foo" service, Tutor runs the "foo-job" service that is found in the ``env/local/docker-compose.jobs.yml`` file. By default, Tutor comes with a few services in this file: mysql-job, lms-job, cms-job, forum-job. If your plugin requires running custom services during initialisation, you will need to add them to the ``docker-compose.jobs.yml`` template. To do so, just use the "local-docker-compose-jobs-services" patch. +To initialise a "foo" service, Tutor runs the "foo-job" service that is found in the ``env/local/docker-compose.jobs.yml`` file. By default, Tutor comes with a few services in this file: mysql-job, lms-job, cms-job. If your plugin requires running custom services during initialisation, you will need to add them to the ``docker-compose.jobs.yml`` template. To do so, just use the "local-docker-compose-jobs-services" patch. In Kubernetes, the approach is the same, except that jobs are implemented as actual job objects in the ``k8s/jobs.yml`` template. To add your own services there, your plugin should implement the "k8s-jobs" patch. diff --git a/requirements/plugins.txt b/requirements/plugins.txt index 7e74b29..d0644c9 100644 --- a/requirements/plugins.txt +++ b/requirements/plugins.txt @@ -2,6 +2,7 @@ tutor-android>=12.0.0,<13.0.0 tutor-discovery>=12.0.0,<13.0.0 tutor-ecommerce>=12.0.0,<13.0.0 +tutor-forum>=12.0.0,<13.0.0 tutor-license>=12.0.0,<13.0.0 tutor-mfe>=12.0.0,<13.0.0 tutor-minio>=12.0.0,<13.0.0 diff --git a/tutor/commands/images.py b/tutor/commands/images.py index 0e6da5a..af08e91 100644 --- a/tutor/commands/images.py +++ b/tutor/commands/images.py @@ -10,7 +10,7 @@ from .. import plugins from ..types import Config from .context import Context -BASE_IMAGE_NAMES = ["openedx", "forum", "permissions"] +BASE_IMAGE_NAMES = ["openedx", "permissions"] VENDOR_IMAGES = [ "caddy", "elasticsearch", diff --git a/tutor/config.py b/tutor/config.py index 7770c82..3511ede 100644 --- a/tutor/config.py +++ b/tutor/config.py @@ -165,7 +165,6 @@ def upgrade_obsolete(config: Config) -> None: for name in [ "ACTIVATE_LMS", "ACTIVATE_CMS", - "ACTIVATE_FORUM", "ACTIVATE_ELASTICSEARCH", "ACTIVATE_MONGODB", "ACTIVATE_MYSQL", diff --git a/tutor/jobs.py b/tutor/jobs.py index 2238bad..c3bc3da 100644 --- a/tutor/jobs.py +++ b/tutor/jobs.py @@ -63,7 +63,7 @@ def initialise(runner: BaseJobRunner, limit_to: Optional[str] = None) -> None: runner.run_job_from_template( service, plugin_name, "hooks", service, "pre-init" ) - for service in ["lms", "cms", "forum"]: + for service in ["lms", "cms"]: if limit_to is None or limit_to == service: fmt.echo_info("Initialising {}...".format(service)) runner.run_job_from_template(service, "hooks", service, "init") diff --git a/tutor/templates/apps/openedx/config/lms.env.json b/tutor/templates/apps/openedx/config/lms.env.json index 3b73079..e39533a 100644 --- a/tutor/templates/apps/openedx/config/lms.env.json +++ b/tutor/templates/apps/openedx/config/lms.env.json @@ -35,8 +35,6 @@ "CELERY_BROKER_USER": "{{ REDIS_USERNAME }}", "CELERY_BROKER_PASSWORD": "{{ REDIS_PASSWORD }}", "ALTERNATE_WORKER_QUEUES": "cms", - "COMMENTS_SERVICE_URL": "http://{{ FORUM_HOST }}:4567", - "COMMENTS_SERVICE_KEY": "forumapikey", "ENABLE_COMPREHENSIVE_THEMING": true, "COMPREHENSIVE_THEME_DIRS": ["/openedx/themes"], "STATIC_ROOT_BASE": "/openedx/staticfiles", diff --git a/tutor/templates/apps/openedx/settings/lms/development.py b/tutor/templates/apps/openedx/settings/lms/development.py index 09c92a4..1038890 100644 --- a/tutor/templates/apps/openedx/settings/lms/development.py +++ b/tutor/templates/apps/openedx/settings/lms/development.py @@ -18,7 +18,6 @@ CMS_ROOT_URL = "http://{}".format(CMS_BASE) LOGIN_REDIRECT_WHITELIST.append(CMS_BASE) FEATURES['ENABLE_COURSEWARE_MICROFRONTEND'] = False -COMMENTS_SERVICE_URL = "http://{{ FORUM_HOST }}:4567" LOGGING["loggers"]["oauth2_provider"] = { "handlers": ["console"], diff --git a/tutor/templates/build/forum/Dockerfile b/tutor/templates/build/forum/Dockerfile deleted file mode 100644 index cdb059e..0000000 --- a/tutor/templates/build/forum/Dockerfile +++ /dev/null @@ -1,48 +0,0 @@ -FROM docker.io/ruby:2.5.7-slim-stretch -MAINTAINER Overhang.io - -ENV DEBIAN_FRONTEND=noninteractive -RUN apt update && \ - apt upgrade -y && \ - apt install -y git wget autoconf bison build-essential libssl-dev libyaml-dev libreadline6-dev zlib1g-dev libncurses5-dev libffi-dev libgdbm-dev - -# Install dockerize to wait for mongodb/elasticsearch availability -ARG DOCKERIZE_VERSION=v0.6.1 -RUN wget -O /tmp/dockerize.tar.gz https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \ - && tar -C /usr/local/bin -xzvf /tmp/dockerize.tar.gz \ - && rm /tmp/dockerize.tar.gz - -# Create unprivileged "app" user -RUN useradd --home-dir /app --create-home --shell /bin/bash --uid 1000 app - -# Copy custom scripts -COPY ./bin /app/bin -RUN chmod a+x /app/bin/* -ENV PATH :${PATH} - -# From then on, run as unprivileged app user -USER app - -# Install rake and bundler -ENV PATH "/app/bin:/app/.gem/ruby/2.5.0/bin:$PATH" -RUN gem install --user-install bundler --version 1.17.3 -RUN gem install --user-install rake --version 13.0.1 - -# Install forum -RUN git clone https://github.com/edx/cs_comments_service.git --branch {{ OPENEDX_COMMON_VERSION }} --depth 1 /app/cs_comments_service -WORKDIR /app/cs_comments_service -RUN bundle install --deployment - -ENTRYPOINT ["docker-entrypoint.sh"] - -ENV SINATRA_ENV staging -ENV NEW_RELIC_ENABLE false -ENV API_KEY forumapikey -ENV SEARCH_SERVER "http://elasticsearch:9200" -ENV MONGODB_AUTH "" -ENV MONGOID_AUTH_MECH "" -ENV MONGODB_HOST "mongodb" -ENV MONGODB_PORT "27017" -ENV MONGODB_DATABASE "cs_comments_service" -EXPOSE 4567 -CMD ./bin/unicorn -c config/unicorn_tcp.rb -I '.' diff --git a/tutor/templates/build/forum/bin/docker-entrypoint.sh b/tutor/templates/build/forum/bin/docker-entrypoint.sh deleted file mode 100755 index 4f5ac7c..0000000 --- a/tutor/templates/build/forum/bin/docker-entrypoint.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -e - -export MONGOHQ_URL="mongodb://$MONGODB_AUTH$MONGODB_HOST:$MONGODB_PORT/$MONGODB_DATABASE" -# the search server variable was renamed after the upgrade to elasticsearch 7 -export SEARCH_SERVER_ES7="$SEARCH_SERVER" - -# make sure that there is an actual authentication mechanism in place, if necessary -if [ -n "$MONGODB_AUTH" ] -then - export MONGOID_AUTH_MECH=":scram" -fi - -echo "Waiting for mongodb/elasticsearch..." -dockerize -wait tcp://$MONGODB_HOST:$MONGODB_PORT -wait $SEARCH_SERVER -wait-retry-interval 5s -timeout 600s - -exec "$@" diff --git a/tutor/templates/config.yml b/tutor/templates/config.yml index 503fa1e..5d68f83 100644 --- a/tutor/templates/config.yml +++ b/tutor/templates/config.yml @@ -11,7 +11,6 @@ LMS_HOST: "www.myopenedx.com" # The following are default values RUN_LMS: true RUN_CMS: true -RUN_FORUM: true RUN_ELASTICSEARCH: true RUN_MONGODB: true RUN_MYSQL: true @@ -29,7 +28,6 @@ DOCKER_IMAGE_OPENEDX: "{{ DOCKER_REGISTRY }}overhangio/openedx:{{ TUTOR_VERSION DOCKER_IMAGE_OPENEDX_DEV: "openedx-dev" DOCKER_IMAGE_CADDY: "{{ DOCKER_REGISTRY }}caddy:2.3.0" DOCKER_IMAGE_ELASTICSEARCH: "{{ DOCKER_REGISTRY }}elasticsearch:7.10.1" -DOCKER_IMAGE_FORUM: "{{ DOCKER_REGISTRY }}overhangio/openedx-forum:{{ TUTOR_VERSION }}" DOCKER_IMAGE_MONGODB: "{{ DOCKER_REGISTRY }}mongo:4.2.17" DOCKER_IMAGE_MYSQL: "{{ DOCKER_REGISTRY }}mysql:5.7.35" DOCKER_IMAGE_ELASTICSEARCH: "{{ DOCKER_REGISTRY }}elasticsearch:7.10.1" @@ -44,8 +42,6 @@ ELASTICSEARCH_SCHEME: "http" ELASTICSEARCH_HEAP_SIZE: 1g ENABLE_HTTPS: false ENABLE_WEB_PROXY: true -FORUM_HOST: "forum" -FORUM_MONGODB_DATABASE: "cs_comments_service" JWT_COMMON_AUDIENCE: "openedx" JWT_COMMON_ISSUER: "{% if ENABLE_HTTPS %}https{% else %}http{% endif %}://{{ LMS_HOST }}/oauth2" JWT_COMMON_SECRET_KEY: "{{ OPENEDX_SECRET_KEY }}" diff --git a/tutor/templates/k8s/deployments.yml b/tutor/templates/k8s/deployments.yml index 77b58bc..3841d78 100644 --- a/tutor/templates/k8s/deployments.yml +++ b/tutor/templates/k8s/deployments.yml @@ -134,45 +134,6 @@ spec: configMap: name: openedx-config {% endif %} -{% if RUN_FORUM %} ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: forum - labels: - app.kubernetes.io/name: forum -spec: - selector: - matchLabels: - app.kubernetes.io/name: forum - template: - metadata: - labels: - app.kubernetes.io/name: forum - spec: - securityContext: - runAsUser: 1000 - runAsGroup: 1000 - containers: - - name: forum - image: {{ DOCKER_IMAGE_FORUM }} - ports: - - containerPort: 4567 - env: - - name: SEARCH_SERVER - value: "{{ ELASTICSEARCH_SCHEME }}://{{ ELASTICSEARCH_HOST }}:{{ ELASTICSEARCH_PORT }}" - - name: MONGODB_AUTH - value: "{% if MONGODB_USERNAME and MONGODB_PASSWORD %}{{ MONGODB_USERNAME}}:{{ MONGODB_PASSWORD }}@{% endif %}" - - name: MONGODB_HOST - value: "{{ MONGODB_HOST }}" - - name: MONGODB_PORT - value: "{{ MONGODB_PORT }}" - - name: MONGODB_DATABASE - value: "{{ FORUM_MONGODB_DATABASE }}" - securityContext: - allowPrivilegeEscalation: false -{% endif %} {% if RUN_LMS %} --- apiVersion: apps/v1 diff --git a/tutor/templates/k8s/jobs.yml b/tutor/templates/k8s/jobs.yml index 07e166e..d514d01 100644 --- a/tutor/templates/k8s/jobs.yml +++ b/tutor/templates/k8s/jobs.yml @@ -77,32 +77,5 @@ spec: containers: - name: mysql image: {{ DOCKER_IMAGE_MYSQL }} -{% if RUN_FORUM %} ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: forum-job - labels: - app.kubernetes.io/component: job -spec: - template: - spec: - restartPolicy: Never - containers: - - name: forum - image: {{ DOCKER_IMAGE_FORUM }} - env: - - name: SEARCH_SERVER - value: "{{ ELASTICSEARCH_SCHEME }}://{{ ELASTICSEARCH_HOST }}:{{ ELASTICSEARCH_PORT }}" - - name: MONGODB_AUTH - value: "{% if MONGODB_USERNAME and MONGODB_PASSWORD %}{{ MONGODB_USERNAME}}:{{ MONGODB_PASSWORD }}@{% endif %}" - - name: MONGODB_HOST - value: "{{ MONGODB_HOST }}" - - name: MONGODB_PORT - value: "{{ MONGODB_PORT }}" - - name: MONGODB_DATABASE - value: "{{ FORUM_MONGODB_DATABASE }}" -{% endif %} {{ patch("k8s-jobs") }} diff --git a/tutor/templates/k8s/services.yml b/tutor/templates/k8s/services.yml index 68e78ec..b768098 100644 --- a/tutor/templates/k8s/services.yml +++ b/tutor/templates/k8s/services.yml @@ -28,20 +28,6 @@ spec: selector: app.kubernetes.io/name: cms {% endif %} -{% if RUN_FORUM %} ---- -apiVersion: v1 -kind: Service -metadata: - name: forum -spec: - type: NodePort - ports: - - port: 4567 - protocol: TCP - selector: - app.kubernetes.io/name: forum -{% endif %} {% if RUN_LMS %} --- apiVersion: v1 diff --git a/tutor/templates/local/docker-compose.jobs.yml b/tutor/templates/local/docker-compose.jobs.yml index c9fc0c9..01fb809 100644 --- a/tutor/templates/local/docker-compose.jobs.yml +++ b/tutor/templates/local/docker-compose.jobs.yml @@ -27,14 +27,4 @@ services: - ../apps/openedx/config:/openedx/config:ro depends_on: {{ [("mysql", RUN_MYSQL)]|list_if }} - forum-job: - image: {{ DOCKER_IMAGE_FORUM }} - environment: - SEARCH_SERVER: "{{ ELASTICSEARCH_SCHEME }}://{{ ELASTICSEARCH_HOST }}:{{ ELASTICSEARCH_PORT }}" - MONGODB_AUTH: "{% if MONGODB_USERNAME and MONGODB_PASSWORD %}{{ MONGODB_USERNAME}}:{{ MONGODB_PASSWORD }}@{% endif %}" - MONGODB_HOST: "{{ MONGODB_HOST }}" - MONGODB_PORT: "{{ MONGODB_PORT }}" - MONGODB_DATABASE: "{{ FORUM_MONGODB_DATABASE }}" - depends_on: {{ [("elasticsearch", RUN_ELASTICSEARCH), ("mongodb", RUN_MONGODB)]|list_if }} - {{ patch("local-docker-compose-jobs-services")|indent(4) }} diff --git a/tutor/templates/local/docker-compose.yml b/tutor/templates/local/docker-compose.yml index 4d27c24..480af3e 100644 --- a/tutor/templates/local/docker-compose.yml +++ b/tutor/templates/local/docker-compose.yml @@ -97,21 +97,6 @@ services: HOSTNAME: "{{ LMS_HOST }}" {% endif %} - ############# Forum - - {% if RUN_FORUM %} - forum: - image: {{ DOCKER_IMAGE_FORUM }} - environment: - SEARCH_SERVER: "{{ ELASTICSEARCH_SCHEME }}://{{ ELASTICSEARCH_HOST }}:{{ ELASTICSEARCH_PORT }}" - MONGODB_AUTH: "{% if MONGODB_USERNAME and MONGODB_PASSWORD %}{{ MONGODB_USERNAME}}:{{ MONGODB_PASSWORD }}@{% endif %}" - MONGODB_HOST: "{{ MONGODB_HOST }}" - MONGODB_PORT: "{{ MONGODB_PORT }}" - MONGODB_DATABASE: "{{ FORUM_MONGODB_DATABASE }}" - restart: unless-stopped - depends_on: {{ [("elasticsearch", RUN_ELASTICSEARCH), ("mongodb", RUN_MONGODB)]|list_if }} - {% endif %} - ############# LMS and CMS {% if RUN_LMS %} @@ -132,7 +117,6 @@ services: - lms-permissions {% if RUN_MYSQL %}- mysql{% endif %} {% if RUN_ELASTICSEARCH %}- elasticsearch{% endif %} - {% if RUN_FORUM %}- forum{% endif %} {% if RUN_MONGODB %}- mongodb{% endif %} {% if RUN_REDIS %}- redis{% endif %} {% if RUN_SMTP %}- smtp{% endif %}