Pluggable lms/cms/forum

LMS, CMS and forum can now be independently (de)activated. This will be
useful to deploy Tutor on multiple servers.

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Latest
 
+- [Feature] Pluggable LMS/CMS/forum
 - [Improvement] Safer environment overwrite. Thanks @silviot! 👐
 - [Security] Fix Jinja2 vulnerability
 - [Improvement] Improve CLI cold start performance

docs/configuration.rst

@@ -25,13 +25,31 @@ Once the base configuration is created or updated, the environment is automatica
 
 With an up-to-date environment, Tutor is ready to launch an Open edX platform and perform usual operations. Below, we document some of the configuration parameters.
 
+Individual service activation
+-----------------------------
+
+- ``ACTIVATE_LMS`` (default: ``true``)
+- ``ACTIVATE_CMS`` (default: ``true``)
+- ``ACTIVATE_FORUM`` (default: ``true``)
+- ``ACTIVATE_ELASTICSEARCH`` (default: ``true``)
+- ``ACTIVATE_MEMCACHED`` (default: ``true``)
+- ``ACTIVATE_MONGODB`` (default: ``true``)
+- ``ACTIVATE_MYSQL`` (default: ``true``)
+- ``ACTIVATE_RABBITMQ`` (default: ``true``)
+- ``ACTIVATE_SMTP`` (default: ``true``)
+- ``ACTIVATE_HTTPS`` (default: ``false``)
+- ``ACTIVATE_NOTES`` (default: ``false``)
+- ``ACTIVATE_XQUEUE`` (default: ``false``)
+
+Every single Open edX service may be (de)activated at will by these configuration parameters. This is useful if you want, for instance, to distribute the various Open edX services on different servers.
+
 Docker
 ------
 
 .. _docker_images:
 
 Custom images
--------------
+~~~~~~~~~~~~~
 
 - ``DOCKER_IMAGE_OPENEDX`` (default: ``"regis/openedx:ironwood"``)
 - ``DOCKER_IMAGE_ANDROID`` (default: ``"regis/openedx-android:ironwood"``)
@@ -42,7 +60,7 @@ Custom images
 These configuration parameters define which image to run for each service.
 
 Custom registry
----------------
+~~~~~~~~~~~~~~~
 
 - ``DOCKER_REGISTRY`` (default: ``""``)
 

tutor/local.py

@@ -6,11 +6,12 @@ from time import sleep
 import click
 
 from . import config as tutor_config
+from . import env as tutor_env
+from . import exceptions
 from . import fmt
 from . import opts
 from . import scripts
 from . import utils
-from . import env as tutor_env
 
 
 @click.group(
@@ -85,12 +86,15 @@ restart all services.""",
 @opts.root
 @click.argument('service')
 def restart(root, service):
+    config = tutor_config.load(root)
     command = ["restart"]
     if service == "openedx":
-        command += ["lms", "cms", "lms_worker", "cms_worker"]
+        if config["ACTIVATE_LMS"]:
+            command += ["lms", "lms_worker"]
+        if config["ACTIVATE_CMS"]:
+            command += ["cms", "cms_worker"]
     elif service != "all":
         command += [service]
-    config = tutor_config.load(root)
     docker_compose(root, config, *command)
 
 @click.command(
@@ -229,11 +233,15 @@ def logs(root, follow, tail, service):
 @click.argument("name")
 @click.argument("email")
 def createuser(root, superuser, staff, name, email):
+    config = tutor_config.load(root)
+    check_service_is_activated(config, "lms")
     scripts.create_user(root, run_bash, superuser, staff, name, email)
 
 @click.command(help="Import the demo course")
 @opts.root
 def importdemocourse(root):
+    config = tutor_config.load(root)
+    check_service_is_activated(config, "cms")
     click.echo(fmt.info("Importing demo course"))
     scripts.import_demo_course(root, run_bash)
     click.echo(fmt.info("Re-indexing courses"))
@@ -242,6 +250,8 @@ def importdemocourse(root):
 @click.command(help="Re-index courses for better searching")
 @opts.root
 def indexcourses(root):
+    config = tutor_config.load(root)
+    check_service_is_activated(config, "cms")
     scripts.index_courses(root, run_bash)
 
 @click.command(
@@ -261,6 +271,10 @@ def portainer(root, port):
     click.echo(fmt.info("View the Portainer UI at http://localhost:{port}".format(port=port)))
     utils.docker_run(*docker_run)
 
+def check_service_is_activated(config, service):
+    if not config["ACTIVATE_" + service.upper()]:
+        raise exceptions.TutorError("This command may only be executed on the server where the {} is running".format(service))
+
 def run_bash(root, service, command):
     config = tutor_config.load(root)
     docker_compose(root, config, "run", "--rm", service, "bash", "-e", "-c", command)

tutor/scripts.py

@@ -6,22 +6,28 @@ from . import fmt
 
 def migrate(root, run_func):
     config = tutor_config.load(root)
-    click.echo(fmt.info("Creating lms/cms databases..."))
-    run_template(root, config, "lms", "create_databases.sh", run_func)
-    click.echo(fmt.info("Running lms migrations..."))
-    run_template(root, config, "lms", "migrate_lms.sh", run_func)
-    click.echo(fmt.info("Running cms migrations..."))
-    run_template(root, config, "cms", "migrate_cms.sh", run_func)
-    click.echo(fmt.info("Running forum migrations..."))
-    run_template(root, config, "forum", "migrate_forum.sh", run_func)
+
+    click.echo(fmt.info("Creating all databases..."))
+    run_template(root, config, "openedx-assets", "create_databases.sh", run_func)
+
+    if config["ACTIVATE_LMS"]:
+        click.echo(fmt.info("Running lms migrations..."))
+        run_template(root, config, "lms", "migrate_lms.sh", run_func)
+    if config["ACTIVATE_CMS"]:
+        click.echo(fmt.info("Running cms migrations..."))
+        run_template(root, config, "cms", "migrate_cms.sh", run_func)
+    if config["ACTIVATE_FORUM"]:
+        click.echo(fmt.info("Running forum migrations..."))
+        run_template(root, config, "forum", "migrate_forum.sh", run_func)
     if config["ACTIVATE_NOTES"]:
         click.echo(fmt.info("Running notes migrations..."))
         run_template(root, config, "notes", "migrate_django.sh", run_func)
     if config["ACTIVATE_XQUEUE"]:
         click.echo(fmt.info("Running xqueue migrations..."))
         run_template(root, config, "xqueue", "migrate_django.sh", run_func)
-    click.echo(fmt.info("Creating oauth2 users..."))
-    run_template(root, config, "lms", "oauth2.sh", run_func)
+    if config["ACTIVATE_LMS"]:
+        click.echo(fmt.info("Creating oauth2 users..."))
+        run_template(root, config, "lms", "oauth2.sh", run_func)
     click.echo(fmt.info("Databases ready."))
 
 def create_user(root, run_func, superuser, staff, name, email):

tutor/templates/apps/nginx/cms.conf

@@ -1,3 +1,4 @@
+{% if ACTIVATE_CMS %}
 upstream cms-backend {
     server cms:8000 fail_timeout=0;
 }
@@ -72,3 +73,4 @@ server {
     expires epoch;
   }
 }
+{% endif %}

tutor/templates/apps/nginx/lms.conf

@@ -1,3 +1,4 @@
+{% if ACTIVATE_LMS %}
 upstream lms-backend {
     server lms:8000 fail_timeout=0;
 }
@@ -88,3 +89,4 @@ server {
     expires epoch;
   }
 }
+{% endif %}

tutor/templates/apps/openedx/config/lms.env.json

@@ -25,7 +25,7 @@
   "CELERY_BROKER_HOSTNAME": "{{ RABBITMQ_HOST }}",
   "CELERY_BROKER_USER": "{{ RABBITMQ_USERNAME }}",
   "CELERY_BROKER_PASSWORD": "{{ RABBITMQ_PASSWORD }}",
-  "COMMENTS_SERVICE_URL": "http://forum:4567",
+  "COMMENTS_SERVICE_URL": "http://{{ FORUM_HOST }}:4567",
   "COMMENTS_SERVICE_KEY": "forumapikey",
   "COMPREHENSIVE_THEME_DIRS": ["/openedx/themes"],
   "MEDIA_ROOT": "/openedx/data/uploads/",

tutor/templates/config-base.yml

@@ -14,7 +14,4 @@ XQUEUE_AUTH_PASSWORD: "{{ RAND8 }}"
 XQUEUE_MYSQL_PASSWORD: "{{ RAND8 }}"
 XQUEUE_SECRET_KEY: "{{ RAND24 }}"
 ANDROID_OAUTH2_SECRET: "{{ RAND24 }}"
-ACTIVATE_HTTPS: false
-ACTIVATE_NOTES: false
-ACTIVATE_XQUEUE: false
 ID: "{{ RAND8 }}"

tutor/templates/config-defaults.yml

@@ -1,10 +1,16 @@
 ---
+ACTIVATE_LMS: true
+ACTIVATE_CMS: true
+ACTIVATE_FORUM: true
 ACTIVATE_ELASTICSEARCH: true
 ACTIVATE_MEMCACHED: true
 ACTIVATE_MONGODB: true
 ACTIVATE_MYSQL: true
 ACTIVATE_RABBITMQ: true
 ACTIVATE_SMTP: true
+ACTIVATE_HTTPS: false
+ACTIVATE_NOTES: false
+ACTIVATE_XQUEUE: false
 ANDROID_RELEASE_STORE_PASSWORD: "android store password"
 ANDROID_RELEASE_KEY_PASSWORD: "android release key password"
 ANDROID_RELEASE_KEY_ALIAS: "android release key alias"
@@ -24,6 +30,7 @@ DOCKER_REGISTRY: ""
 LOCAL_PROJECT_NAME: "tutor_local"
 ELASTICSEARCH_HOST: "elasticsearch"
 ELASTICSEARCH_PORT: 9200
+FORUM_HOST: "forum"
 MEMCACHED_HOST: "memcached"
 MEMCACHED_PORT: 11211
 MONGODB_HOST: "mongodb"

tutor/templates/local/docker-compose.yml

@@ -60,12 +60,12 @@ services:
     volumes:
       - ../apps/nginx:/etc/nginx/conf.d/
       - ../../data/openedx:/var/www/openedx:ro
-      - ../../data/cms:/openedx/data/cms/:ro
-      - ../../data/lms:/openedx/data/lms/:ro
-      - ../../data/letsencrypt:/etc/letsencrypt/:ro
+      {% if ACTIVATE_CMS %}- ../../data/cms:/openedx/data/cms/:ro{% endif %}
+      {% if ACTIVATE_LMS %}- ../../data/lms:/openedx/data/lms/:ro{% endif %}
+      {% if ACTIVATE_HTTPS %}- ../../data/letsencrypt:/etc/letsencrypt/:ro{% endif %}
     depends_on:
-      - lms
-      - cms
+      {% if ACTIVATE_LMS %}- lms{% endif %}
+      {% if ACTIVATE_CMS %}- cms {% endif %}
       {% if ACTIVATE_NOTES %}- notes{% endif %}
 
   {% if ACTIVATE_RABBITMQ %}
@@ -84,6 +84,7 @@ services:
 
   ############# Forum
 
+  {% if ACTIVATE_FORUM %}
   forum:
     image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_FORUM }}
     environment:
@@ -93,9 +94,11 @@ services:
     depends_on:
       {% if ACTIVATE_ELASTICSEARCH %}- elasticsearch{% endif %}
       {% if ACTIVATE_MONGODB %}- mongodb{% endif %}
+  {% endif %}
 
   ############# LMS and CMS
 
+  {% if ACTIVATE_LMS %}
   lms:
     image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_OPENEDX }}
     environment:
@@ -109,13 +112,15 @@ services:
       - ../../data/lms:/openedx/data
     depends_on:
       {% if ACTIVATE_ELASTICSEARCH %}- elasticsearch{% endif %}
-      - forum
+      {% if ACTIVATE_FORUM %}- forum{% endif %}
       {% if ACTIVATE_MEMCACHED %}- memcached{% endif %}
       {% if ACTIVATE_MONGODB %}- mongodb{% endif %}
       {% if ACTIVATE_MYSQL %}- mysql{% endif %}
       {% if ACTIVATE_RABBITMQ %}- rabbitmq{% endif %}
       {% if ACTIVATE_SMTP %}- smtp{% endif %}
+  {% endif %}
 
+  {% if ACTIVATE_CMS %}
   cms:
     image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_OPENEDX }}
     environment:
@@ -134,10 +139,11 @@ services:
       {% if ACTIVATE_MYSQL %}- mysql{% endif %}
       {% if ACTIVATE_RABBITMQ %}- rabbitmq{% endif %}
       {% if ACTIVATE_SMTP %}- smtp{% endif %}
+  {% endif %}
 
   ############# LMS and CMS workers
 
-  # We could probably create one service per queue here. For small instances, it is not necessary.
+  {% if ACTIVATE_LMS %}
   lms_worker:
     image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_OPENEDX }}
     environment:
@@ -152,8 +158,10 @@ services:
       - ../apps/openedx/config/:/openedx/config/
       - ../../data/lms:/openedx/data
     depends_on:
-      - lms
+      {% if ACTIVATE_LMS %}- lms {% endif %}
+  {% endif %}
 
+  {% if ACTIVATE_CMS %}
   cms_worker:
     image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_OPENEDX }}
     environment:
@@ -168,7 +176,8 @@ services:
       - ../apps/openedx/config/:/openedx/config/
       - ../../data/cms:/openedx/data
     depends_on:
-      - cms
+      {% if ACTIVATE_CMS %}- cms {% endif %}
+  {% endif %}
 
   {% if ACTIVATE_NOTES %}
   ############# Notes: backend store for edX Student Notes