🔌 Introduce a plugin system for tutor

This adds the basic feratures that we need for a working plugin system,
but there are still many TODOs in the codebase.

.gitignore

@@ -2,9 +2,9 @@
 !.gitignore
 TODO
 __pycache__
+*.egg-info/
 
 /tutor.spec
 /build/tutor
 /dist/
 /releases/
-/tutor_openedx.egg-info/

Makefile

@@ -1,5 +1,5 @@
 .DEFAULT_GOAL := help
-BLACK_OPTS = --exclude templates ./tutor ./tests
+BLACK_OPTS = --exclude templates ./tutor ./tests ./plugins
 
 ###### Development
 
@@ -16,15 +16,21 @@ test-format: ## Run code formatting tests
 test-lint: ## Run code linting tests
 	pylint --errors-only tutor tests plugins
 
-test-unit: ## Run unit tests
+test-unit: test-unit-core test-unit-plugins ## Run unit tests
+
+test-unit-core:
 	python3 -m unittest discover tests
 
+test-unit-plugins:
+	python3 -m unittest discover plugins/minio/tests
+
 format: ## Format code automatically
 	black $(BLACK_OPTS)
 
 ###### Deployment
 
 bundle: ## Bundle the tutor package in a single "dist/tutor" executable
+	# TODO bundle plugins
 	pyinstaller --onefile --name=tutor --add-data=./tutor/templates:./tutor/templates ./bin/main
 dist/tutor:
 	$(MAKE) bundle
@@ -52,6 +58,7 @@ ci-info: ## Print info about environment
 ci-install: ## Install requirements
 	pip3 install -U setuptools
 	pip3 install -r requirements/dev.txt
+	pip3 install -r requirements/plugins.txt
 
 ci-bundle: ## Create bundle and run basic tests
 	$(MAKE) bundle

plugins/minio/README.rst

@@ -0,0 +1,11 @@
+TODO
+
+- This is mainly for production. Does not work with `tutor dev` commands.
+- For local testing, you need to set MINIO_HOST to minio.localhost:
+
+    tutor config save -y --set MINIO_HOST=minio.localhost
+  
+- You need `minio.LMS_HOST` domain name. For local development, the MinIO admin dashboard is at minio.localhost. For authentication, use MINIO_ACCESS_KEY and MINIO_SECRET_KEY:
+
+    tutor config printvalue OPENEDX_AWS_ACCESS_KEY
+    tutor config printvalue OPENEDX_AWS_SECRET_ACCESS_KEY

plugins/minio/setup.py

@@ -0,0 +1,41 @@
+import io
+import os
+from setuptools import setup
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+with io.open(os.path.join(here, "README.rst"), "rt", encoding="utf8") as f:
+    readme = f.read()
+
+
+setup(
+    name="tutor-minio",
+    version="0.0.1",
+    url="https://docs.tutor.overhang.io/",
+    project_urls={
+        "Documentation": "https://docs.tutor.overhang.io/",
+        "Code": "https://github.com/regisb/tutor/tree/master/plugins/minio",
+        "Issue tracker": "https://github.com/regisb/tutor/issues",
+        "Community": "https://discuss.overhang.io",
+    },
+    license="AGPLv3",
+    author="Régis Behmo",
+    author_email="regis@behmo.com",
+    description="A Tutor plugin for object storage in MinIO",
+    long_description=readme,
+    packages=["tutorminio"],
+    include_package_data=True,
+    python_requires=">=3.5",
+    install_requires=["click>=7.0"],
+    entry_points={"tutor.plugin.v0": ["minio = tutorminio.plugin"]},
+    classifiers=[
+        "Development Status :: 3 - Alpha",
+        "Intended Audience :: Developers",
+        "License :: OSI Approved :: GNU Affero General Public License v3",
+        "Operating System :: OS Independent",
+        "Programming Language :: Python",
+        "Programming Language :: Python :: 3.5",
+        "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.7",
+    ],
+)

plugins/minio/tests/test_plugin.py

@@ -0,0 +1,12 @@
+import unittest
+
+from tutorminio import plugin
+
+
+class PluginTests(unittest.TestCase):
+    def test_patches(self):
+        patches = dict(plugin.patches())
+        self.assertIn("local-docker-compose-services", patches)
+        self.assertTrue(
+            patches["local-docker-compose-services"].startswith("# MinIO\n")
+        )

plugins/minio/tutorminio/patches/https-create.patch

@@ -0,0 +1,2 @@
+# MinIO
+certbot certonly --standalone -n --agree-tos -m admin@{{ LMS_HOST }} -d {{ MINIO_HOST }}

plugins/minio/tutorminio/patches/lms-env.patch

@@ -0,0 +1,3 @@
+"FILE_UPLOAD_STORAGE_BUCKET_NAME": "{{ MINIO_FILE_UPLOAD_BUCKET_NAME }}",
+"COURSE_IMPORT_EXPORT_BUCKET": "{{ MINIO_COURSE_IMPORT_EXPORT_BUCKET }}",
+"AWS_S3_CUSTOM_DOMAIN": "{{ MINIO_HOST }}"

plugins/minio/tutorminio/patches/local-docker-compose-services.patch

@@ -0,0 +1,20 @@
+# MinIO
+minio:
+  image: {{ MINIO_DOCKER_REGISTRY }}{{ MINIO_DOCKER_IMAGE_SERVER }}
+  volumes:
+    - ../../data/minio:/data
+  ports:
+    - "9000:9000"
+  environment:
+    MINIO_ACCESS_KEY: "{{ OPENEDX_AWS_ACCESS_KEY }}"
+    MINIO_SECRET_KEY: "{{ OPENEDX_AWS_SECRET_ACCESS_KEY }}"
+  command: server /data
+  restart: unless-stopped
+
+minio-client:
+  image: {{ MINIO_DOCKER_REGISTRY }}{{ MINIO_DOCKER_IMAGE_CLIENT }}
+  entrypoint: sh
+  command: "mc ls"
+  restart: "no"
+  depends_on:
+    - minio

plugins/minio/tutorminio/patches/nginx-extra.patch

@@ -0,0 +1,26 @@
+# MinIO public service
+upstream minio-backend {
+    server minio:9000 fail_timeout=0;
+}
+server {
+  {% if ACTIVATE_HTTPS %}listen 443 {{ "" if WEB_PROXY else "ssl" }};{% else %}listen 80;{% endif %}
+  server_name minio.localhost {{ MINIO_HOST }};
+
+  {% if ACTIVATE_HTTPS and not WEB_PROXY %}
+  ssl_certificate /etc/letsencrypt/live/{{ MINIO_HOST }}/fullchain.pem;
+  ssl_certificate_key /etc/letsencrypt/live/{{ MINIO_HOST }}/privkey.pem;
+  {% endif %}
+
+  # Disables server version feedback on pages and in headers
+  server_tokens off;
+
+  location / {
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_set_header X-Forwarded-Port $server_port;
+    proxy_set_header X-Forwarded-For $remote_addr;
+    proxy_set_header Host $http_host;
+    proxy_redirect off;
+
+    proxy_pass http://minio-backend;
+  }
+}

plugins/minio/tutorminio/patches/openedx-auth.patch

@@ -0,0 +1,2 @@
+"AWS_STORAGE_BUCKET_NAME": "{{ MINIO_BUCKET_NAME }}",
+"AWS_S3_CUSTOM_DOMAIN": "{{ MINIO_HOST }}"

plugins/minio/tutorminio/patches/openedx-common-settings.patch

@@ -0,0 +1 @@
+AWS_S3_USE_SSL = {{ "True" if ACTIVATE_HTTPS else "False" }}

plugins/minio/tutorminio/plugin.py

@@ -0,0 +1,40 @@
+import os
+from glob import glob
+
+HERE = os.path.abspath(os.path.dirname(__file__))
+
+config = {
+    "set": {
+        "OPENEDX_AWS_ACCESS_KEY": "openedx",
+        "OPENEDX_AWS_SECRET_ACCESS_KEY": "{{ 24|random_string }}",
+    },
+    "defaults": {
+        "BUCKET_NAME": "openedx",
+        "FILE_UPLOAD_BUCKET_NAME": "openedxuploads",
+        "COURSE_IMPORT_EXPORT_BUCKET": "openedxcourseimportexport",
+        "HOST": "minio.{{ LMS_HOST }}",
+        "DOCKER_REGISTRY": "{{ DOCKER_REGISTRY }}",
+        "DOCKER_IMAGE_CLIENT": "minio/mc:RELEASE.2019-05-23T01-33-27Z",
+        "DOCKER_IMAGE_SERVER": "minio/minio:RELEASE.2019-05-23T00-29-34Z",
+    },
+}
+
+scripts = {
+    "init": [
+        {
+            "service": "mysql-client",
+            "command": """mc config host add minio http://minio:9000 {{ OPENEDX_AWS_ACCESS_KEY }} {{ OPENEDX_AWS_SECRET_ACCESS_KEY }} --api s3v4
+            mc mb minio {{ FILE_UPLOAD_BUCKET_NAME }} {{ COURSE_IMPORT_EXPORT_BUCKET }}""",
+        }
+    ]
+}
+
+
+def patches(*_args):
+    all_patches = {}
+    for path in glob(os.path.join(HERE, "patches", "*.patch")):
+        with open(path) as patch_file:
+            name = os.path.basename(path)[:-6]
+            content = patch_file.read()
+            all_patches[name] = content
+    return all_patches

requirements/plugins.txt

@@ -0,0 +1 @@
+-e ./plugins/minio

tests/test_config.py

@@ -16,6 +16,12 @@ class ConfigTests(unittest.TestCase):
         self.assertNotIn("TUTOR_VERSION", defaults)
 
     def test_merge(self):
+        config1 = {"x": "y"}
+        config2 = {"x": "z"}
+        tutor_config.merge(config1, config2)
+        self.assertEqual({"x": "y"}, config1)
+
+    def test_merge_render(self):
         config = {}
         defaults = tutor_config.load_defaults()
         with unittest.mock.patch.object(

tests/test_env.py

@@ -39,7 +39,7 @@ class EnvTests(unittest.TestCase):
         config = {}
         tutor_config.merge(config, tutor_config.load_defaults())
         config["MYSQL_ROOT_PASSWORD"] = "testpassword"
-        rendered = env.render_file(config, "scripts", "create_databases.sh")
+        rendered = env.render_file(config, "scripts", "mysql-client", "createdatabases")
         self.assertIn("testpassword", rendered)
 
     @unittest.mock.patch.object(tutor_config.fmt, "echo")
@@ -58,3 +58,22 @@ class EnvTests(unittest.TestCase):
         defaults["ACTIVATE_HTTPS"] = True
         with tempfile.TemporaryDirectory() as root:
             env.render_full(root, defaults)
+
+    def test_patch(self):
+        patches = {"plugin1": "abcd", "plugin2": "efgh"}
+        with unittest.mock.patch.object(
+            env.plugins, "iter_patches", return_value=patches.items()
+        ) as mock_iter_patches:
+            rendered = env.render_str({}, '{{ patch("location") }}')
+            mock_iter_patches.assert_called_once_with({}, "location")
+        self.assertEqual("abcd\nefgh", rendered)
+
+    def test_patch_separator_suffix(self):
+        patches = {"plugin1": "abcd", "plugin2": "efgh"}
+        with unittest.mock.patch.object(
+            env.plugins, "iter_patches", return_value=patches.items()
+        ) as mock_iter_patches:
+            rendered = env.render_str(
+                {}, '{{ patch("location", separator=",\n", suffix=",") }}'
+            )
+        self.assertEqual("abcd,\nefgh,", rendered)

tests/test_plugins.py

@@ -0,0 +1,90 @@
+import unittest
+import unittest.mock
+
+from tutor import exceptions
+from tutor import plugins
+
+
+class PluginsTests(unittest.TestCase):
+    def setUp(self):
+        plugins.Patches.CACHE.clear()
+
+    def test_iter_installed(self):
+        with unittest.mock.patch.object(
+            plugins.pkg_resources, "iter_entry_points", return_value=[]
+        ):
+            self.assertEqual([], list(plugins.iter_installed()))
+
+    def test_is_installed(self):
+        self.assertFalse(plugins.is_installed("dummy"))
+
+    def test_enable(self):
+        config = {plugins.CONFIG_KEY: []}
+        with unittest.mock.patch.object(plugins, "is_installed", return_value=True):
+            plugins.enable(config, "plugin2")
+            plugins.enable(config, "plugin1")
+        self.assertEqual(["plugin1", "plugin2"], config[plugins.CONFIG_KEY])
+
+    def test_enable_twice(self):
+        config = {plugins.CONFIG_KEY: []}
+        with unittest.mock.patch.object(plugins, "is_installed", return_value=True):
+            plugins.enable(config, "plugin1")
+            plugins.enable(config, "plugin1")
+        self.assertEqual(["plugin1"], config[plugins.CONFIG_KEY])
+
+    def test_enable_not_installed_plugin(self):
+        config = {"PLUGINS": []}
+        with unittest.mock.patch.object(plugins, "is_installed", return_value=False):
+            self.assertRaises(exceptions.TutorError, plugins.enable, config, "plugin1")
+
+    def test_patches(self):
+        class plugin1:
+            patches = {"patch1": "Hello {{ ID }}"}
+
+        with unittest.mock.patch.object(
+            plugins, "iter_enabled", return_value=[("plugin1", plugin1)]
+        ):
+            patches = list(plugins.iter_patches({}, "patch1"))
+        self.assertEqual([("plugin1", "Hello {{ ID }}")], patches)
+
+    def test_plugin_without_patches(self):
+        class plugin1:
+            pass
+
+        with unittest.mock.patch.object(
+            plugins, "iter_enabled", return_value=[("plugin1", plugin1)]
+        ):
+            patches = list(plugins.iter_patches({}, "patch1"))
+        self.assertEqual([], patches)
+
+    def test_configure(self):
+        config = {"ID": "oldid"}
+
+        class plugin1:
+            config = {
+                "add": {"PARAM1": "value1", "PARAM2": "value2"},
+                "set": {"ID": "newid"},
+                "defaults": {"PARAM3": "value3"},
+            }
+
+        with unittest.mock.patch.object(
+            plugins, "iter_enabled", return_value=[("plugin1", plugin1)]
+        ):
+            add_config, set_config, defaults_config = plugins.load_config(config)
+        self.assertEqual(
+            {"PLUGIN1_PARAM1": "value1", "PLUGIN1_PARAM2": "value2"}, add_config
+        )
+        self.assertEqual({"ID": "newid"}, set_config)
+        self.assertEqual({"PLUGIN1_PARAM3": "value3"}, defaults_config)
+
+    def test_scripts(self):
+        class plugin1:
+            scripts = {"init": [{"service": "myclient", "command": "init command"}]}
+
+        with unittest.mock.patch.object(
+            plugins, "iter_enabled", return_value=[("plugin1", plugin1)]
+        ):
+            self.assertEqual(
+                [("plugin1", "myclient", "init command")],
+                list(plugins.iter_scripts({}, "init")),
+            )

tests/test_scripts.py

@@ -13,6 +13,6 @@ class ScriptsTests(unittest.TestCase):
     def test_run(self):
         config = tutor_config.load_defaults()
         runner = DummyRunner("/tmp", config)
-        rendered_script = runner.render("create_databases.sh")
-        runner.run("someservice", "create_databases.sh")
-        runner.exec.assert_called_once_with("someservice", rendered_script)
+        rendered_script = runner.render("mysql-client", "createdatabases")
+        runner.run("mysql-client", "createdatabases")
+        runner.exec.assert_called_once_with("mysql-client", rendered_script)

tutor/commands/cli.py

@@ -10,6 +10,7 @@ from .dev import dev
 from .images import images_command
 from .k8s import k8s
 from .local import local
+from .plugins import plugins_command
 from .ui import ui
 from .webui import webui
 from ..__about__ import __version__
@@ -47,6 +48,7 @@ cli.add_command(k8s)
 cli.add_command(ui)
 cli.add_command(webui)
 cli.add_command(print_help)
+cli.add_command(plugins_command)
 
 if __name__ == "__main__":
     main()

tutor/commands/config.py

@@ -8,6 +8,7 @@ from .. import exceptions
 from .. import env
 from .. import fmt
 from .. import opts
+from .. import plugins
 from .. import serialize
 from .. import utils
 from ..__about__ import __version__
@@ -142,10 +143,14 @@ def load_defaults():
 
 
 def load_current(root, defaults):
+    """
+    Note: this modifies the defaults. TODO this is not that great.
+    """
     convert_json2yml(root)
     config = load_user(root)
     load_env(config, defaults)
     load_required(config, defaults)
+    load_plugins(config, defaults)
     return config
 
 
@@ -192,6 +197,15 @@ def load_required(config, defaults):
             config[key] = env.render_str(config, defaults[key])
 
 
+def load_plugins(config, defaults):
+    add_config, set_config, defaults_config = plugins.load_config(config)
+    merge(config, add_config)
+    # TODO this might have unexpected consequences if plugins have conflicting configurations. Maybe we should print warning messages.
+    merge(config, set_config)
+    # TODO this modifies defaults, which is ugly.
+    merge(defaults, defaults_config)
+
+
 def upgrade_obsolete(config):
     # Openedx-specific mysql passwords
     if "MYSQL_PASSWORD" in config:

tutor/commands/dev.py

@@ -41,6 +41,11 @@ def run(root, edx_platform_path, edx_platform_settings, service, command, args):
 @click.argument("service", type=click.Choice(["lms", "cms"]))
 def runserver(root, edx_platform_path, edx_platform_settings, service):
     port = service_port(service)
+    from .. import fmt
+
+    fmt.echo_info(
+        "The {} service will be available at http://localhost:{}".format(service, port)
+    )
     docker_compose_run_with_port(
         root,
         edx_platform_path,

tutor/commands/local.py

@@ -169,7 +169,7 @@ def https_create(root):
         fmt.echo_info("HTTPS is not activated: certificate generation skipped")
         return
 
-    script = runner.render("https_create.sh")
+    script = runner.render("certbot", "create")
 
     if config["WEB_PROXY"]:
         fmt.echo_info(

tutor/commands/plugins.py

@@ -0,0 +1,44 @@
+import click
+
+from . import config as tutor_config
+from .. import opts
+from .. import plugins
+
+
+@click.group(
+    name="plugins",
+    short_help="Manage Tutor plugins",
+    help="Manage Tutor plugins to add new features and customize your Open edX platform",
+)
+def plugins_command():
+    pass
+
+
+@click.command(name="list", help="List installed plugins")
+@opts.root
+def list_command(root):
+    config = tutor_config.load(root)
+    for name, _ in plugins.iter_installed():
+        status = "" if plugins.is_enabled(config, name) else " (disabled)"
+        print("{plugin}{status}".format(plugin=name, status=status))
+
+
+@click.command(help="Enable a plugin")
+@click.argument("plugin")
+@opts.root
+def enable(plugin, root):
+    config = tutor_config.load(root)
+    plugins.enable(config, plugin)
+    tutor_config.save_config(root, config)
+    tutor_config.save(root, silent=True)
+
+
+@click.command(help="Disable a plugin")
+@opts.root
+def disable(root):
+    pass
+
+
+plugins_command.add_command(list_command)
+plugins_command.add_command(enable)
+plugins_command.add_command(disable)

tutor/env.py

@@ -6,6 +6,7 @@ import jinja2
 
 from . import exceptions
 from . import fmt
+from . import plugins
 from . import utils
 from .__about__ import __version__
 
@@ -39,13 +40,17 @@ class Renderer:
     @classmethod
     def render_str(cls, config, text):
         template = cls.environment().from_string(text)
-        return cls.__render(template, config)
+        return cls.__render(config, template)
 
     @classmethod
     def render_file(cls, config, path):
-        template = cls.environment().get_template(path)
         try:
-            return cls.__render(template, config)
+            template = cls.environment().get_template(path)
+        except:
+            fmt.echo_error("Error loading template " + path)
+            raise
+        try:
+            return cls.__render(config, template)
         except (jinja2.exceptions.TemplateError, exceptions.TutorError):
             fmt.echo_error("Error rendering template " + path)
             raise
@@ -54,14 +59,35 @@ class Renderer:
             raise
 
     @classmethod
-    def __render(cls, template, config):
+    def __render(cls, config, template):
+        def patch(name, separator="\n", suffix=""):
+            return cls.__render_patch(config, name, separator=separator, suffix=suffix)
+
         try:
-            return template.render(**config)
+            return template.render(patch=patch, **config)
         except jinja2.exceptions.UndefinedError as e:
             raise exceptions.TutorError(
                 "Missing configuration value: {}".format(e.args[0])
             )
 
+    @classmethod
+    def __render_patch(cls, config, name, separator="\n", suffix=""):
+        patches = []
+        for plugin, patch in plugins.iter_patches(config, name):
+            patch_template = cls.environment().from_string(patch)
+            try:
+                patches.append(patch_template.render(**config))
+            except jinja2.exceptions.UndefinedError as e:
+                raise exceptions.TutorError(
+                    "Missing configuration value: {} in patch '{}' from plugin {}".format(
+                        e.args[0], name, plugin
+                    )
+                )
+        rendered = separator.join(patches)
+        if rendered:
+            rendered += suffix
+        return rendered
+
 
 def render_full(root, config):
     """
@@ -194,6 +220,7 @@ def is_part_of_env(path):
         basename.startswith(".")
         or basename.endswith(".pyc")
         or basename == "__pycache__"
+        or basename == "partials"
     )
 
 

tutor/plugins.py

@@ -0,0 +1,155 @@
+import pkg_resources
+
+from . import exceptions
+
+"""
+Tutor plugins are regular python packages that have a 'tutor.plugin.v1' entrypoint. This entrypoint must point to a module or a class that implements I don't know what (yet).
+TODO
+"""
+
+# TODO switch to v1
+ENTRYPOINT = "tutor.plugin.v0"
+CONFIG_KEY = "PLUGINS"
+
+
+class Patches:
+    """
+    Provide a patch cache on which we can conveniently iterate without having to parse again all plugin patches for every environment file.
+
+    The CACHE static attribute is a dict of the form:
+
+        {
+            "patchname": {
+                "pluginname": "patch content",
+                ...
+            },
+            ...
+        }
+    """
+
+    CACHE = {}
+
+    def __init__(self, config, name):
+        self.name = name
+        if not self.CACHE:
+            self.fill_cache(config)
+
+    def __iter__(self):
+        """
+        Yields:
+            plugin name (str)
+            patch content (str)
+        """
+        plugin_patches = self.CACHE.get(self.name, {})
+        plugins = sorted(plugin_patches.keys())
+        for plugin in plugins:
+            yield plugin, plugin_patches[plugin]
+
+    @classmethod
+    def fill_cache(cls, config):
+        for plugin_name, plugin in iter_enabled(config):
+            patches = get_callable_attr(plugin, "patches")
+            for patch_name, content in patches.items():
+                if patch_name not in cls.CACHE:
+                    cls.CACHE[patch_name] = {}
+                cls.CACHE[patch_name][plugin_name] = content
+
+
+def get_callable_attr(plugin, attr_name):
+    attr = getattr(plugin, attr_name, {})
+    if callable(attr):
+        # TODO pass config here for initialization
+        attr = attr()
+    return attr
+
+
+def is_installed(name):
+    plugin_names = [name for name, _ in iter_installed()]
+    return name in plugin_names
+
+
+def iter_installed():
+    for entrypoint in pkg_resources.iter_entry_points(ENTRYPOINT):
+        yield entrypoint.name, entrypoint.load()
+
+
+def enable(config, name):
+    if not is_installed(name):
+        raise exceptions.TutorError("plugin '{}' is not installed.".format(name))
+    if is_enabled(config, name):
+        return
+    if CONFIG_KEY not in config:
+        config[CONFIG_KEY] = []
+    config[CONFIG_KEY].append(name)
+    config[CONFIG_KEY].sort()
+
+
+def iter_enabled(config):
+    for name, plugin in iter_installed():
+        if is_enabled(config, name):
+            yield name, plugin
+
+
+def is_enabled(config, name):
+    return name in config.get(CONFIG_KEY, [])
+
+
+def iter_patches(config, name):
+    for plugin, patch in Patches(config, name):
+        yield plugin, patch
+
+
+def load_config(config):
+    """
+    TODO Return:
+    
+    add_config (dict): config key/values to add, if they are not already present
+    set_config (dict): config key/values to set and override existing values
+    """
+    add_config = {}
+    set_config = {}
+    defaults_config = {}
+    for plugin_name, plugin in iter_enabled(config):
+        plugin_prefix = plugin_name.upper() + "_"
+        plugin_config = get_callable_attr(plugin, "config")
+
+        # Add new config key/values
+        for key, value in plugin_config.get("add", {}).items():
+            add_config[plugin_prefix + key] = value
+
+        # Set existing config key/values
+        for key, value in plugin_config.get("set", {}).items():
+            # TODO crash in case of conflicting keys between plugins
+            # If key already exists
+            set_config[key] = value
+
+        # Create new defaults
+        for key, value in plugin_config.get("defaults", {}).items():
+            defaults_config[plugin_prefix + key] = value
+    return add_config, set_config, defaults_config
+
+
+def iter_scripts(config, script_name):
+    """
+    Scripts are of the form:
+
+    scripts = {
+        "script-name": [
+            {
+                "service": "service-name",
+                "command": "...",
+            },
+            ...
+        ],
+        ...
+    }
+    """
+    for plugin_name, plugin in iter_enabled(config):
+        scripts = get_callable_attr(plugin, "scripts")
+        for script in scripts.get(script_name, []):
+            try:
+                yield plugin_name, script["service"], script["command"]
+            except KeyError as e:
+                raise exceptions.TutorError(
+                    "plugin script configuration requires key '{}'".format(e.args[0])
+                )

tutor/scripts.py

@@ -1,6 +1,7 @@
 from . import env
 from . import exceptions
 from . import fmt
+from . import plugins
 
 
 class BaseRunner:
@@ -8,12 +9,12 @@ class BaseRunner:
         self.root = root
         self.config = config
 
-    def render(self, script, config=None):
+    def render(self, service, script, config=None):
         config = config or self.config
-        return env.render_file(config, "scripts", script).strip()
+        return env.render_file(config, "scripts", service, script).strip()
 
     def run(self, service, script, config=None):
-        command = self.render(script, config=config)
+        command = self.render(service, script, config=config)
         self.exec(service, command)
 
     def exec(self, service, command):
@@ -33,26 +34,18 @@ class BaseRunner:
 
 def migrate(runner):
     fmt.echo_info("Creating all databases...")
-    runner.run("mysql-client", "create_databases.sh")
+    runner.run("mysql-client", "createdatabases")
 
-    if runner.is_activated("lms"):
-        fmt.echo_info("Running lms migrations...")
-        runner.run("lms", "migrate_lms.sh")
-    if runner.is_activated("cms"):
-        fmt.echo_info("Running cms migrations...")
-        runner.run("cms", "migrate_cms.sh")
-    if runner.is_activated("forum"):
-        fmt.echo_info("Running forum migrations...")
-        runner.run("forum", "migrate_forum.sh")
-    if runner.is_activated("notes"):
-        fmt.echo_info("Running notes migrations...")
-        runner.run("notes", "migrate_django.sh")
-    if runner.is_activated("xqueue"):
-        fmt.echo_info("Running xqueue migrations...")
-        runner.run("xqueue", "migrate_django.sh")
-    if runner.is_activated("lms"):
-        fmt.echo_info("Creating oauth2 users...")
-        runner.run("lms", "oauth2.sh")
+    for service in ["lms", "cms", "forum", "notes", "xqueue"]:
+        if runner.is_activated(service):
+            fmt.echo_info("Running {} migrations...".format(service))
+            runner.run(service, "init")
+    # TODO it's really ugly to load the config from the runner
+    for plugin_name, service, command in plugins.iter_scripts(runner.config, "init"):
+        fmt.echo_info(
+            "Plugin {}: running init for service {}...".format(plugin_name, service)
+        )
+        runner.run(service, "init")
     fmt.echo_info("Databases ready.")
 
 
@@ -63,14 +56,14 @@ def create_user(runner, superuser, staff, name, email):
         config["OPTS"] += " --superuser"
     if staff:
         config["OPTS"] += " --staff"
-    runner.run("lms", "create_user.sh", config=config)
+    runner.run("lms", "createuser", config=config)
 
 
 def import_demo_course(runner):
     runner.check_service_is_activated("cms")
-    runner.run("cms", "import_demo_course.sh")
+    runner.run("cms", "importdemocourse")
 
 
 def index_courses(runner):
     runner.check_service_is_activated("cms")
-    runner.run("cms", "index_courses.sh")
+    runner.run("cms", "indexcourses")

tutor/templates/apps/nginx/extra.conf

@@ -34,3 +34,5 @@ server {
   }
 }
 {% endif %}
+
+{{ patch("nginx-extra") }}

tutor/templates/apps/openedx/config/cms.auth.json

@@ -1,46 +1 @@
-{
-  "SECRET_KEY": "{{ SECRET_KEY }}",
-  "AWS_ACCESS_KEY_ID": "",
-  "AWS_SECRET_ACCESS_KEY": "",
-  "XQUEUE_INTERFACE": {
-    "django_auth": {
-      "username": "{{ XQUEUE_AUTH_USERNAME }}",
-      "password": "{{ XQUEUE_AUTH_PASSWORD }}"
-    },
-    "url": "http://xqueue:8040"
-  },
-  "CONTENTSTORE": { 
-    "ENGINE": "xmodule.contentstore.mongo.MongoContentStore",                       
-    "DOC_STORE_CONFIG": {                                                           
-      "host": "{{ MONGODB_HOST }}",
-      "port": {{ MONGODB_PORT }},
-      {% if MONGODB_USERNAME and MONGODB_PASSWORD %}
-      "user": "{{ MONGODB_USERNAME }}",
-      "password": "{{ MONGODB_PASSWORD }}",
-      {% endif %}
-      "db": "{{ MONGODB_DATABASE }}"
-    } 
-  },  
-  "DOC_STORE_CONFIG": {
-    "host": "{{ MONGODB_HOST }}",
-    "port": {{ MONGODB_PORT }},
-    {% if MONGODB_USERNAME and MONGODB_PASSWORD %}
-    "user": "{{ MONGODB_USERNAME }}",
-    "password": "{{ MONGODB_PASSWORD }}",
-    {% endif %}
-    "db": "{{ MONGODB_DATABASE }}"
-  },
-  "DATABASES": {
-    "default": {
-      "ENGINE": "django.db.backends.mysql",
-      "HOST": "{{ MYSQL_HOST }}",
-      "PORT": {{ MYSQL_PORT }},
-      "NAME": "{{ OPENEDX_MYSQL_DATABASE }}",
-      "USER": "{{ OPENEDX_MYSQL_USERNAME }}",
-      "PASSWORD": "{{ OPENEDX_MYSQL_PASSWORD }}",
-      "ATOMIC_REQUESTS": true
-    }
-  },
-  "EMAIL_HOST_USER": "{{ SMTP_USERNAME }}",
-  "EMAIL_HOST_PASSWORD": "{{ SMTP_PASSWORD }}"
-}
+{% include "apps/openedx/config/partials/auth.json" %}

tutor/templates/apps/openedx/config/lms.auth.json

@@ -1,46 +1 @@
-{
-  "SECRET_KEY": "{{ SECRET_KEY }}",
-  "AWS_ACCESS_KEY_ID": "",
-  "AWS_SECRET_ACCESS_KEY": "",
-  "XQUEUE_INTERFACE": {
-    "django_auth": {
-      "username": "{{ XQUEUE_AUTH_USERNAME }}",
-      "password": "{{ XQUEUE_AUTH_PASSWORD }}"
-    },
-    "url": "http://xqueue:8040"
-  },
-  "CONTENTSTORE": { 
-    "ENGINE": "xmodule.contentstore.mongo.MongoContentStore",                       
-    "DOC_STORE_CONFIG": {                                                           
-      "host": "{{ MONGODB_HOST }}",
-      "port": {{ MONGODB_PORT }},
-      {% if MONGODB_USERNAME and MONGODB_PASSWORD %}
-      "user": "{{ MONGODB_USERNAME }}",
-      "password": "{{ MONGODB_PASSWORD }}",
-      {% endif %}
-      "db": "{{ MONGODB_DATABASE }}"
-    } 
-  },  
-  "DOC_STORE_CONFIG": {
-    "host": "{{ MONGODB_HOST }}",
-    "port": {{ MONGODB_PORT }},
-    {% if MONGODB_USERNAME and MONGODB_PASSWORD %}
-    "user": "{{ MONGODB_USERNAME }}",
-    "password": "{{ MONGODB_PASSWORD }}",
-    {% endif %}
-    "db": "{{ MONGODB_DATABASE }}"
-  },
-  "DATABASES": {
-    "default": {
-      "ENGINE": "django.db.backends.mysql",
-      "HOST": "{{ MYSQL_HOST }}",
-      "PORT": {{ MYSQL_PORT }},
-      "NAME": "{{ OPENEDX_MYSQL_DATABASE }}",
-      "USER": "{{ OPENEDX_MYSQL_USERNAME }}",
-      "PASSWORD": "{{ OPENEDX_MYSQL_PASSWORD }}",
-      "ATOMIC_REQUESTS": true
-    }
-  },
-  "EMAIL_HOST_USER": "{{ SMTP_USERNAME }}",
-  "EMAIL_HOST_PASSWORD": "{{ SMTP_PASSWORD }}"
-}
+{% include "apps/openedx/config/partials/auth.json" %}

tutor/templates/apps/openedx/config/lms.env.json

@@ -45,6 +45,7 @@
   "LANGUAGE_CODE": "{{ LANGUAGE_CODE }}",
   "LOGIN_REDIRECT_WHITELIST": ["{{ CMS_HOST }}", "studio.localhost"],
   {% if ACTIVATE_HTTPS %}"SESSION_COOKIE_DOMAIN": ".{{ LMS_HOST|common_domain(CMS_HOST) }}",{% endif %}
+  {{ patch("lms-env", separator=",\n", suffix=",")|indent(2) }}
   "CACHES": {
     "default": {
       "KEY_PREFIX": "default",

tutor/templates/apps/openedx/config/partials/auth.json

@@ -0,0 +1,47 @@
+{
+  "SECRET_KEY": "{{ SECRET_KEY }}",
+  "AWS_ACCESS_KEY_ID": "{{ OPENEDX_AWS_ACCESS_KEY }}",
+  "AWS_SECRET_ACCESS_KEY": "{{ OPENEDX_AWS_SECRET_ACCESS_KEY }}",
+  "XQUEUE_INTERFACE": {
+    "django_auth": {
+      "username": "{{ XQUEUE_AUTH_USERNAME }}",
+      "password": "{{ XQUEUE_AUTH_PASSWORD }}"
+    },
+    "url": "http://xqueue:8040"
+  },
+  {{ patch("openedx-auth", separator=",\n", suffix=",")|indent(2) }}
+  "CONTENTSTORE": { 
+    "ENGINE": "xmodule.contentstore.mongo.MongoContentStore",                       
+    "DOC_STORE_CONFIG": {                                                           
+      "host": "{{ MONGODB_HOST }}",
+      "port": {{ MONGODB_PORT }},
+      {% if MONGODB_USERNAME and MONGODB_PASSWORD %}
+      "user": "{{ MONGODB_USERNAME }}",
+      "password": "{{ MONGODB_PASSWORD }}",
+      {% endif %}
+      "db": "{{ MONGODB_DATABASE }}"
+    } 
+  },  
+  "DOC_STORE_CONFIG": {
+    "host": "{{ MONGODB_HOST }}",
+    "port": {{ MONGODB_PORT }},
+    {% if MONGODB_USERNAME and MONGODB_PASSWORD %}
+    "user": "{{ MONGODB_USERNAME }}",
+    "password": "{{ MONGODB_PASSWORD }}",
+    {% endif %}
+    "db": "{{ MONGODB_DATABASE }}"
+  },
+  "DATABASES": {
+    "default": {
+      "ENGINE": "django.db.backends.mysql",
+      "HOST": "{{ MYSQL_HOST }}",
+      "PORT": {{ MYSQL_PORT }},
+      "NAME": "{{ OPENEDX_MYSQL_DATABASE }}",
+      "USER": "{{ OPENEDX_MYSQL_USERNAME }}",
+      "PASSWORD": "{{ OPENEDX_MYSQL_PASSWORD }}",
+      "ATOMIC_REQUESTS": true
+    }
+  },
+  "EMAIL_HOST_USER": "{{ SMTP_USERNAME }}",
+  "EMAIL_HOST_PASSWORD": "{{ SMTP_PASSWORD }}"
+}

tutor/templates/apps/openedx/settings/cms/common.py

@@ -24,6 +24,8 @@ LOGGING["loggers"]["tracking"]["handlers"] = ["console", "local", "tracking"]
 
 LOCALE_PATHS.append("/openedx/locale")
 
+{{ patch("openedx-common-settings") }}
+
 # Create folders if necessary
 for folder in [LOG_DIR, MEDIA_ROOT, STATIC_ROOT_BASE]:
     if not os.path.exists(folder):

tutor/templates/config.yml

@@ -28,6 +28,8 @@ ACTIVATE_NOTES: false
 ACTIVATE_RABBITMQ: true
 ACTIVATE_SMTP: true
 ACTIVATE_XQUEUE: false
+OPENEDX_AWS_ACCESS_KEY: ""
+OPENEDX_AWS_SECRET_ACCESS_KEY: ""
 ANDROID_RELEASE_STORE_PASSWORD: "android store password"
 ANDROID_RELEASE_KEY_PASSWORD: "android release key password"
 ANDROID_RELEASE_KEY_ALIAS: "android release key alias"
@@ -66,6 +68,7 @@ NOTES_HOST: "notes.{{ LMS_HOST }}"
 NOTES_MYSQL_DATABASE: "notes"
 NOTES_MYSQL_USERNAME: "notes"
 PLATFORM_NAME: "My Open edX"
+PLUGINS: []
 RABBITMQ_HOST: "rabbitmq"
 RABBITMQ_USERNAME: ""
 RABBITMQ_PASSWORD: ""

tutor/templates/local/docker-compose.yml

@@ -230,3 +230,5 @@ services:
     depends_on:
       {% if ACTIVATE_MYSQL %}- mysql{% endif %}
   {% endif %}
+  
+  {{ patch("local-docker-compose-services")|indent(2) }}

tutor/templates/scripts/certbot/create

@@ -1,2 +1,3 @@
 certbot certonly --standalone -n --agree-tos -m admin@{{ LMS_HOST }} -d {{ LMS_HOST }} -d {{ CMS_HOST }} -d preview.{{ LMS_HOST }}
 {% if ACTIVATE_NOTES %}certbot certonly --standalone -n --agree-tos -m admin@{{ LMS_HOST }} -d {{ NOTES_HOST }}{% endif %}
+{{ patch("https-create") }}

tutor/templates/scripts/lms/init

@@ -1,3 +1,5 @@
+dockerize -wait tcp://{{ MYSQL_HOST }}:{{ MYSQL_PORT }} -timeout 20s && ./manage.py lms migrate
+
 ./manage.py lms create_oauth2_client \
     "http://androidapp.com" "http://androidapp.com/redirect" public \
     --client_id android --client_secret {{ ANDROID_OAUTH2_SECRET }} \
@@ -9,4 +11,4 @@
     "http://notes.openedx:8000" "http://notes.openedx:8000/complete/edx-oidc/" confidential \
     --client_name edx-notes --client_id notes --client_secret {{ NOTES_OAUTH2_SECRET }} \
     --trusted --logout_uri "http://notes.openedx:8000/logout/" --username notes
-{% endif %}
+{% endif %}

tutor/templates/scripts/migrate_lms.sh

@@ -1 +0,0 @@
-dockerize -wait tcp://{{ MYSQL_HOST }}:{{ MYSQL_PORT }} -timeout 20s && ./manage.py lms migrate

tutor/templates/scripts/xqueue/init

@@ -0,0 +1 @@
+./manage.py migrate