🔌 Introduce a plugin system for tutor
This adds the basic feratures that we need for a working plugin system, but there are still many TODOs in the codebase.
This commit is contained in:
Régis Behmo
parent
38394c0824
commit
3b108d21de
|
|
@ -2,9 +2,9 @@
|
|||
!.gitignore
|
||||
TODO
|
||||
__pycache__
|
||||
*.egg-info/
|
||||
|
||||
/tutor.spec
|
||||
/build/tutor
|
||||
/dist/
|
||||
/releases/
|
||||
/tutor_openedx.egg-info/
|
||||
|
|
|
|||
11
Makefile
11
Makefile
|
|
@ -1,5 +1,5 @@
|
|||
.DEFAULT_GOAL := help
|
||||
BLACK_OPTS = --exclude templates ./tutor ./tests
|
||||
BLACK_OPTS = --exclude templates ./tutor ./tests ./plugins
|
||||
|
||||
###### Development
|
||||
|
||||
|
|
@ -16,15 +16,21 @@ test-format: ## Run code formatting tests
|
|||
test-lint: ## Run code linting tests
|
||||
pylint --errors-only tutor tests plugins
|
||||
|
||||
test-unit: ## Run unit tests
|
||||
test-unit: test-unit-core test-unit-plugins ## Run unit tests
|
||||
|
||||
test-unit-core:
|
||||
python3 -m unittest discover tests
|
||||
|
||||
test-unit-plugins:
|
||||
python3 -m unittest discover plugins/minio/tests
|
||||
|
||||
format: ## Format code automatically
|
||||
black $(BLACK_OPTS)
|
||||
|
||||
###### Deployment
|
||||
|
||||
bundle: ## Bundle the tutor package in a single "dist/tutor" executable
|
||||
# TODO bundle plugins
|
||||
pyinstaller --onefile --name=tutor --add-data=./tutor/templates:./tutor/templates ./bin/main
|
||||
dist/tutor:
|
||||
$(MAKE) bundle
|
||||
|
|
@ -52,6 +58,7 @@ ci-info: ## Print info about environment
|
|||
ci-install: ## Install requirements
|
||||
pip3 install -U setuptools
|
||||
pip3 install -r requirements/dev.txt
|
||||
pip3 install -r requirements/plugins.txt
|
||||
|
||||
ci-bundle: ## Create bundle and run basic tests
|
||||
$(MAKE) bundle
|
||||
|
|
|
|||
|
|
@ -0,0 +1,11 @@
|
|||
TODO
|
||||
|
||||
- This is mainly for production. Does not work with `tutor dev` commands.
|
||||
- For local testing, you need to set MINIO_HOST to minio.localhost:
|
||||
|
||||
tutor config save -y --set MINIO_HOST=minio.localhost
|
||||
|
||||
- You need `minio.LMS_HOST` domain name. For local development, the MinIO admin dashboard is at minio.localhost. For authentication, use MINIO_ACCESS_KEY and MINIO_SECRET_KEY:
|
||||
|
||||
tutor config printvalue OPENEDX_AWS_ACCESS_KEY
|
||||
tutor config printvalue OPENEDX_AWS_SECRET_ACCESS_KEY
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
import io
|
||||
import os
|
||||
from setuptools import setup
|
||||
|
||||
here = os.path.abspath(os.path.dirname(__file__))
|
||||
|
||||
with io.open(os.path.join(here, "README.rst"), "rt", encoding="utf8") as f:
|
||||
readme = f.read()
|
||||
|
||||
|
||||
setup(
|
||||
name="tutor-minio",
|
||||
version="0.0.1",
|
||||
url="https://docs.tutor.overhang.io/",
|
||||
project_urls={
|
||||
"Documentation": "https://docs.tutor.overhang.io/",
|
||||
"Code": "https://github.com/regisb/tutor/tree/master/plugins/minio",
|
||||
"Issue tracker": "https://github.com/regisb/tutor/issues",
|
||||
"Community": "https://discuss.overhang.io",
|
||||
},
|
||||
license="AGPLv3",
|
||||
author="Régis Behmo",
|
||||
author_email="regis@behmo.com",
|
||||
description="A Tutor plugin for object storage in MinIO",
|
||||
long_description=readme,
|
||||
packages=["tutorminio"],
|
||||
include_package_data=True,
|
||||
python_requires=">=3.5",
|
||||
install_requires=["click>=7.0"],
|
||||
entry_points={"tutor.plugin.v0": ["minio = tutorminio.plugin"]},
|
||||
classifiers=[
|
||||
"Development Status :: 3 - Alpha",
|
||||
"Intended Audience :: Developers",
|
||||
"License :: OSI Approved :: GNU Affero General Public License v3",
|
||||
"Operating System :: OS Independent",
|
||||
"Programming Language :: Python",
|
||||
"Programming Language :: Python :: 3.5",
|
||||
"Programming Language :: Python :: 3.6",
|
||||
"Programming Language :: Python :: 3.7",
|
||||
],
|
||||
)
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
import unittest
|
||||
|
||||
from tutorminio import plugin
|
||||
|
||||
|
||||
class PluginTests(unittest.TestCase):
|
||||
def test_patches(self):
|
||||
patches = dict(plugin.patches())
|
||||
self.assertIn("local-docker-compose-services", patches)
|
||||
self.assertTrue(
|
||||
patches["local-docker-compose-services"].startswith("# MinIO\n")
|
||||
)
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
# MinIO
|
||||
certbot certonly --standalone -n --agree-tos -m admin@{{ LMS_HOST }} -d {{ MINIO_HOST }}
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
"FILE_UPLOAD_STORAGE_BUCKET_NAME": "{{ MINIO_FILE_UPLOAD_BUCKET_NAME }}",
|
||||
"COURSE_IMPORT_EXPORT_BUCKET": "{{ MINIO_COURSE_IMPORT_EXPORT_BUCKET }}",
|
||||
"AWS_S3_CUSTOM_DOMAIN": "{{ MINIO_HOST }}"
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
# MinIO
|
||||
minio:
|
||||
image: {{ MINIO_DOCKER_REGISTRY }}{{ MINIO_DOCKER_IMAGE_SERVER }}
|
||||
volumes:
|
||||
- ../../data/minio:/data
|
||||
ports:
|
||||
- "9000:9000"
|
||||
environment:
|
||||
MINIO_ACCESS_KEY: "{{ OPENEDX_AWS_ACCESS_KEY }}"
|
||||
MINIO_SECRET_KEY: "{{ OPENEDX_AWS_SECRET_ACCESS_KEY }}"
|
||||
command: server /data
|
||||
restart: unless-stopped
|
||||
|
||||
minio-client:
|
||||
image: {{ MINIO_DOCKER_REGISTRY }}{{ MINIO_DOCKER_IMAGE_CLIENT }}
|
||||
entrypoint: sh
|
||||
command: "mc ls"
|
||||
restart: "no"
|
||||
depends_on:
|
||||
- minio
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
# MinIO public service
|
||||
upstream minio-backend {
|
||||
server minio:9000 fail_timeout=0;
|
||||
}
|
||||
server {
|
||||
{% if ACTIVATE_HTTPS %}listen 443 {{ "" if WEB_PROXY else "ssl" }};{% else %}listen 80;{% endif %}
|
||||
server_name minio.localhost {{ MINIO_HOST }};
|
||||
|
||||
{% if ACTIVATE_HTTPS and not WEB_PROXY %}
|
||||
ssl_certificate /etc/letsencrypt/live/{{ MINIO_HOST }}/fullchain.pem;
|
||||
ssl_certificate_key /etc/letsencrypt/live/{{ MINIO_HOST }}/privkey.pem;
|
||||
{% endif %}
|
||||
|
||||
# Disables server version feedback on pages and in headers
|
||||
server_tokens off;
|
||||
|
||||
location / {
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_set_header X-Forwarded-Port $server_port;
|
||||
proxy_set_header X-Forwarded-For $remote_addr;
|
||||
proxy_set_header Host $http_host;
|
||||
proxy_redirect off;
|
||||
|
||||
proxy_pass http://minio-backend;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
"AWS_STORAGE_BUCKET_NAME": "{{ MINIO_BUCKET_NAME }}",
|
||||
"AWS_S3_CUSTOM_DOMAIN": "{{ MINIO_HOST }}"
|
||||
|
|
@ -0,0 +1 @@
|
|||
AWS_S3_USE_SSL = {{ "True" if ACTIVATE_HTTPS else "False" }}
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
import os
|
||||
from glob import glob
|
||||
|
||||
HERE = os.path.abspath(os.path.dirname(__file__))
|
||||
|
||||
config = {
|
||||
"set": {
|
||||
"OPENEDX_AWS_ACCESS_KEY": "openedx",
|
||||
"OPENEDX_AWS_SECRET_ACCESS_KEY": "{{ 24|random_string }}",
|
||||
},
|
||||
"defaults": {
|
||||
"BUCKET_NAME": "openedx",
|
||||
"FILE_UPLOAD_BUCKET_NAME": "openedxuploads",
|
||||
"COURSE_IMPORT_EXPORT_BUCKET": "openedxcourseimportexport",
|
||||
"HOST": "minio.{{ LMS_HOST }}",
|
||||
"DOCKER_REGISTRY": "{{ DOCKER_REGISTRY }}",
|
||||
"DOCKER_IMAGE_CLIENT": "minio/mc:RELEASE.2019-05-23T01-33-27Z",
|
||||
"DOCKER_IMAGE_SERVER": "minio/minio:RELEASE.2019-05-23T00-29-34Z",
|
||||
},
|
||||
}
|
||||
|
||||
scripts = {
|
||||
"init": [
|
||||
{
|
||||
"service": "mysql-client",
|
||||
"command": """mc config host add minio http://minio:9000 {{ OPENEDX_AWS_ACCESS_KEY }} {{ OPENEDX_AWS_SECRET_ACCESS_KEY }} --api s3v4
|
||||
mc mb minio {{ FILE_UPLOAD_BUCKET_NAME }} {{ COURSE_IMPORT_EXPORT_BUCKET }}""",
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
def patches(*_args):
|
||||
all_patches = {}
|
||||
for path in glob(os.path.join(HERE, "patches", "*.patch")):
|
||||
with open(path) as patch_file:
|
||||
name = os.path.basename(path)[:-6]
|
||||
content = patch_file.read()
|
||||
all_patches[name] = content
|
||||
return all_patches
|
||||
|
|
@ -0,0 +1 @@
|
|||
-e ./plugins/minio
|
||||
|
|
@ -16,6 +16,12 @@ class ConfigTests(unittest.TestCase):
|
|||
self.assertNotIn("TUTOR_VERSION", defaults)
|
||||
|
||||
def test_merge(self):
|
||||
config1 = {"x": "y"}
|
||||
config2 = {"x": "z"}
|
||||
tutor_config.merge(config1, config2)
|
||||
self.assertEqual({"x": "y"}, config1)
|
||||
|
||||
def test_merge_render(self):
|
||||
config = {}
|
||||
defaults = tutor_config.load_defaults()
|
||||
with unittest.mock.patch.object(
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ class EnvTests(unittest.TestCase):
|
|||
config = {}
|
||||
tutor_config.merge(config, tutor_config.load_defaults())
|
||||
config["MYSQL_ROOT_PASSWORD"] = "testpassword"
|
||||
rendered = env.render_file(config, "scripts", "create_databases.sh")
|
||||
rendered = env.render_file(config, "scripts", "mysql-client", "createdatabases")
|
||||
self.assertIn("testpassword", rendered)
|
||||
|
||||
@unittest.mock.patch.object(tutor_config.fmt, "echo")
|
||||
|
|
@ -58,3 +58,22 @@ class EnvTests(unittest.TestCase):
|
|||
defaults["ACTIVATE_HTTPS"] = True
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
env.render_full(root, defaults)
|
||||
|
||||
def test_patch(self):
|
||||
patches = {"plugin1": "abcd", "plugin2": "efgh"}
|
||||
with unittest.mock.patch.object(
|
||||
env.plugins, "iter_patches", return_value=patches.items()
|
||||
) as mock_iter_patches:
|
||||
rendered = env.render_str({}, '{{ patch("location") }}')
|
||||
mock_iter_patches.assert_called_once_with({}, "location")
|
||||
self.assertEqual("abcd\nefgh", rendered)
|
||||
|
||||
def test_patch_separator_suffix(self):
|
||||
patches = {"plugin1": "abcd", "plugin2": "efgh"}
|
||||
with unittest.mock.patch.object(
|
||||
env.plugins, "iter_patches", return_value=patches.items()
|
||||
) as mock_iter_patches:
|
||||
rendered = env.render_str(
|
||||
{}, '{{ patch("location", separator=",\n", suffix=",") }}'
|
||||
)
|
||||
self.assertEqual("abcd,\nefgh,", rendered)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,90 @@
|
|||
import unittest
|
||||
import unittest.mock
|
||||
|
||||
from tutor import exceptions
|
||||
from tutor import plugins
|
||||
|
||||
|
||||
class PluginsTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
plugins.Patches.CACHE.clear()
|
||||
|
||||
def test_iter_installed(self):
|
||||
with unittest.mock.patch.object(
|
||||
plugins.pkg_resources, "iter_entry_points", return_value=[]
|
||||
):
|
||||
self.assertEqual([], list(plugins.iter_installed()))
|
||||
|
||||
def test_is_installed(self):
|
||||
self.assertFalse(plugins.is_installed("dummy"))
|
||||
|
||||
def test_enable(self):
|
||||
config = {plugins.CONFIG_KEY: []}
|
||||
with unittest.mock.patch.object(plugins, "is_installed", return_value=True):
|
||||
plugins.enable(config, "plugin2")
|
||||
plugins.enable(config, "plugin1")
|
||||
self.assertEqual(["plugin1", "plugin2"], config[plugins.CONFIG_KEY])
|
||||
|
||||
def test_enable_twice(self):
|
||||
config = {plugins.CONFIG_KEY: []}
|
||||
with unittest.mock.patch.object(plugins, "is_installed", return_value=True):
|
||||
plugins.enable(config, "plugin1")
|
||||
plugins.enable(config, "plugin1")
|
||||
self.assertEqual(["plugin1"], config[plugins.CONFIG_KEY])
|
||||
|
||||
def test_enable_not_installed_plugin(self):
|
||||
config = {"PLUGINS": []}
|
||||
with unittest.mock.patch.object(plugins, "is_installed", return_value=False):
|
||||
self.assertRaises(exceptions.TutorError, plugins.enable, config, "plugin1")
|
||||
|
||||
def test_patches(self):
|
||||
class plugin1:
|
||||
patches = {"patch1": "Hello {{ ID }}"}
|
||||
|
||||
with unittest.mock.patch.object(
|
||||
plugins, "iter_enabled", return_value=[("plugin1", plugin1)]
|
||||
):
|
||||
patches = list(plugins.iter_patches({}, "patch1"))
|
||||
self.assertEqual([("plugin1", "Hello {{ ID }}")], patches)
|
||||
|
||||
def test_plugin_without_patches(self):
|
||||
class plugin1:
|
||||
pass
|
||||
|
||||
with unittest.mock.patch.object(
|
||||
plugins, "iter_enabled", return_value=[("plugin1", plugin1)]
|
||||
):
|
||||
patches = list(plugins.iter_patches({}, "patch1"))
|
||||
self.assertEqual([], patches)
|
||||
|
||||
def test_configure(self):
|
||||
config = {"ID": "oldid"}
|
||||
|
||||
class plugin1:
|
||||
config = {
|
||||
"add": {"PARAM1": "value1", "PARAM2": "value2"},
|
||||
"set": {"ID": "newid"},
|
||||
"defaults": {"PARAM3": "value3"},
|
||||
}
|
||||
|
||||
with unittest.mock.patch.object(
|
||||
plugins, "iter_enabled", return_value=[("plugin1", plugin1)]
|
||||
):
|
||||
add_config, set_config, defaults_config = plugins.load_config(config)
|
||||
self.assertEqual(
|
||||
{"PLUGIN1_PARAM1": "value1", "PLUGIN1_PARAM2": "value2"}, add_config
|
||||
)
|
||||
self.assertEqual({"ID": "newid"}, set_config)
|
||||
self.assertEqual({"PLUGIN1_PARAM3": "value3"}, defaults_config)
|
||||
|
||||
def test_scripts(self):
|
||||
class plugin1:
|
||||
scripts = {"init": [{"service": "myclient", "command": "init command"}]}
|
||||
|
||||
with unittest.mock.patch.object(
|
||||
plugins, "iter_enabled", return_value=[("plugin1", plugin1)]
|
||||
):
|
||||
self.assertEqual(
|
||||
[("plugin1", "myclient", "init command")],
|
||||
list(plugins.iter_scripts({}, "init")),
|
||||
)
|
||||
|
|
@ -13,6 +13,6 @@ class ScriptsTests(unittest.TestCase):
|
|||
def test_run(self):
|
||||
config = tutor_config.load_defaults()
|
||||
runner = DummyRunner("/tmp", config)
|
||||
rendered_script = runner.render("create_databases.sh")
|
||||
runner.run("someservice", "create_databases.sh")
|
||||
runner.exec.assert_called_once_with("someservice", rendered_script)
|
||||
rendered_script = runner.render("mysql-client", "createdatabases")
|
||||
runner.run("mysql-client", "createdatabases")
|
||||
runner.exec.assert_called_once_with("mysql-client", rendered_script)
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ from .dev import dev
|
|||
from .images import images_command
|
||||
from .k8s import k8s
|
||||
from .local import local
|
||||
from .plugins import plugins_command
|
||||
from .ui import ui
|
||||
from .webui import webui
|
||||
from ..__about__ import __version__
|
||||
|
|
@ -47,6 +48,7 @@ cli.add_command(k8s)
|
|||
cli.add_command(ui)
|
||||
cli.add_command(webui)
|
||||
cli.add_command(print_help)
|
||||
cli.add_command(plugins_command)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ from .. import exceptions
|
|||
from .. import env
|
||||
from .. import fmt
|
||||
from .. import opts
|
||||
from .. import plugins
|
||||
from .. import serialize
|
||||
from .. import utils
|
||||
from ..__about__ import __version__
|
||||
|
|
@ -142,10 +143,14 @@ def load_defaults():
|
|||
|
||||
|
||||
def load_current(root, defaults):
|
||||
"""
|
||||
Note: this modifies the defaults. TODO this is not that great.
|
||||
"""
|
||||
convert_json2yml(root)
|
||||
config = load_user(root)
|
||||
load_env(config, defaults)
|
||||
load_required(config, defaults)
|
||||
load_plugins(config, defaults)
|
||||
return config
|
||||
|
||||
|
||||
|
|
@ -192,6 +197,15 @@ def load_required(config, defaults):
|
|||
config[key] = env.render_str(config, defaults[key])
|
||||
|
||||
|
||||
def load_plugins(config, defaults):
|
||||
add_config, set_config, defaults_config = plugins.load_config(config)
|
||||
merge(config, add_config)
|
||||
# TODO this might have unexpected consequences if plugins have conflicting configurations. Maybe we should print warning messages.
|
||||
merge(config, set_config)
|
||||
# TODO this modifies defaults, which is ugly.
|
||||
merge(defaults, defaults_config)
|
||||
|
||||
|
||||
def upgrade_obsolete(config):
|
||||
# Openedx-specific mysql passwords
|
||||
if "MYSQL_PASSWORD" in config:
|
||||
|
|
|
|||
|
|
@ -41,6 +41,11 @@ def run(root, edx_platform_path, edx_platform_settings, service, command, args):
|
|||
@click.argument("service", type=click.Choice(["lms", "cms"]))
|
||||
def runserver(root, edx_platform_path, edx_platform_settings, service):
|
||||
port = service_port(service)
|
||||
from .. import fmt
|
||||
|
||||
fmt.echo_info(
|
||||
"The {} service will be available at http://localhost:{}".format(service, port)
|
||||
)
|
||||
docker_compose_run_with_port(
|
||||
root,
|
||||
edx_platform_path,
|
||||
|
|
|
|||
|
|
@ -169,7 +169,7 @@ def https_create(root):
|
|||
fmt.echo_info("HTTPS is not activated: certificate generation skipped")
|
||||
return
|
||||
|
||||
script = runner.render("https_create.sh")
|
||||
script = runner.render("certbot", "create")
|
||||
|
||||
if config["WEB_PROXY"]:
|
||||
fmt.echo_info(
|
||||
|
|
|
|||
|
|
@ -0,0 +1,44 @@
|
|||
import click
|
||||
|
||||
from . import config as tutor_config
|
||||
from .. import opts
|
||||
from .. import plugins
|
||||
|
||||
|
||||
@click.group(
|
||||
name="plugins",
|
||||
short_help="Manage Tutor plugins",
|
||||
help="Manage Tutor plugins to add new features and customize your Open edX platform",
|
||||
)
|
||||
def plugins_command():
|
||||
pass
|
||||
|
||||
|
||||
@click.command(name="list", help="List installed plugins")
|
||||
@opts.root
|
||||
def list_command(root):
|
||||
config = tutor_config.load(root)
|
||||
for name, _ in plugins.iter_installed():
|
||||
status = "" if plugins.is_enabled(config, name) else " (disabled)"
|
||||
print("{plugin}{status}".format(plugin=name, status=status))
|
||||
|
||||
|
||||
@click.command(help="Enable a plugin")
|
||||
@click.argument("plugin")
|
||||
@opts.root
|
||||
def enable(plugin, root):
|
||||
config = tutor_config.load(root)
|
||||
plugins.enable(config, plugin)
|
||||
tutor_config.save_config(root, config)
|
||||
tutor_config.save(root, silent=True)
|
||||
|
||||
|
||||
@click.command(help="Disable a plugin")
|
||||
@opts.root
|
||||
def disable(root):
|
||||
pass
|
||||
|
||||
|
||||
plugins_command.add_command(list_command)
|
||||
plugins_command.add_command(enable)
|
||||
plugins_command.add_command(disable)
|
||||
37
tutor/env.py
37
tutor/env.py
|
|
@ -6,6 +6,7 @@ import jinja2
|
|||
|
||||
from . import exceptions
|
||||
from . import fmt
|
||||
from . import plugins
|
||||
from . import utils
|
||||
from .__about__ import __version__
|
||||
|
||||
|
|
@ -39,13 +40,17 @@ class Renderer:
|
|||
@classmethod
|
||||
def render_str(cls, config, text):
|
||||
template = cls.environment().from_string(text)
|
||||
return cls.__render(template, config)
|
||||
return cls.__render(config, template)
|
||||
|
||||
@classmethod
|
||||
def render_file(cls, config, path):
|
||||
template = cls.environment().get_template(path)
|
||||
try:
|
||||
return cls.__render(template, config)
|
||||
template = cls.environment().get_template(path)
|
||||
except:
|
||||
fmt.echo_error("Error loading template " + path)
|
||||
raise
|
||||
try:
|
||||
return cls.__render(config, template)
|
||||
except (jinja2.exceptions.TemplateError, exceptions.TutorError):
|
||||
fmt.echo_error("Error rendering template " + path)
|
||||
raise
|
||||
|
|
@ -54,14 +59,35 @@ class Renderer:
|
|||
raise
|
||||
|
||||
@classmethod
|
||||
def __render(cls, template, config):
|
||||
def __render(cls, config, template):
|
||||
def patch(name, separator="\n", suffix=""):
|
||||
return cls.__render_patch(config, name, separator=separator, suffix=suffix)
|
||||
|
||||
try:
|
||||
return template.render(**config)
|
||||
return template.render(patch=patch, **config)
|
||||
except jinja2.exceptions.UndefinedError as e:
|
||||
raise exceptions.TutorError(
|
||||
"Missing configuration value: {}".format(e.args[0])
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def __render_patch(cls, config, name, separator="\n", suffix=""):
|
||||
patches = []
|
||||
for plugin, patch in plugins.iter_patches(config, name):
|
||||
patch_template = cls.environment().from_string(patch)
|
||||
try:
|
||||
patches.append(patch_template.render(**config))
|
||||
except jinja2.exceptions.UndefinedError as e:
|
||||
raise exceptions.TutorError(
|
||||
"Missing configuration value: {} in patch '{}' from plugin {}".format(
|
||||
e.args[0], name, plugin
|
||||
)
|
||||
)
|
||||
rendered = separator.join(patches)
|
||||
if rendered:
|
||||
rendered += suffix
|
||||
return rendered
|
||||
|
||||
|
||||
def render_full(root, config):
|
||||
"""
|
||||
|
|
@ -194,6 +220,7 @@ def is_part_of_env(path):
|
|||
basename.startswith(".")
|
||||
or basename.endswith(".pyc")
|
||||
or basename == "__pycache__"
|
||||
or basename == "partials"
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,155 @@
|
|||
import pkg_resources
|
||||
|
||||
from . import exceptions
|
||||
|
||||
"""
|
||||
Tutor plugins are regular python packages that have a 'tutor.plugin.v1' entrypoint. This entrypoint must point to a module or a class that implements I don't know what (yet).
|
||||
TODO
|
||||
"""
|
||||
|
||||
# TODO switch to v1
|
||||
ENTRYPOINT = "tutor.plugin.v0"
|
||||
CONFIG_KEY = "PLUGINS"
|
||||
|
||||
|
||||
class Patches:
|
||||
"""
|
||||
Provide a patch cache on which we can conveniently iterate without having to parse again all plugin patches for every environment file.
|
||||
|
||||
The CACHE static attribute is a dict of the form:
|
||||
|
||||
{
|
||||
"patchname": {
|
||||
"pluginname": "patch content",
|
||||
...
|
||||
},
|
||||
...
|
||||
}
|
||||
"""
|
||||
|
||||
CACHE = {}
|
||||
|
||||
def __init__(self, config, name):
|
||||
self.name = name
|
||||
if not self.CACHE:
|
||||
self.fill_cache(config)
|
||||
|
||||
def __iter__(self):
|
||||
"""
|
||||
Yields:
|
||||
plugin name (str)
|
||||
patch content (str)
|
||||
"""
|
||||
plugin_patches = self.CACHE.get(self.name, {})
|
||||
plugins = sorted(plugin_patches.keys())
|
||||
for plugin in plugins:
|
||||
yield plugin, plugin_patches[plugin]
|
||||
|
||||
@classmethod
|
||||
def fill_cache(cls, config):
|
||||
for plugin_name, plugin in iter_enabled(config):
|
||||
patches = get_callable_attr(plugin, "patches")
|
||||
for patch_name, content in patches.items():
|
||||
if patch_name not in cls.CACHE:
|
||||
cls.CACHE[patch_name] = {}
|
||||
cls.CACHE[patch_name][plugin_name] = content
|
||||
|
||||
|
||||
def get_callable_attr(plugin, attr_name):
|
||||
attr = getattr(plugin, attr_name, {})
|
||||
if callable(attr):
|
||||
# TODO pass config here for initialization
|
||||
attr = attr()
|
||||
return attr
|
||||
|
||||
|
||||
def is_installed(name):
|
||||
plugin_names = [name for name, _ in iter_installed()]
|
||||
return name in plugin_names
|
||||
|
||||
|
||||
def iter_installed():
|
||||
for entrypoint in pkg_resources.iter_entry_points(ENTRYPOINT):
|
||||
yield entrypoint.name, entrypoint.load()
|
||||
|
||||
|
||||
def enable(config, name):
|
||||
if not is_installed(name):
|
||||
raise exceptions.TutorError("plugin '{}' is not installed.".format(name))
|
||||
if is_enabled(config, name):
|
||||
return
|
||||
if CONFIG_KEY not in config:
|
||||
config[CONFIG_KEY] = []
|
||||
config[CONFIG_KEY].append(name)
|
||||
config[CONFIG_KEY].sort()
|
||||
|
||||
|
||||
def iter_enabled(config):
|
||||
for name, plugin in iter_installed():
|
||||
if is_enabled(config, name):
|
||||
yield name, plugin
|
||||
|
||||
|
||||
def is_enabled(config, name):
|
||||
return name in config.get(CONFIG_KEY, [])
|
||||
|
||||
|
||||
def iter_patches(config, name):
|
||||
for plugin, patch in Patches(config, name):
|
||||
yield plugin, patch
|
||||
|
||||
|
||||
def load_config(config):
|
||||
"""
|
||||
TODO Return:
|
||||
|
||||
add_config (dict): config key/values to add, if they are not already present
|
||||
set_config (dict): config key/values to set and override existing values
|
||||
"""
|
||||
add_config = {}
|
||||
set_config = {}
|
||||
defaults_config = {}
|
||||
for plugin_name, plugin in iter_enabled(config):
|
||||
plugin_prefix = plugin_name.upper() + "_"
|
||||
plugin_config = get_callable_attr(plugin, "config")
|
||||
|
||||
# Add new config key/values
|
||||
for key, value in plugin_config.get("add", {}).items():
|
||||
add_config[plugin_prefix + key] = value
|
||||
|
||||
# Set existing config key/values
|
||||
for key, value in plugin_config.get("set", {}).items():
|
||||
# TODO crash in case of conflicting keys between plugins
|
||||
# If key already exists
|
||||
set_config[key] = value
|
||||
|
||||
# Create new defaults
|
||||
for key, value in plugin_config.get("defaults", {}).items():
|
||||
defaults_config[plugin_prefix + key] = value
|
||||
return add_config, set_config, defaults_config
|
||||
|
||||
|
||||
def iter_scripts(config, script_name):
|
||||
"""
|
||||
Scripts are of the form:
|
||||
|
||||
scripts = {
|
||||
"script-name": [
|
||||
{
|
||||
"service": "service-name",
|
||||
"command": "...",
|
||||
},
|
||||
...
|
||||
],
|
||||
...
|
||||
}
|
||||
"""
|
||||
for plugin_name, plugin in iter_enabled(config):
|
||||
scripts = get_callable_attr(plugin, "scripts")
|
||||
for script in scripts.get(script_name, []):
|
||||
try:
|
||||
yield plugin_name, script["service"], script["command"]
|
||||
except KeyError as e:
|
||||
raise exceptions.TutorError(
|
||||
"plugin script configuration requires key '{}'".format(e.args[0])
|
||||
)
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
from . import env
|
||||
from . import exceptions
|
||||
from . import fmt
|
||||
from . import plugins
|
||||
|
||||
|
||||
class BaseRunner:
|
||||
|
|
@ -8,12 +9,12 @@ class BaseRunner:
|
|||
self.root = root
|
||||
self.config = config
|
||||
|
||||
def render(self, script, config=None):
|
||||
def render(self, service, script, config=None):
|
||||
config = config or self.config
|
||||
return env.render_file(config, "scripts", script).strip()
|
||||
return env.render_file(config, "scripts", service, script).strip()
|
||||
|
||||
def run(self, service, script, config=None):
|
||||
command = self.render(script, config=config)
|
||||
command = self.render(service, script, config=config)
|
||||
self.exec(service, command)
|
||||
|
||||
def exec(self, service, command):
|
||||
|
|
@ -33,26 +34,18 @@ class BaseRunner:
|
|||
|
||||
def migrate(runner):
|
||||
fmt.echo_info("Creating all databases...")
|
||||
runner.run("mysql-client", "create_databases.sh")
|
||||
runner.run("mysql-client", "createdatabases")
|
||||
|
||||
if runner.is_activated("lms"):
|
||||
fmt.echo_info("Running lms migrations...")
|
||||
runner.run("lms", "migrate_lms.sh")
|
||||
if runner.is_activated("cms"):
|
||||
fmt.echo_info("Running cms migrations...")
|
||||
runner.run("cms", "migrate_cms.sh")
|
||||
if runner.is_activated("forum"):
|
||||
fmt.echo_info("Running forum migrations...")
|
||||
runner.run("forum", "migrate_forum.sh")
|
||||
if runner.is_activated("notes"):
|
||||
fmt.echo_info("Running notes migrations...")
|
||||
runner.run("notes", "migrate_django.sh")
|
||||
if runner.is_activated("xqueue"):
|
||||
fmt.echo_info("Running xqueue migrations...")
|
||||
runner.run("xqueue", "migrate_django.sh")
|
||||
if runner.is_activated("lms"):
|
||||
fmt.echo_info("Creating oauth2 users...")
|
||||
runner.run("lms", "oauth2.sh")
|
||||
for service in ["lms", "cms", "forum", "notes", "xqueue"]:
|
||||
if runner.is_activated(service):
|
||||
fmt.echo_info("Running {} migrations...".format(service))
|
||||
runner.run(service, "init")
|
||||
# TODO it's really ugly to load the config from the runner
|
||||
for plugin_name, service, command in plugins.iter_scripts(runner.config, "init"):
|
||||
fmt.echo_info(
|
||||
"Plugin {}: running init for service {}...".format(plugin_name, service)
|
||||
)
|
||||
runner.run(service, "init")
|
||||
fmt.echo_info("Databases ready.")
|
||||
|
||||
|
||||
|
|
@ -63,14 +56,14 @@ def create_user(runner, superuser, staff, name, email):
|
|||
config["OPTS"] += " --superuser"
|
||||
if staff:
|
||||
config["OPTS"] += " --staff"
|
||||
runner.run("lms", "create_user.sh", config=config)
|
||||
runner.run("lms", "createuser", config=config)
|
||||
|
||||
|
||||
def import_demo_course(runner):
|
||||
runner.check_service_is_activated("cms")
|
||||
runner.run("cms", "import_demo_course.sh")
|
||||
runner.run("cms", "importdemocourse")
|
||||
|
||||
|
||||
def index_courses(runner):
|
||||
runner.check_service_is_activated("cms")
|
||||
runner.run("cms", "index_courses.sh")
|
||||
runner.run("cms", "indexcourses")
|
||||
|
|
|
|||
|
|
@ -34,3 +34,5 @@ server {
|
|||
}
|
||||
}
|
||||
{% endif %}
|
||||
|
||||
{{ patch("nginx-extra") }}
|
||||
|
|
|
|||
|
|
@ -1,46 +1 @@
|
|||
{
|
||||
"SECRET_KEY": "{{ SECRET_KEY }}",
|
||||
"AWS_ACCESS_KEY_ID": "",
|
||||
"AWS_SECRET_ACCESS_KEY": "",
|
||||
"XQUEUE_INTERFACE": {
|
||||
"django_auth": {
|
||||
"username": "{{ XQUEUE_AUTH_USERNAME }}",
|
||||
"password": "{{ XQUEUE_AUTH_PASSWORD }}"
|
||||
},
|
||||
"url": "http://xqueue:8040"
|
||||
},
|
||||
"CONTENTSTORE": {
|
||||
"ENGINE": "xmodule.contentstore.mongo.MongoContentStore",
|
||||
"DOC_STORE_CONFIG": {
|
||||
"host": "{{ MONGODB_HOST }}",
|
||||
"port": {{ MONGODB_PORT }},
|
||||
{% if MONGODB_USERNAME and MONGODB_PASSWORD %}
|
||||
"user": "{{ MONGODB_USERNAME }}",
|
||||
"password": "{{ MONGODB_PASSWORD }}",
|
||||
{% endif %}
|
||||
"db": "{{ MONGODB_DATABASE }}"
|
||||
}
|
||||
},
|
||||
"DOC_STORE_CONFIG": {
|
||||
"host": "{{ MONGODB_HOST }}",
|
||||
"port": {{ MONGODB_PORT }},
|
||||
{% if MONGODB_USERNAME and MONGODB_PASSWORD %}
|
||||
"user": "{{ MONGODB_USERNAME }}",
|
||||
"password": "{{ MONGODB_PASSWORD }}",
|
||||
{% endif %}
|
||||
"db": "{{ MONGODB_DATABASE }}"
|
||||
},
|
||||
"DATABASES": {
|
||||
"default": {
|
||||
"ENGINE": "django.db.backends.mysql",
|
||||
"HOST": "{{ MYSQL_HOST }}",
|
||||
"PORT": {{ MYSQL_PORT }},
|
||||
"NAME": "{{ OPENEDX_MYSQL_DATABASE }}",
|
||||
"USER": "{{ OPENEDX_MYSQL_USERNAME }}",
|
||||
"PASSWORD": "{{ OPENEDX_MYSQL_PASSWORD }}",
|
||||
"ATOMIC_REQUESTS": true
|
||||
}
|
||||
},
|
||||
"EMAIL_HOST_USER": "{{ SMTP_USERNAME }}",
|
||||
"EMAIL_HOST_PASSWORD": "{{ SMTP_PASSWORD }}"
|
||||
}
|
||||
{% include "apps/openedx/config/partials/auth.json" %}
|
||||
|
|
@ -1,46 +1 @@
|
|||
{
|
||||
"SECRET_KEY": "{{ SECRET_KEY }}",
|
||||
"AWS_ACCESS_KEY_ID": "",
|
||||
"AWS_SECRET_ACCESS_KEY": "",
|
||||
"XQUEUE_INTERFACE": {
|
||||
"django_auth": {
|
||||
"username": "{{ XQUEUE_AUTH_USERNAME }}",
|
||||
"password": "{{ XQUEUE_AUTH_PASSWORD }}"
|
||||
},
|
||||
"url": "http://xqueue:8040"
|
||||
},
|
||||
"CONTENTSTORE": {
|
||||
"ENGINE": "xmodule.contentstore.mongo.MongoContentStore",
|
||||
"DOC_STORE_CONFIG": {
|
||||
"host": "{{ MONGODB_HOST }}",
|
||||
"port": {{ MONGODB_PORT }},
|
||||
{% if MONGODB_USERNAME and MONGODB_PASSWORD %}
|
||||
"user": "{{ MONGODB_USERNAME }}",
|
||||
"password": "{{ MONGODB_PASSWORD }}",
|
||||
{% endif %}
|
||||
"db": "{{ MONGODB_DATABASE }}"
|
||||
}
|
||||
},
|
||||
"DOC_STORE_CONFIG": {
|
||||
"host": "{{ MONGODB_HOST }}",
|
||||
"port": {{ MONGODB_PORT }},
|
||||
{% if MONGODB_USERNAME and MONGODB_PASSWORD %}
|
||||
"user": "{{ MONGODB_USERNAME }}",
|
||||
"password": "{{ MONGODB_PASSWORD }}",
|
||||
{% endif %}
|
||||
"db": "{{ MONGODB_DATABASE }}"
|
||||
},
|
||||
"DATABASES": {
|
||||
"default": {
|
||||
"ENGINE": "django.db.backends.mysql",
|
||||
"HOST": "{{ MYSQL_HOST }}",
|
||||
"PORT": {{ MYSQL_PORT }},
|
||||
"NAME": "{{ OPENEDX_MYSQL_DATABASE }}",
|
||||
"USER": "{{ OPENEDX_MYSQL_USERNAME }}",
|
||||
"PASSWORD": "{{ OPENEDX_MYSQL_PASSWORD }}",
|
||||
"ATOMIC_REQUESTS": true
|
||||
}
|
||||
},
|
||||
"EMAIL_HOST_USER": "{{ SMTP_USERNAME }}",
|
||||
"EMAIL_HOST_PASSWORD": "{{ SMTP_PASSWORD }}"
|
||||
}
|
||||
{% include "apps/openedx/config/partials/auth.json" %}
|
||||
|
|
@ -45,6 +45,7 @@
|
|||
"LANGUAGE_CODE": "{{ LANGUAGE_CODE }}",
|
||||
"LOGIN_REDIRECT_WHITELIST": ["{{ CMS_HOST }}", "studio.localhost"],
|
||||
{% if ACTIVATE_HTTPS %}"SESSION_COOKIE_DOMAIN": ".{{ LMS_HOST|common_domain(CMS_HOST) }}",{% endif %}
|
||||
{{ patch("lms-env", separator=",\n", suffix=",")|indent(2) }}
|
||||
"CACHES": {
|
||||
"default": {
|
||||
"KEY_PREFIX": "default",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,47 @@
|
|||
{
|
||||
"SECRET_KEY": "{{ SECRET_KEY }}",
|
||||
"AWS_ACCESS_KEY_ID": "{{ OPENEDX_AWS_ACCESS_KEY }}",
|
||||
"AWS_SECRET_ACCESS_KEY": "{{ OPENEDX_AWS_SECRET_ACCESS_KEY }}",
|
||||
"XQUEUE_INTERFACE": {
|
||||
"django_auth": {
|
||||
"username": "{{ XQUEUE_AUTH_USERNAME }}",
|
||||
"password": "{{ XQUEUE_AUTH_PASSWORD }}"
|
||||
},
|
||||
"url": "http://xqueue:8040"
|
||||
},
|
||||
{{ patch("openedx-auth", separator=",\n", suffix=",")|indent(2) }}
|
||||
"CONTENTSTORE": {
|
||||
"ENGINE": "xmodule.contentstore.mongo.MongoContentStore",
|
||||
"DOC_STORE_CONFIG": {
|
||||
"host": "{{ MONGODB_HOST }}",
|
||||
"port": {{ MONGODB_PORT }},
|
||||
{% if MONGODB_USERNAME and MONGODB_PASSWORD %}
|
||||
"user": "{{ MONGODB_USERNAME }}",
|
||||
"password": "{{ MONGODB_PASSWORD }}",
|
||||
{% endif %}
|
||||
"db": "{{ MONGODB_DATABASE }}"
|
||||
}
|
||||
},
|
||||
"DOC_STORE_CONFIG": {
|
||||
"host": "{{ MONGODB_HOST }}",
|
||||
"port": {{ MONGODB_PORT }},
|
||||
{% if MONGODB_USERNAME and MONGODB_PASSWORD %}
|
||||
"user": "{{ MONGODB_USERNAME }}",
|
||||
"password": "{{ MONGODB_PASSWORD }}",
|
||||
{% endif %}
|
||||
"db": "{{ MONGODB_DATABASE }}"
|
||||
},
|
||||
"DATABASES": {
|
||||
"default": {
|
||||
"ENGINE": "django.db.backends.mysql",
|
||||
"HOST": "{{ MYSQL_HOST }}",
|
||||
"PORT": {{ MYSQL_PORT }},
|
||||
"NAME": "{{ OPENEDX_MYSQL_DATABASE }}",
|
||||
"USER": "{{ OPENEDX_MYSQL_USERNAME }}",
|
||||
"PASSWORD": "{{ OPENEDX_MYSQL_PASSWORD }}",
|
||||
"ATOMIC_REQUESTS": true
|
||||
}
|
||||
},
|
||||
"EMAIL_HOST_USER": "{{ SMTP_USERNAME }}",
|
||||
"EMAIL_HOST_PASSWORD": "{{ SMTP_PASSWORD }}"
|
||||
}
|
||||
|
|
@ -24,6 +24,8 @@ LOGGING["loggers"]["tracking"]["handlers"] = ["console", "local", "tracking"]
|
|||
|
||||
LOCALE_PATHS.append("/openedx/locale")
|
||||
|
||||
{{ patch("openedx-common-settings") }}
|
||||
|
||||
# Create folders if necessary
|
||||
for folder in [LOG_DIR, MEDIA_ROOT, STATIC_ROOT_BASE]:
|
||||
if not os.path.exists(folder):
|
||||
|
|
|
|||
|
|
@ -28,6 +28,8 @@ ACTIVATE_NOTES: false
|
|||
ACTIVATE_RABBITMQ: true
|
||||
ACTIVATE_SMTP: true
|
||||
ACTIVATE_XQUEUE: false
|
||||
OPENEDX_AWS_ACCESS_KEY: ""
|
||||
OPENEDX_AWS_SECRET_ACCESS_KEY: ""
|
||||
ANDROID_RELEASE_STORE_PASSWORD: "android store password"
|
||||
ANDROID_RELEASE_KEY_PASSWORD: "android release key password"
|
||||
ANDROID_RELEASE_KEY_ALIAS: "android release key alias"
|
||||
|
|
@ -66,6 +68,7 @@ NOTES_HOST: "notes.{{ LMS_HOST }}"
|
|||
NOTES_MYSQL_DATABASE: "notes"
|
||||
NOTES_MYSQL_USERNAME: "notes"
|
||||
PLATFORM_NAME: "My Open edX"
|
||||
PLUGINS: []
|
||||
RABBITMQ_HOST: "rabbitmq"
|
||||
RABBITMQ_USERNAME: ""
|
||||
RABBITMQ_PASSWORD: ""
|
||||
|
|
|
|||
|
|
@ -230,3 +230,5 @@ services:
|
|||
depends_on:
|
||||
{% if ACTIVATE_MYSQL %}- mysql{% endif %}
|
||||
{% endif %}
|
||||
|
||||
{{ patch("local-docker-compose-services")|indent(2) }}
|
||||
|
|
@ -1,2 +1,3 @@
|
|||
certbot certonly --standalone -n --agree-tos -m admin@{{ LMS_HOST }} -d {{ LMS_HOST }} -d {{ CMS_HOST }} -d preview.{{ LMS_HOST }}
|
||||
{% if ACTIVATE_NOTES %}certbot certonly --standalone -n --agree-tos -m admin@{{ LMS_HOST }} -d {{ NOTES_HOST }}{% endif %}
|
||||
{{ patch("https-create") }}
|
||||
|
|
@ -1,3 +1,5 @@
|
|||
dockerize -wait tcp://{{ MYSQL_HOST }}:{{ MYSQL_PORT }} -timeout 20s && ./manage.py lms migrate
|
||||
|
||||
./manage.py lms create_oauth2_client \
|
||||
"http://androidapp.com" "http://androidapp.com/redirect" public \
|
||||
--client_id android --client_secret {{ ANDROID_OAUTH2_SECRET }} \
|
||||
|
|
@ -9,4 +11,4 @@
|
|||
"http://notes.openedx:8000" "http://notes.openedx:8000/complete/edx-oidc/" confidential \
|
||||
--client_name edx-notes --client_id notes --client_secret {{ NOTES_OAUTH2_SECRET }} \
|
||||
--trusted --logout_uri "http://notes.openedx:8000/logout/" --username notes
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
|
|
@ -1 +0,0 @@
|
|||
dockerize -wait tcp://{{ MYSQL_HOST }}:{{ MYSQL_PORT }} -timeout 20s && ./manage.py lms migrate
|
||||
|
|
@ -0,0 +1 @@
|
|||
./manage.py migrate
|
||||
Loading…
Reference in New Issue