From 07a0323d8e8e68ba67535dbbf8c2d59f694004d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Tue, 2 Jul 2019 22:16:44 +0200 Subject: [PATCH] Move Xqueue to a dedicated plugin This gives us the opportunity to develop new hooks: build-image and remote-image. --- CHANGELOG.md | 4 + Makefile | 7 +- docs/configuration.rst | 9 -- docs/plugins.rst | 70 +++++++- plugins/minio/setup.py | 1 - plugins/xqueue/README.rst | 28 ++++ plugins/xqueue/setup.py | 40 +++++ plugins/xqueue/tests/__init__.py | 0 .../tutorxqueue/patches/k8s-deployments | 43 +++++ .../xqueue/tutorxqueue/patches/k8s-services | 12 ++ .../patches/kustomization-configmapgenerator | 3 + .../patches/local-docker-compose-services | 24 +++ .../xqueue/tutorxqueue/patches/openedx-auth | 7 + plugins/xqueue/tutorxqueue/plugin.py | 35 ++++ .../xqueue}/apps/xqueue/settings/tutor.py | 0 .../templates/xqueue}/build/xqueue/Dockerfile | 0 .../templates/xqueue/hooks/mysql-client/init | 2 + .../templates/xqueue}/hooks/xqueue/init | 0 requirements/plugins.txt | 1 + tests/test_env.py | 46 ++++++ tests/test_plugins.py | 2 +- tutor/commands/images.py | 150 ++++++++++-------- tutor/config.py | 7 +- tutor/env.py | 30 +++- tutor/images.py | 21 +++ tutor/interactive.py | 6 - tutor/plugins.py | 4 +- tutor/scripts.py | 13 +- .../apps/openedx/config/partials/auth.json | 7 - tutor/templates/config.yml | 8 - tutor/templates/hooks/mysql-client/init | 5 - tutor/templates/k8s/deployments.yml | 45 ------ tutor/templates/k8s/services.yml | 14 -- tutor/templates/kustomization.yml | 4 +- tutor/templates/local/docker-compose.yml | 27 ---- 35 files changed, 454 insertions(+), 221 deletions(-) create mode 100644 plugins/xqueue/README.rst create mode 100644 plugins/xqueue/setup.py create mode 100644 plugins/xqueue/tests/__init__.py create mode 100644 plugins/xqueue/tutorxqueue/patches/k8s-deployments create mode 100644 plugins/xqueue/tutorxqueue/patches/k8s-services create mode 100644 plugins/xqueue/tutorxqueue/patches/kustomization-configmapgenerator create mode 100644 plugins/xqueue/tutorxqueue/patches/local-docker-compose-services create mode 100644 plugins/xqueue/tutorxqueue/patches/openedx-auth create mode 100644 plugins/xqueue/tutorxqueue/plugin.py rename {tutor/templates => plugins/xqueue/tutorxqueue/templates/xqueue}/apps/xqueue/settings/tutor.py (100%) rename {tutor/templates => plugins/xqueue/tutorxqueue/templates/xqueue}/build/xqueue/Dockerfile (100%) create mode 100644 plugins/xqueue/tutorxqueue/templates/xqueue/hooks/mysql-client/init rename {tutor/templates => plugins/xqueue/tutorxqueue/templates/xqueue}/hooks/xqueue/init (100%) create mode 100644 tutor/images.py diff --git a/CHANGELOG.md b/CHANGELOG.md index dc974d4..f539a94 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ Note: Breaking changes between versions are indicated by "💥". +## Latest + +- [Improvement] Move Xqueue to a dedicated plugin + ## 3.4.3 (2019-06-24) - [Bugfix] Fix missing password values from generated configuration diff --git a/Makefile b/Makefile index a764b4c..0bc0b78 100644 --- a/Makefile +++ b/Makefile @@ -14,6 +14,7 @@ package: ## Build a package ready to upload to pypi package-plugins: ## Build packages for each plugin cd plugins/minio && python3 setup.py sdist --dist-dir=../../dist/ + cd plugins/xqueue && python3 setup.py sdist --dist-dir=../../dist/ test: test-lint test-unit test-format test-packages ## Run all tests by decreasing order or priority @@ -21,7 +22,7 @@ test-format: ## Run code formatting tests black --check --diff $(BLACK_OPTS) test-lint: ## Run code linting tests - pylint --errors-only ${SRC_DIRS} + pylint --errors-only --ignore=templates ${SRC_DIRS} test-unit: test-unit-core test-unit-plugins ## Run unit tests @@ -30,6 +31,7 @@ test-unit-core: ## Run unit tests on core test-unit-plugins: ## Run unit tests on plugins python3 -m unittest discover plugins/minio/tests + python3 -m unittest discover plugins/xqueue/tests test-packages: package package-plugins ## Test that packages can be uploaded to pypi twine check dist/tutor-*.tar.gz @@ -110,7 +112,8 @@ ci-github: ./releases/github-release ## Upload assets to github --replace ci-config-images: - tutor config save --set ACTIVATE_NOTES=true --set ACTIVATE_XQUEUE=true + tutor plugin enable xqueue + tutor config save --set ACTIVATE_NOTES=true ci-build-images: ci-config-images ## Build docker images tutor images build all diff --git a/docs/configuration.rst b/docs/configuration.rst index e1c51cd..70c3744 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -49,7 +49,6 @@ Individual service activation - ``ACTIVATE_SMTP`` (default: ``true``) - ``ACTIVATE_HTTPS`` (default: ``false``) - ``ACTIVATE_NOTES`` (default: ``false``) -- ``ACTIVATE_XQUEUE`` (default: ``false``) Every single Open edX service may be (de)activated at will by these configuration parameters. This is useful if you want, for instance, to distribute the various Open edX services on different servers. @@ -65,7 +64,6 @@ Custom images - ``DOCKER_IMAGE_ANDROID`` (default: ``"overhangio/openedx-android:{{ TUTOR_VERSION }}"``) - ``DOCKER_IMAGE_FORUM`` (default: ``"overhangio/openedx-forum:{{ TUTOR_VERSION }}"``) - ``DOCKER_IMAGE_NOTES`` (default: ``"overhangio/openedx-notes:{{ TUTOR_VERSION }}"``) -- ``DOCKER_IMAGE_XQUEUE`` (default: ``"overhangio/openedx-xqueue:{{ TUTOR_VERSION }}"``) These configuration parameters define which image to run for each service. By default, the docker image tag matches the Tutor version it was built with. @@ -188,13 +186,6 @@ You should beware that the ``notes.`` domain name should be activated If you would like to host the notes service at a different domain name, you can set the ``NOTES_HOST`` configuration variable. In particular, in development you should set this configuration variable to ``notes.localhost`` in order to be able to access the notes service from the LMS. Otherwise you will get a "Sorry, we could not search the store for annotations" error. -Xqueue -****** - -- ``ACTIVATE_XQUEUE`` (default: ``false``) - -`Xqueue `_ is for grading problems with external services. If you don't know what it is, you probably don't need it. - .. _customise: Custom Open edX docker image diff --git a/docs/plugins.rst b/docs/plugins.rst index c6db0b2..7339381 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -33,7 +33,7 @@ API (v0) Note: The API for developing Tutor plugins is still considered unstable: profound changes should be expected for some time. -There are two mechanisms by which a plugin can integrate with Tutor: patches and hooks. Patches affect the rendered environment templates, while hooks are actions that are run during the lifetime of an Open edX platform. A plugin indicates which templates it patches, and which hooks it needs to run. +There are two mechanisms by which a plugin can integrate with Tutor: patches and hooks. Patches affect the rendered environment templates, while hooks are actions that are run during the lifetime of an Open edX platform. A plugin indicates which templates it patches, and which hooks it needs to run. A plugin can also affect the project configuration by adding new values and modifying existing values. Entrypoint ~~~~~~~~~~ @@ -90,14 +90,63 @@ This will add a Redis instance to the services run with ``tutor local`` commands ``hooks`` ~~~~~~~~~ -Hooks are services that are run during the lifetime of the platform. Currently, there is just one ``init`` hook. You should add there the services that will be run during initialisation, for instance for database creation and migrations. +Hooks are actions that are run during the lifetime of the platform. Each hook has a different specification. + +``init`` +++++++++ + +The services that will be run during initialisation should be added to the ``init`` hook, for instance for database creation and migrations. Example:: - hooks = {"init": ["myservice1", "myservice2"]} + hooks = { + "init": ["myservice1", "myservice2"] + } During initialisation, "myservice1" and "myservice2" will be run in sequence with the commands defined in the templates ``myplugin/hooks/myservice1/init`` and ``myplugin/hooks/myservice2/init``. +``build-image`` ++++++++++++++++ + +This is a hook that will be run to build a docker image for the requested service. + +Example:: + + hooks = { + "build-image": {"myimage": "myimage:latest"} + } + +With this hook, users will be able to build the ``myimage:latest`` docker image by running:: + + tutor images build myimage + +or:: + + tutor images build all + +This assumes that there is a ``Dockerfile`` file in the ``myplugin/build/myimage`` subfolder of the plugin templates directory. + +``remote-image`` +++++++++++++++++ + +This hook allows pulling/pushing images from/to a docker registry. + +Example:: + + hooks = { + "remote-image": {"myimage": "myimage:latest"}, + } + +With this hook, users will be able to pull and push the ``myimage:latest`` docker image by running:: + + tutor images pull myimage + tutor images push myimage + + or:: + + tutor images pull all + tutor images push all + ``templates`` ~~~~~~~~~~~~~ @@ -114,13 +163,22 @@ With the above declaration, you can store plugin-specific templates in the ``tem Existing plugins ---------------- -There exists just one Tutor plugin, for now. In the future, Xqueue and Student Notes will be moved outside of the main configuration and will have their own plugin. - MinIO ~~~~~ :: + pip install tutor-minio tutor plugins enable minio -See the `plugin documentation `_. \ No newline at end of file +See the `plugin documentation `_. + +Xqueue +~~~~~~ + +:: + + pip install tutor-xqueue + tutor plugins enable xqueue + +See the `plugin documentation `_. diff --git a/plugins/minio/setup.py b/plugins/minio/setup.py index 97f7da7..2fba2f2 100644 --- a/plugins/minio/setup.py +++ b/plugins/minio/setup.py @@ -26,7 +26,6 @@ setup( packages=["tutorminio"], include_package_data=True, python_requires=">=3.5", - install_requires=["click>=7.0"], entry_points={"tutor.plugin.v0": ["minio = tutorminio.plugin"]}, classifiers=[ "Development Status :: 3 - Alpha", diff --git a/plugins/xqueue/README.rst b/plugins/xqueue/README.rst new file mode 100644 index 0000000..b9888d2 --- /dev/null +++ b/plugins/xqueue/README.rst @@ -0,0 +1,28 @@ +Xqueue external grading system plugin for `Tutor `_ +=================================================================================== + +This is a plugin for `Tutor `_ that provides the Xqueue external grading system for Open edX platforms. If you don't know what it is, you probably don't need it. + +Installation +------------ + +The plugin is currently bundled with the `binary releases of Tutor `_. If you have installed Tutor from source, you will have to install this plugin from source, too:: + + pip install tutor-xqueue + +Then, to enable this plugin, run:: + + tutor plugins enable xqueue + +Configuration +------------- + +- ``XQUEUE_AUTH_PASSWORD`` (default: ``"{{ 8|random_string }}"``) +- ``XQUEUE_MYSQL_PASSWORD`` (default: ``"{{ 8|random_string }}"``) +- ``XQUEUE_SECRET_KEY`` (default: ``"{{ 24|random_string }}"``) +- ``XQUEUE_DOCKER_IMAGE`` (default: ``"overhangio/openedx-xqueue:{{ TUTOR_VERSION }}"``) +- ``XQUEUE_AUTH_USERNAME`` (default: ``"lms"``) +- ``XQUEUE_MYSQL_DATABASE`` (default: ``"xqueue"`` +- ``XQUEUE_MYSQL_USERNAME`` (default: ``"xqueue"``) + +These values can be modified with ``tutor config save --set PARAM_NAME=VALUE`` commands. diff --git a/plugins/xqueue/setup.py b/plugins/xqueue/setup.py new file mode 100644 index 0000000..674a53a --- /dev/null +++ b/plugins/xqueue/setup.py @@ -0,0 +1,40 @@ +import io +import os +from setuptools import setup + +here = os.path.abspath(os.path.dirname(__file__)) + +with io.open(os.path.join(here, "README.rst"), "rt", encoding="utf8") as f: + readme = f.read() + + +setup( + name="tutor-xqueue", + version="0.0.1", + url="https://docs.tutor.overhang.io/", + project_urls={ + "Documentation": "https://docs.tutor.overhang.io/", + "Code": "https://github.com/overhangio/tutor/tree/master/plugins/minio", + "Issue tracker": "https://github.com/overhangio/tutor/issues", + "Community": "https://discuss.overhang.io", + }, + license="AGPLv3", + author="Overhang.io", + author_email="contact@overhang.io", + description="A Tutor plugin for Xqueue (external grading system)", + long_description=readme, + packages=["tutorxqueue"], + include_package_data=True, + python_requires=">=3.5", + entry_points={"tutor.plugin.v0": ["xqueue = tutorxqueue.plugin"]}, + classifiers=[ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: GNU Affero General Public License v3", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + ], +) diff --git a/plugins/xqueue/tests/__init__.py b/plugins/xqueue/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugins/xqueue/tutorxqueue/patches/k8s-deployments b/plugins/xqueue/tutorxqueue/patches/k8s-deployments new file mode 100644 index 0000000..7cc0f14 --- /dev/null +++ b/plugins/xqueue/tutorxqueue/patches/k8s-deployments @@ -0,0 +1,43 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: xqueue + labels: + app.kubernetes.io/name: xqueue +spec: + selector: + matchLabels: + app.kubernetes.io/name: xqueue + template: + metadata: + labels: + app.kubernetes.io/name: xqueue + spec: + containers: + - name: xqueue + image: {{ DOCKER_REGISTRY }}{{ XQUEUE_DOCKER_IMAGE }} + ports: + - containerPort: 8040 + env: + - name: DJANGO_SETTINGS_MODULE + value: xqueue.tutor + volumeMounts: + - mountPath: /openedx/xqueue/xqueue/tutor.py + name: settings + subPath: tutor.py + - name: xqueue-consumer + image: {{ DOCKER_REGISTRY }}{{ XQUEUE_DOCKER_IMAGE }} + command: ["sh", "-e", "-c"] + args: ["while true; do echo 'running consumers'; ./manage.py run_consumer; sleep 10; done"] + env: + - name: DJANGO_SETTINGS_MODULE + value: xqueue.tutor + volumeMounts: + - mountPath: /openedx/xqueue/xqueue/tutor.py + name: settings + subPath: tutor.py + volumes: + - name: settings + configMap: + name: xqueue-settings \ No newline at end of file diff --git a/plugins/xqueue/tutorxqueue/patches/k8s-services b/plugins/xqueue/tutorxqueue/patches/k8s-services new file mode 100644 index 0000000..d4a227e --- /dev/null +++ b/plugins/xqueue/tutorxqueue/patches/k8s-services @@ -0,0 +1,12 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: xqueue +spec: + type: NodePort + ports: + - port: 8040 + protocol: TCP + selector: + app.kubernetes.io/name: xqueue \ No newline at end of file diff --git a/plugins/xqueue/tutorxqueue/patches/kustomization-configmapgenerator b/plugins/xqueue/tutorxqueue/patches/kustomization-configmapgenerator new file mode 100644 index 0000000..bab7196 --- /dev/null +++ b/plugins/xqueue/tutorxqueue/patches/kustomization-configmapgenerator @@ -0,0 +1,3 @@ +- name: xqueue-settings + files: + - apps/xqueue/settings/tutor.py \ No newline at end of file diff --git a/plugins/xqueue/tutorxqueue/patches/local-docker-compose-services b/plugins/xqueue/tutorxqueue/patches/local-docker-compose-services new file mode 100644 index 0000000..5424688 --- /dev/null +++ b/plugins/xqueue/tutorxqueue/patches/local-docker-compose-services @@ -0,0 +1,24 @@ +############# Xqueue: external grading of Open edX problems +xqueue: + image: {{ DOCKER_REGISTRY }}{{ XQUEUE_DOCKER_IMAGE }} + volumes: + - ../plugins/xqueue/apps/settings/tutor.py:/openedx/xqueue/xqueue/tutor.py + - ../../data/xqueue:/openedx/data + environment: + DJANGO_SETTINGS_MODULE: xqueue.tutor + restart: unless-stopped + {% if ACTIVATE_MYSQL %}depends_on: + - mysql{% endif %} + +xqueue_consumer: + image: {{ DOCKER_REGISTRY }}{{ XQUEUE_DOCKER_IMAGE }} + volumes: + - ../plugins/xqueue/apps/settings/tutor.py:/openedx/xqueue/xqueue/tutor.py + - ../../data/xqueue:/openedx/data + environment: + DJANGO_SETTINGS_MODULE: xqueue.tutor + restart: unless-stopped + entrypoint: ["sh", "-e", "-c"] + command: ["while true; do echo 'running consumers'; ./manage.py run_consumer; sleep 10; done"] + {% if ACTIVATE_MYSQL %}depends_on: + - mysql{% endif %} \ No newline at end of file diff --git a/plugins/xqueue/tutorxqueue/patches/openedx-auth b/plugins/xqueue/tutorxqueue/patches/openedx-auth new file mode 100644 index 0000000..e4704c8 --- /dev/null +++ b/plugins/xqueue/tutorxqueue/patches/openedx-auth @@ -0,0 +1,7 @@ +"XQUEUE_INTERFACE": { + "django_auth": { + "username": "{{ XQUEUE_AUTH_USERNAME }}", + "password": "{{ XQUEUE_AUTH_PASSWORD }}" + }, + "url": "http://xqueue:8040" +} \ No newline at end of file diff --git a/plugins/xqueue/tutorxqueue/plugin.py b/plugins/xqueue/tutorxqueue/plugin.py new file mode 100644 index 0000000..fc77962 --- /dev/null +++ b/plugins/xqueue/tutorxqueue/plugin.py @@ -0,0 +1,35 @@ +from glob import glob +import os + +HERE = os.path.abspath(os.path.dirname(__file__)) + +config = { + "add": { + "AUTH_PASSWORD": "{{ 8|random_string }}", + "MYSQL_PASSWORD": "{{ 8|random_string }}", + "SECRET_KEY": "{{ 24|random_string }}", + }, + "defaults": { + "DOCKER_IMAGE": "overhangio/openedx-xqueue:{{ TUTOR_VERSION }}", + "AUTH_USERNAME": "lms", + "MYSQL_DATABASE": "xqueue", + "MYSQL_USERNAME": "xqueue", + }, +} + +templates = os.path.join(HERE, "templates") +hooks = { + "init": ["mysql-client", "xqueue"], + "build-image": {"xqueue": "{{ XQUEUE_DOCKER_IMAGE }}"}, + "remote-image": {"xqueue": "{{ XQUEUE_DOCKER_IMAGE }}"}, +} + + +def patches(): + all_patches = {} + for path in glob(os.path.join(HERE, "patches", "*")): + with open(path) as patch_file: + name = os.path.basename(path) + content = patch_file.read() + all_patches[name] = content + return all_patches diff --git a/tutor/templates/apps/xqueue/settings/tutor.py b/plugins/xqueue/tutorxqueue/templates/xqueue/apps/xqueue/settings/tutor.py similarity index 100% rename from tutor/templates/apps/xqueue/settings/tutor.py rename to plugins/xqueue/tutorxqueue/templates/xqueue/apps/xqueue/settings/tutor.py diff --git a/tutor/templates/build/xqueue/Dockerfile b/plugins/xqueue/tutorxqueue/templates/xqueue/build/xqueue/Dockerfile similarity index 100% rename from tutor/templates/build/xqueue/Dockerfile rename to plugins/xqueue/tutorxqueue/templates/xqueue/build/xqueue/Dockerfile diff --git a/plugins/xqueue/tutorxqueue/templates/xqueue/hooks/mysql-client/init b/plugins/xqueue/tutorxqueue/templates/xqueue/hooks/mysql-client/init new file mode 100644 index 0000000..9731e4b --- /dev/null +++ b/plugins/xqueue/tutorxqueue/templates/xqueue/hooks/mysql-client/init @@ -0,0 +1,2 @@ +mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" --port {{ MYSQL_PORT }} -e 'CREATE DATABASE IF NOT EXISTS {{ XQUEUE_MYSQL_DATABASE }};' +mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" --port {{ MYSQL_PORT }} -e 'GRANT ALL ON {{ XQUEUE_MYSQL_DATABASE }}.* TO "{{ XQUEUE_MYSQL_USERNAME }}"@"%" IDENTIFIED BY "{{ XQUEUE_MYSQL_PASSWORD }}";' diff --git a/tutor/templates/hooks/xqueue/init b/plugins/xqueue/tutorxqueue/templates/xqueue/hooks/xqueue/init similarity index 100% rename from tutor/templates/hooks/xqueue/init rename to plugins/xqueue/tutorxqueue/templates/xqueue/hooks/xqueue/init diff --git a/requirements/plugins.txt b/requirements/plugins.txt index 2fe4c42..94727f0 100644 --- a/requirements/plugins.txt +++ b/requirements/plugins.txt @@ -1 +1,2 @@ -e ./plugins/minio +-e ./plugins/xqueue diff --git a/tests/test_env.py b/tests/test_env.py index e2b12ad..cd52abc 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -1,3 +1,4 @@ +import os import tempfile import unittest import unittest.mock @@ -8,6 +9,9 @@ from tutor import exceptions class EnvTests(unittest.TestCase): + def setUp(self): + env.Renderer.reset() + def test_walk_templates(self): templates = list(env.walk_templates("local")) self.assertIn("local/docker-compose.yml", templates) @@ -52,6 +56,10 @@ class EnvTests(unittest.TestCase): defaults = tutor_config.load_defaults() with tempfile.TemporaryDirectory() as root: env.render_full(root, defaults) + self.assertTrue(os.path.exists(os.path.join(root, "env", "version"))) + self.assertTrue( + os.path.exists(os.path.join(root, "env", "local", "docker-compose.yml")) + ) def test_render_full_with_https(self): defaults = tutor_config.load_defaults() @@ -77,3 +85,41 @@ class EnvTests(unittest.TestCase): {}, '{{ patch("location", separator=",\n", suffix=",") }}' ) self.assertEqual("abcd,\nefgh,", rendered) + + def test_plugin_templates(self): + with tempfile.TemporaryDirectory() as plugin_templates: + # Create two templates + os.makedirs(os.path.join(plugin_templates, "plugin1", "apps")) + with open( + os.path.join(plugin_templates, "plugin1", "unrendered.txt"), "w" + ) as f: + f.write("This file should not be rendered") + with open( + os.path.join(plugin_templates, "plugin1", "apps", "rendered.txt"), "w" + ) as f: + f.write("Hello my ID is {{ ID }}") + + # Create configuration + config = {"ID": "abcd"} + + # Create a single plugin + with unittest.mock.patch.object( + env.plugins, + "iter_templates", + return_value=[("plugin1", plugin_templates)], + ): + with tempfile.TemporaryDirectory() as root: + # Render plugin templates + env.save_plugin_templates("plugin1", plugin_templates, root, config) + + # Check that plugin template was rendered + dst_unrendered = os.path.join( + root, "env", "plugins", "plugin1", "unrendered.txt" + ) + dst_rendered = os.path.join( + root, "env", "plugins", "plugin1", "apps", "rendered.txt" + ) + self.assertFalse(os.path.exists(dst_unrendered)) + self.assertTrue(os.path.exists(dst_rendered)) + with open(dst_rendered) as f: + self.assertEqual("Hello my ID is abcd", f.read()) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 82f590a..4bea28c 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -155,7 +155,7 @@ class PluginsTests(unittest.TestCase): plugins.Plugins, "iter_enabled", return_value=[("plugin1", plugin1)] ): self.assertEqual( - [("plugin1", "myclient")], list(plugins.iter_hooks({}, "init")) + [("plugin1", ["myclient"])], list(plugins.iter_hooks({}, "init")) ) def test_iter_templates(self): diff --git a/tutor/commands/images.py b/tutor/commands/images.py index 9e51cdb..cc95a95 100644 --- a/tutor/commands/images.py +++ b/tutor/commands/images.py @@ -2,9 +2,9 @@ import click from .. import config as tutor_config from .. import env as tutor_env -from .. import fmt +from .. import images from .. import opts -from .. import utils +from .. import plugins @click.group(name="images", short_help="Manage docker images") @@ -12,30 +12,12 @@ def images_command(): pass -OPENEDX_IMAGES = ["openedx", "forum", "notes", "xqueue", "android"] -VENDOR_IMAGES = [ - "elasticsearch", - "memcached", - "mongodb", - "mysql", - "nginx", - "rabbitmq", - "smtp", -] -argument_openedx_image = click.argument( - "image", type=click.Choice(["all"] + OPENEDX_IMAGES) -) -argument_image = click.argument( - "image", type=click.Choice(["all"] + OPENEDX_IMAGES + VENDOR_IMAGES) -) - - @click.command( short_help="Build docker images", help="Build the docker images necessary for an Open edX platform.", ) @opts.root -@argument_openedx_image +@click.argument("image") @click.option( "--no-cache", is_flag=True, help="Do not use cache when building the image" ) @@ -47,36 +29,67 @@ argument_image = click.argument( ) def build(root, image, no_cache, build_arg): config = tutor_config.load(root) - for img in openedx_image_names(config, image): - tag = get_tag(config, img) - fmt.echo_info("Building image {}".format(tag)) - command = ["build", "-t", tag, tutor_env.pathjoin(root, "build", img)] - if no_cache: - command.append("--no-cache") - for arg in build_arg: - command += ["--build-arg", arg] - utils.docker(*command) + + # Build base images + for img in openedx_image_names(config): + if image in [img, "all"]: + tag = get_tag(config, img) + images.build( + tutor_env.pathjoin(root, "build", img), + tag, + no_cache=no_cache, + build_args=build_arg, + ) + + # Build plugin images + for plugin, hook in plugins.iter_hooks(config, "build-image"): + for img, tag in hook.items(): + if image in [img, "all"]: + tag = tutor_env.render_str(config, tag) + images.build( + tutor_env.pathjoin(root, "plugins", plugin, "build", img), + tag, + no_cache=no_cache, + build_args=build_arg, + ) @click.command(short_help="Pull images from the Docker registry") @opts.root -@argument_image +@click.argument("image") def pull(root, image): config = tutor_config.load(root) - for img in image_names(config, image): - tag = get_tag(config, img) - fmt.echo_info("Pulling image {}".format(tag)) - utils.execute("docker", "pull", tag) + # Pull base images + for img in image_names(config): + if image in [img, "all"]: + tag = get_tag(config, img) + images.pull(tag) + + # Pull plugin images + for _plugin, hook in plugins.iter_hooks(config, "remote-image"): + for img, tag in hook.items(): + if image in [img, "all"]: + tag = config["DOCKER_REGISTRY"] + tutor_env.render_str(config, tag) + images.pull(tag) @click.command(short_help="Push images to the Docker registry") @opts.root -@argument_openedx_image +@click.argument("image") def push(root, image): config = tutor_config.load(root) - for tag in openedx_image_tags(config, image): - fmt.echo_info("Pushing image {}".format(tag)) - utils.execute("docker", "push", tag) + # Push base images + for img in openedx_image_names(config): + if image in [img, "all"]: + tag = get_tag(config, img) + images.push(tag) + + # Push plugin images + for _plugin, hook in plugins.iter_hooks(config, "remote-image"): + for img, tag in hook.items(): + if image in [img, "all"]: + tag = config["DOCKER_REGISTRY"] + tutor_env.render_str(config, tag) + images.push(tag) def get_tag(config, name): @@ -84,41 +97,38 @@ def get_tag(config, name): return "{registry}{image}".format(registry=config["DOCKER_REGISTRY"], image=image) -def image_names(config, image): - return openedx_image_names(config, image) + vendor_image_names(config, image) +def image_names(config): + return openedx_image_names(config) + vendor_image_names(config) -def openedx_image_tags(config, image): - for img in openedx_image_names(config, image): - yield get_tag(config, img) +def openedx_image_names(config): + openedx_images = ["openedx", "forum", "notes", "android"] + if not config["ACTIVATE_NOTES"]: + openedx_images.remove("notes") + return openedx_images -def openedx_image_names(config, image): - if image == "all": - images = OPENEDX_IMAGES[:] - if not config["ACTIVATE_XQUEUE"]: - images.remove("xqueue") - if not config["ACTIVATE_NOTES"]: - images.remove("notes") - return images - return [image] - - -def vendor_image_names(config, image): - if image == "all": - images = VENDOR_IMAGES[:] - if not config["ACTIVATE_ELASTICSEARCH"]: - images.remove("elasticsearch") - if not config["ACTIVATE_MEMCACHED"]: - images.remove("memcached") - if not config["ACTIVATE_MONGODB"]: - images.remove("mongodb") - if not config["ACTIVATE_MYSQL"]: - images.remove("mysql") - if not config["ACTIVATE_RABBITMQ"]: - images.remove("rabbitmq") - return images - return [image] +def vendor_image_names(config): + vendor_images = [ + "elasticsearch", + "memcached", + "mongodb", + "mysql", + "nginx", + "rabbitmq", + "smtp", + ] + if not config["ACTIVATE_ELASTICSEARCH"]: + vendor_images.remove("elasticsearch") + if not config["ACTIVATE_MEMCACHED"]: + vendor_images.remove("memcached") + if not config["ACTIVATE_MONGODB"]: + vendor_images.remove("mongodb") + if not config["ACTIVATE_MYSQL"]: + vendor_images.remove("mysql") + if not config["ACTIVATE_RABBITMQ"]: + vendor_images.remove("rabbitmq") + return vendor_images images_command.add_command(build) diff --git a/tutor/config.py b/tutor/config.py index 3112016..dcee7fd 100644 --- a/tutor/config.py +++ b/tutor/config.py @@ -98,9 +98,6 @@ def load_required(config, defaults): "NOTES_MYSQL_PASSWORD", "NOTES_SECRET_KEY", "NOTES_OAUTH2_SECRET", - "XQUEUE_AUTH_PASSWORD", - "XQUEUE_MYSQL_PASSWORD", - "XQUEUE_SECRET_KEY", "ANDROID_OAUTH2_SECRET", "ID", ]: @@ -140,6 +137,10 @@ def upgrade_obsolete(config): config["OPENEDX_MYSQL_DATABASE"] = config.pop("MYSQL_DATABASE") if "MYSQL_USERNAME" in config: config["OPENEDX_MYSQL_USERNAME"] = config.pop("MYSQL_USERNAME") + if "ACTIVATE_XQUEUE" in config: + if config["ACTIVATE_XQUEUE"]: + plugins.enable(config, "xqueue") + config.pop("ACTIVATE_XQUEUE") def convert_json2yml(root): diff --git a/tutor/env.py b/tutor/env.py index 8a60376..ded3385 100644 --- a/tutor/env.py +++ b/tutor/env.py @@ -104,11 +104,27 @@ def render_full(root, config): """ for subdir in ["android", "apps", "k8s", "local", "webui"]: save_subdir(subdir, root, config) + for plugin, path in plugins.iter_templates(config): + save_plugin_templates(plugin, path, root, config) copy_subdir("build", root) save_file(VERSION_FILENAME, root, config) save_file("kustomization.yml", root, config) +def save_plugin_templates(plugin, plugin_path, root, config): + """ + Save plugin templates to plugins//*. + Only the "apps" and "build" subfolders are rendered. + TODO we should delete this folder when the plugin is disabled. + """ + for subdir in ["apps", "build"]: + path = os.path.join(plugin_path, plugin, subdir) + for src in walk_templates(path, root=plugin_path): + dst = pathjoin(root, "plugins", src) + rendered = render_file(config, src) + write_to(rendered, dst) + + def save_subdir(subdir, root, config): """ Render the templates located in `subdir` and store them with the same @@ -124,9 +140,13 @@ def save_file(path, root, config): """ dst = pathjoin(root, path) rendered = render_file(config, path) - utils.ensure_file_directory_exists(dst) - with open(dst, "w") as of: - of.write(rendered) + write_to(rendered, dst) + + +def write_to(content, path): + utils.ensure_file_directory_exists(path) + with open(path, "w") as of: + of.write(content) def render_file(config, *path): @@ -223,7 +243,7 @@ def read(*path): return fi.read() -def walk_templates(subdir): +def walk_templates(subdir, root=TEMPLATES_ROOT): """ Iterate on the template files from `templates/`. @@ -234,7 +254,7 @@ def walk_templates(subdir): if not is_part_of_env(dirpath): continue for filename in filenames: - path = os.path.join(os.path.relpath(dirpath, TEMPLATES_ROOT), filename) + path = os.path.join(os.path.relpath(dirpath, root), filename) if is_part_of_env(path): yield path diff --git a/tutor/images.py b/tutor/images.py new file mode 100644 index 0000000..9129d16 --- /dev/null +++ b/tutor/images.py @@ -0,0 +1,21 @@ +from . import fmt +from . import utils + + +def build(path, tag, no_cache=False, build_args=None): + fmt.echo_info("Building image {}".format(tag)) + command = ["build", "-t", tag, path] + build_args = build_args or {} + if no_cache: + command.append("--no-cache") + for arg in build_args: + command += ["--build-arg", arg] + utils.docker(*command) + +def pull(tag): + fmt.echo_info("Pulling image {}".format(tag)) + utils.execute("docker", "pull", tag) + +def push(tag): + fmt.echo_info("Pushing image {}".format(tag)) + utils.execute("docker", "push", tag) \ No newline at end of file diff --git a/tutor/interactive.py b/tutor/interactive.py index dd657b0..c0646a7 100644 --- a/tutor/interactive.py +++ b/tutor/interactive.py @@ -134,12 +134,6 @@ def ask_questions(config, defaults): config, defaults, ) - ask_bool( - "Activate Xqueue for external grader services (https://github.com/edx/xqueue)?", - "ACTIVATE_XQUEUE", - config, - defaults, - ) def ask(question, key, config, defaults): diff --git a/tutor/plugins.py b/tutor/plugins.py index e2be641..532215e 100644 --- a/tutor/plugins.py +++ b/tutor/plugins.py @@ -99,9 +99,7 @@ class Plugins: yield plugin, plugin_patches[plugin] def iter_hooks(self, hook_name): - for plugin_name, services in self.hooks.get(hook_name, {}).items(): - for service in services: - yield plugin_name, service + yield from self.hooks.get(hook_name, {}).items() def iter_templates(self): yield from self.templates.items() diff --git a/tutor/scripts.py b/tutor/scripts.py index 150dbf4..7e6f2aa 100644 --- a/tutor/scripts.py +++ b/tutor/scripts.py @@ -37,15 +37,16 @@ class BaseRunner: def initialise(runner): fmt.echo_info("Initialising all services...") runner.run("mysql-client", "hooks", "mysql-client", "init") - for service in ["lms", "cms", "forum", "notes", "xqueue"]: + for service in ["lms", "cms", "forum", "notes"]: if runner.is_activated(service): fmt.echo_info("Initialising {}...".format(service)) runner.run(service, "hooks", service, "init") - for plugin_name, service in runner.iter_plugin_hooks("init"): - fmt.echo_info( - "Plugin {}: running init for service {}...".format(plugin_name, service) - ) - runner.run(service, plugin_name, "hooks", service, "init") + for plugin_name, hook in runner.iter_plugin_hooks("init"): + for service in hook: + fmt.echo_info( + "Plugin {}: running init for service {}...".format(plugin_name, service) + ) + runner.run(service, plugin_name, "hooks", service, "init") fmt.echo_info("All services initialised.") diff --git a/tutor/templates/apps/openedx/config/partials/auth.json b/tutor/templates/apps/openedx/config/partials/auth.json index 4e006be..99a58d9 100644 --- a/tutor/templates/apps/openedx/config/partials/auth.json +++ b/tutor/templates/apps/openedx/config/partials/auth.json @@ -2,13 +2,6 @@ "SECRET_KEY": "{{ SECRET_KEY }}", "AWS_ACCESS_KEY_ID": "{{ OPENEDX_AWS_ACCESS_KEY }}", "AWS_SECRET_ACCESS_KEY": "{{ OPENEDX_AWS_SECRET_ACCESS_KEY }}", - "XQUEUE_INTERFACE": { - "django_auth": { - "username": "{{ XQUEUE_AUTH_USERNAME }}", - "password": "{{ XQUEUE_AUTH_PASSWORD }}" - }, - "url": "http://xqueue:8040" - }, {{ patch("openedx-auth", separator=",\n", suffix=",")|indent(2) }} "CONTENTSTORE": { "ENGINE": "xmodule.contentstore.mongo.MongoContentStore", diff --git a/tutor/templates/config.yml b/tutor/templates/config.yml index 826623c..52eac42 100644 --- a/tutor/templates/config.yml +++ b/tutor/templates/config.yml @@ -6,9 +6,6 @@ OPENEDX_MYSQL_PASSWORD: "{{ 8|random_string }}" NOTES_MYSQL_PASSWORD: "{{ 8|random_string }}" NOTES_SECRET_KEY: "{{ 24|random_string }}" NOTES_OAUTH2_SECRET: "{{ 24|random_string }}" -XQUEUE_AUTH_PASSWORD: "{{ 8|random_string }}" -XQUEUE_MYSQL_PASSWORD: "{{ 8|random_string }}" -XQUEUE_SECRET_KEY: "{{ 24|random_string }}" ANDROID_OAUTH2_SECRET: "{{ 24|random_string }}" ID: "{{ 24|random_string }}" @@ -27,7 +24,6 @@ ACTIVATE_MYSQL: true ACTIVATE_NOTES: false ACTIVATE_RABBITMQ: true ACTIVATE_SMTP: true -ACTIVATE_XQUEUE: false CMS_HOST: "studio.{{ LMS_HOST }}" CONTACT_EMAIL: "contact@{{ LMS_HOST }}" OPENEDX_AWS_ACCESS_KEY: "" @@ -39,7 +35,6 @@ DOCKER_IMAGE_OPENEDX: "overhangio/openedx:{{ TUTOR_VERSION }}" DOCKER_IMAGE_ANDROID: "overhangio/openedx-android:{{ TUTOR_VERSION }}" DOCKER_IMAGE_FORUM: "overhangio/openedx-forum:{{ TUTOR_VERSION }}" DOCKER_IMAGE_NOTES: "overhangio/openedx-notes:{{ TUTOR_VERSION }}" -DOCKER_IMAGE_XQUEUE: "overhangio/openedx-xqueue:{{ TUTOR_VERSION }}" DOCKER_IMAGE_MEMCACHED: "memcached:1.4.38" DOCKER_IMAGE_MONGODB: "mongo:3.2.16" DOCKER_IMAGE_MYSQL: "mysql:5.6.36" @@ -80,6 +75,3 @@ SMTP_PORT: 25 SMTP_USERNAME: "" SMTP_PASSWORD: "" WEB_PROXY: false -XQUEUE_AUTH_USERNAME: "lms" -XQUEUE_MYSQL_DATABASE: "xqueue" -XQUEUE_MYSQL_USERNAME: "xqueue" diff --git a/tutor/templates/hooks/mysql-client/init b/tutor/templates/hooks/mysql-client/init index 813be0a..664549a 100644 --- a/tutor/templates/hooks/mysql-client/init +++ b/tutor/templates/hooks/mysql-client/init @@ -21,8 +21,3 @@ mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" - mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" --port {{ MYSQL_PORT }} -e 'CREATE DATABASE IF NOT EXISTS {{ NOTES_MYSQL_DATABASE }};' mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" --port {{ MYSQL_PORT }} -e 'GRANT ALL ON {{ NOTES_MYSQL_DATABASE }}.* TO "{{ NOTES_MYSQL_USERNAME }}"@"%" IDENTIFIED BY "{{ NOTES_MYSQL_PASSWORD }}";' {% endif %} - -{% if ACTIVATE_XQUEUE %} -mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" --port {{ MYSQL_PORT }} -e 'CREATE DATABASE IF NOT EXISTS {{ XQUEUE_MYSQL_DATABASE }};' -mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" --port {{ MYSQL_PORT }} -e 'GRANT ALL ON {{ XQUEUE_MYSQL_DATABASE }}.* TO "{{ XQUEUE_MYSQL_USERNAME }}"@"%" IDENTIFIED BY "{{ XQUEUE_MYSQL_PASSWORD }}";' -{% endif %} diff --git a/tutor/templates/k8s/deployments.yml b/tutor/templates/k8s/deployments.yml index 3168dcb..8d1d263 100644 --- a/tutor/templates/k8s/deployments.yml +++ b/tutor/templates/k8s/deployments.yml @@ -501,49 +501,4 @@ spec: persistentVolumeClaim: claimName: rabbitmq {% endif %} -{% if ACTIVATE_XQUEUE %} ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: xqueue - labels: - app.kubernetes.io/name: xqueue -spec: - selector: - matchLabels: - app.kubernetes.io/name: xqueue - template: - metadata: - labels: - app.kubernetes.io/name: xqueue - spec: - containers: - - name: xqueue - image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_XQUEUE }} - ports: - - containerPort: 8040 - env: - - name: DJANGO_SETTINGS_MODULE - value: xqueue.tutor - volumeMounts: - - mountPath: /openedx/xqueue/xqueue/tutor.py - name: settings - subPath: tutor.py - - name: xqueue-consumer - image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_XQUEUE }} - command: ["sh", "-e", "-c"] - args: ["while true; do echo 'running consumers'; ./manage.py run_consumer; sleep 10; done"] - env: - - name: DJANGO_SETTINGS_MODULE - value: xqueue.tutor - volumeMounts: - - mountPath: /openedx/xqueue/xqueue/tutor.py - name: settings - subPath: tutor.py - volumes: - - name: settings - configMap: - name: xqueue-settings -{% endif %} {{ patch("k8s-deployments") }} diff --git a/tutor/templates/k8s/services.yml b/tutor/templates/k8s/services.yml index 276910d..5fb1b16 100644 --- a/tutor/templates/k8s/services.yml +++ b/tutor/templates/k8s/services.yml @@ -148,18 +148,4 @@ spec: selector: app.kubernetes.io/name: smtp {% endif %} -{% if ACTIVATE_XQUEUE %} ---- -apiVersion: v1 -kind: Service -metadata: - name: xqueue -spec: - type: NodePort - ports: - - port: 8040 - protocol: TCP - selector: - app.kubernetes.io/name: xqueue -{% endif %} {{ patch("k8s-services") }} \ No newline at end of file diff --git a/tutor/templates/kustomization.yml b/tutor/templates/kustomization.yml index cb965a1..c1aeca3 100644 --- a/tutor/templates/kustomization.yml +++ b/tutor/templates/kustomization.yml @@ -37,6 +37,4 @@ configMapGenerator: {% if ACTIVATE_NOTES %}- name: notes-settings files: - apps/notes/settings/tutor.py{% endif %} -{% if ACTIVATE_XQUEUE %}- name: xqueue-settings - files: - - apps/xqueue/settings/tutor.py{% endif %} \ No newline at end of file +{{ patch("kustomization-configmapgenerator") }} diff --git a/tutor/templates/local/docker-compose.yml b/tutor/templates/local/docker-compose.yml index e704345..a6a78dc 100644 --- a/tutor/templates/local/docker-compose.yml +++ b/tutor/templates/local/docker-compose.yml @@ -207,32 +207,5 @@ services: {% if ACTIVATE_MYSQL %}depends_on: - mysql{% endif %} {% endif %} - - {% if ACTIVATE_XQUEUE %} - ############# Xqueue: external grading of Open edX problems - xqueue: - image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_XQUEUE }} - volumes: - - ../apps/xqueue/settings/tutor.py:/openedx/xqueue/xqueue/tutor.py - - ../../data/xqueue:/openedx/data - environment: - DJANGO_SETTINGS_MODULE: xqueue.tutor - restart: unless-stopped - {% if ACTIVATE_MYSQL %}depends_on: - - mysql{% endif %} - - xqueue_consumer: - image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_XQUEUE }} - volumes: - - ../apps/xqueue/settings/tutor.py:/openedx/xqueue/xqueue/tutor.py - - ../../data/xqueue:/openedx/data - environment: - DJANGO_SETTINGS_MODULE: xqueue.tutor - restart: unless-stopped - entrypoint: ["sh", "-e", "-c"] - command: ["while true; do echo 'running consumers'; ./manage.py run_consumer; sleep 10; done"] - {% if ACTIVATE_MYSQL %}depends_on: - - mysql{% endif %} - {% endif %} {{ patch("local-docker-compose-services")|indent(2) }} \ No newline at end of file