christianlight
/
tutor
mirror of https://github.com/ChristianLight/tutor.git
7
0
Fork 0
Code Issues Releases Activity

Move tutor-minio to dedicated plugin repo

This commit is contained in:
Régis Behmo 2019-08-20 17:46:53 +02:00
parent c9adf68ba3
commit 7790028cf7
31 changed files with 31 additions and 350 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -4,6 +4,7 @@ Note: Breaking changes between versions are indicated by "💥".
## Latest
- [Improvement] Move minio plugin outside of the tutor repo
- [Bugfix/Improvement] Add all plugins (with data) into binary bundle (#242)
## 3.6.2 (2019-08-07)

2
Makefile
View File

@ -13,7 +13,6 @@ package: ## Build a package ready to upload to pypi
python3 setup.py sdist
package-plugins: ## Build packages for each plugin
cd plugins/minio && python3 setup.py sdist --dist-dir=../../dist/
cd plugins/notes && python3 setup.py sdist --dist-dir=../../dist/
cd plugins/xqueue && python3 setup.py sdist --dist-dir=../../dist/
@ -31,7 +30,6 @@ test-unit-core: ## Run unit tests on core
python3 -m unittest discover tests
test-unit-plugins: ## Run unit tests on plugins
python3 -m unittest discover plugins/minio/tests
python3 -m unittest discover plugins/xqueue/tests
test-packages: package package-plugins ## Test that packages can be uploaded to pypi

6
docs/plugins.rst
View File

@ -180,7 +180,7 @@ Existing plugins
----------------
- `Course discovery <https://pypi.org/project/tutor-discovery>`__: Deploy an API for interacting with your course catalog
- `Ecommerce <https://pypi.org/project/tutor-minio>`__: Sell courses and products on your Open edX platform
- `Ecommerce <https://pypi.org/project/tutor-ecommerce>`__: Sell courses and products on your Open edX platform
- `Figures <https://pypi.org/project/tutor-figures>`__: Visualize daily stats about course engagement
- `MinIO <https://github.com/overhangio/tutor/tree/master/plugins/minio>`__: S3 emulator for object storage and scalable Open edX deployment.
- `Xqueue <https://github.com/overhangio/tutor/tree/master/plugins/xqueue>`__: for external grading
- `MinIO <https://pypi.org/project/tutor-minio>`__: S3 emulator for object storage and scalable Open edX deployment.
- `Xqueue <https://pypi.org/project/tutor-xqueue>`__: for external grading

2
plugins/minio/MANIFEST.in
View File

@ -1,2 +0,0 @@
recursive-include tutorminio/patches *
recursive-include tutorminio/templates *

45
plugins/minio/README.rst
View File

@ -1,45 +0,0 @@
Object storage for Open edX with `MinIO <https://www.minio.io/>`_
=================================================================
This is a plugin for `Tutor <https://docs.tutor.overhang.io>`_ that provides S3-like object storage for Open edX platforms. It's S3, but without the dependency on AWS. This is achieved thanks to `MinIO <https://www.minio.io/>`_, an open source project that provides object storage with an API compatible with S3.
In particular, this plugin is essential for `Kubernetes deployment <https://docs.tutor.overhang.io/k8s.html>`_.
Installation
------------
The plugin is currently bundled with the `binary releases of Tutor <https://github.com/overhangio/tutor/releases>`_. If you have installed Tutor from source, you will have to install this plugin from source, too::
pip install tutor-minio
Then, to enable this plugin, run::
tutor plugins enable minio
Configuration
-------------
- ``MINIO_BUCKET_NAME`` (default: ``"openedx"``)
- ``MINIO_FILE_UPLOAD_BUCKET_NAME`` (default: ``"openedxuploads"``)
- ``MINIO_COURSE_IMPORT_EXPORT_BUCKET`` (default: ``"openedxcourseimportexport"``)
- ``MINIO_HOST`` (default: ``"minio.{{ LMS_HOST }}"``)
- ``MINIO_DOCKER_REGISTRY`` (default: ``"{{ DOCKER_REGISTRY }}"``)
- ``MINIO_DOCKER_IMAGE_CLIENT`` (default: ``"minio/mc:RELEASE.2019-05-23T01-33-27Z"``)
- ``MINIO_DOCKER_IMAGE_SERVER`` (default: ``"minio/minio:RELEASE.2019-05-23T00-29-34Z"``)
These values can be modified with ``tutor config save --set PARAM_NAME=VALUE`` commands.
DNS records
-----------
It is assumed that the ``MINIO_HOST`` DNS record points to your server. When running MinIO on your laptop, you should point your services to ``minio.localhost``::
tutor config save --set MINIO_HOST=minio.localhost
Web UI
------
The MinIO web UI can be accessed at http://<MINIO_HOST>. The credentials for accessing the UI can be obtained with::
tutor config printvalue OPENEDX_AWS_ACCESS_KEY
tutor config printvalue OPENEDX_AWS_SECRET_ACCESS_KEY

40
plugins/minio/setup.py
View File

@ -1,40 +0,0 @@
import io
import os
from setuptools import setup, find_packages
here = os.path.abspath(os.path.dirname(__file__))
with io.open(os.path.join(here, "README.rst"), "rt", encoding="utf8") as f:
readme = f.read()
setup(
name="tutor-minio",
version="0.1.1",
url="https://docs.tutor.overhang.io/",
project_urls={
"Documentation": "https://docs.tutor.overhang.io/",
"Code": "https://github.com/overhangio/tutor/tree/master/plugins/minio",
"Issue tracker": "https://github.com/overhangio/tutor/issues",
"Community": "https://discuss.overhang.io",
},
license="AGPLv3",
author="Overhang.io",
author_email="contact@overhang.io",
description="A Tutor plugin for object storage in MinIO",
long_description=readme,
packages=find_packages(exclude=["tests*"]),
include_package_data=True,
python_requires=">=3.5",
entry_points={"tutor.plugin.v0": ["minio = tutorminio.plugin"]},
classifiers=[
"Development Status :: 3 - Alpha",
"Intended Audience :: Developers",
"License :: OSI Approved :: GNU Affero General Public License v3",
"Operating System :: OS Independent",
"Programming Language :: Python",
"Programming Language :: Python :: 3.5",
"Programming Language :: Python :: 3.6",
"Programming Language :: Python :: 3.7",
],
)

0
plugins/minio/tests/__init__.py
View File

12
plugins/minio/tests/test_plugin.py
View File

@ -1,12 +0,0 @@
import unittest
from tutorminio import plugin
class PluginTests(unittest.TestCase):
def test_patches(self):
patches = dict(plugin.patches())
self.assertIn("local-docker-compose-services", patches)
self.assertTrue(
patches["local-docker-compose-services"].startswith("# MinIO\n")
)

0
plugins/minio/tutorminio/__init__.py
View File

2
plugins/minio/tutorminio/patches/cms-env
View File

@ -1,2 +0,0 @@
"FILE_UPLOAD_STORAGE_BUCKET_NAME": "{{ MINIO_FILE_UPLOAD_BUCKET_NAME }}",
"COURSE_IMPORT_EXPORT_BUCKET": "{{ MINIO_COURSE_IMPORT_EXPORT_BUCKET }}"

2
plugins/minio/tutorminio/patches/https-create
View File

@ -1,2 +0,0 @@
# MinIO
certbot certonly --standalone -n --agree-tos -m admin@{{ LMS_HOST }} -d {{ MINIO_HOST }}

57
plugins/minio/tutorminio/patches/k8s-deployments
View File

@ -1,57 +0,0 @@
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: minio
labels:
app.kubernetes.io/name: minio
spec:
selector:
matchLabels:
app.kubernetes.io/name: minio
strategy:
type: Recreate
template:
metadata:
labels:
app.kubernetes.io/name: minio
spec:
containers:
- name: minio
image: {{ MINIO_DOCKER_REGISTRY }}{{ MINIO_DOCKER_IMAGE_SERVER }}
args: ["server", "--address", ":9000", "/data"]
env:
- name: MINIO_ACCESS_KEY
value: "{{ OPENEDX_AWS_ACCESS_KEY }}"
- name: MINIO_SECRET_KEY
value: "{{ OPENEDX_AWS_SECRET_ACCESS_KEY }}"
ports:
- containerPort: 9000
volumeMounts:
- mountPath: /data
name: data
volumes:
- name: data
persistentVolumeClaim:
claimName: minio
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: minio-client
labels:
app.kubernetes.io/name: minio-client
spec:
selector:
matchLabels:
app.kubernetes.io/name: minio-client
template:
metadata:
labels:
app.kubernetes.io/name: minio-client
spec:
containers:
- name: minio
image: {{ MINIO_DOCKER_REGISTRY }}{{ MINIO_DOCKER_IMAGE_CLIENT }}
command: ["sh", "-e", "-c"]
args: ["while true; do echo 'ready'; sleep 10; done"]

18
plugins/minio/tutorminio/patches/k8s-ingress-certificates
View File

@ -1,18 +0,0 @@
---
apiVersion: certmanager.k8s.io/v1alpha1
kind: Certificate
metadata:
name: {{ MINIO_HOST|replace(".", "-") }}
spec:
secretName: {{ MINIO_HOST }}-tls
issuerRef:
name: letsencrypt
commonName: {{ MINIO_HOST }}
dnsNames:
- {{ MINIO_HOST }}
acme:
config:
- http01:
ingress: web
domains:
- {{ MINIO_HOST }}

6
plugins/minio/tutorminio/patches/k8s-ingress-rules
View File

@ -1,6 +0,0 @@
- host: {{ MINIO_HOST }}
http:
paths:
- backend:
serviceName: nginx
servicePort: {% if ACTIVATE_HTTPS %}443{% else %}80{% endif %}

1
plugins/minio/tutorminio/patches/k8s-ingress-tls-hosts
View File

@ -1 +0,0 @@
- {{ MINIO_HOST }}

12
plugins/minio/tutorminio/patches/k8s-services
View File

@ -1,12 +0,0 @@
---
apiVersion: v1
kind: Service
metadata:
name: minio
spec:
type: NodePort
ports:
- port: 9000
protocol: TCP
selector:
app.kubernetes.io/name: minio

14
plugins/minio/tutorminio/patches/k8s-volumes
View File

@ -1,14 +0,0 @@
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: minio
labels:
app.kubernetes.io/component: volume
app.kubernetes.io/name: minio
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 5Gi

2
plugins/minio/tutorminio/patches/lms-env
View File

@ -1,2 +0,0 @@
"FILE_UPLOAD_STORAGE_BUCKET_NAME": "{{ MINIO_FILE_UPLOAD_BUCKET_NAME }}",
"COURSE_IMPORT_EXPORT_BUCKET": "{{ MINIO_COURSE_IMPORT_EXPORT_BUCKET }}"

1
plugins/minio/tutorminio/patches/local-docker-compose-nginx-aliases
View File

@ -1 +0,0 @@
"{{ MINIO_HOST }}"

17
plugins/minio/tutorminio/patches/local-docker-compose-services
View File

@ -1,17 +0,0 @@
# MinIO
minio:
image: {{ MINIO_DOCKER_REGISTRY }}{{ MINIO_DOCKER_IMAGE_SERVER }}
volumes:
- ../../data/minio:/data
environment:
MINIO_ACCESS_KEY: "{{ OPENEDX_AWS_ACCESS_KEY }}"
MINIO_SECRET_KEY: "{{ OPENEDX_AWS_SECRET_ACCESS_KEY }}"
command: server --address ":9000" /data
restart: unless-stopped
minio-client:
image: {{ MINIO_DOCKER_REGISTRY }}{{ MINIO_DOCKER_IMAGE_CLIENT }}
restart: "no"
entrypoint: sh
depends_on:
- minio

37
plugins/minio/tutorminio/patches/nginx-extra
View File

@ -1,37 +0,0 @@
# MinIO public service
upstream minio-backend {
server minio:9000 fail_timeout=0;
}
{% if ACTIVATE_HTTPS %}
server {
server_name {{ MINIO_HOST }};
listen 80;
return 301 https://$server_name$request_uri;
}
{% endif %}
server {
{% if ACTIVATE_HTTPS %}listen 443 {{ "" if WEB_PROXY else "ssl" }};{% else %}listen 80;{% endif %}
server_name minio.localhost {{ MINIO_HOST }};
{% if ACTIVATE_HTTPS and not WEB_PROXY %}
ssl_certificate /etc/letsencrypt/live/{{ MINIO_HOST }}/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/{{ MINIO_HOST }}/privkey.pem;
{% endif %}
# Disables server version feedback on pages and in headers
server_tokens off;
client_max_body_size 0;
location / {
{% if not WEB_PROXY %}
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-Port $server_port;
proxy_set_header X-Forwarded-For $remote_addr;
{% endif %}
proxy_set_header Host $http_host;
proxy_redirect off;
proxy_pass http://minio-backend;
}
}

2
plugins/minio/tutorminio/patches/openedx-auth
View File

@ -1,2 +0,0 @@
"AWS_STORAGE_BUCKET_NAME": "{{ MINIO_BUCKET_NAME }}",
"AWS_S3_CUSTOM_DOMAIN": ""

17
plugins/minio/tutorminio/patches/openedx-common-settings
View File

@ -1,17 +0,0 @@
AWS_S3_HOST = "{{ MINIO_HOST }}"
AWS_S3_USE_SSL = {{ "True" if ACTIVATE_HTTPS else "False" }}
AWS_S3_SECURE_URLS = {{ "True" if ACTIVATE_HTTPS else "False" }}
AWS_S3_CALLING_FORMAT = "boto.s3.connection.OrdinaryCallingFormat"
AWS_AUTO_CREATE_BUCKET = False # explicit is better than implicit
# Configuring boto is required for ora2 because ora2 does not read
# host/port/ssl settings from django. Hence this hack.
# http://docs.pythonboto.org/en/latest/boto_config_tut.html
import os
os.environ["AWS_CREDENTIAL_FILE"] = "/tmp/boto.cfg"
with open("/tmp/boto.cfg", "w") as f:
f.write("""[Boto]
is_secure = {{ "True" if ACTIVATE_HTTPS else "False" }}
[s3]
host = {{ MINIO_HOST }}
calling_format = boto.s3.connection.OrdinaryCallingFormat""")

1
plugins/minio/tutorminio/patches/openedx-lms-common-settings
View File

@ -1 +0,0 @@
ORA2_FILEUPLOAD_BACKEND = "s3"

40
plugins/minio/tutorminio/plugin.py
View File

@ -1,40 +0,0 @@
import os
from glob import glob
HERE = os.path.abspath(os.path.dirname(__file__))
config = {
"set": {
"OPENEDX_AWS_ACCESS_KEY": "openedx",
"OPENEDX_AWS_SECRET_ACCESS_KEY": "{{ 24|random_string }}",
},
"defaults": {
"BUCKET_NAME": "openedx",
"FILE_UPLOAD_BUCKET_NAME": "openedxuploads",
"COURSE_IMPORT_EXPORT_BUCKET": "openedxcourseimportexport",
"HOST": "minio.{{ LMS_HOST }}",
"DOCKER_REGISTRY": "{{ DOCKER_REGISTRY }}",
"DOCKER_IMAGE_CLIENT": "minio/mc:RELEASE.2019-05-23T01-33-27Z",
"DOCKER_IMAGE_SERVER": "minio/minio:RELEASE.2019-05-23T00-29-34Z",
},
}
templates = os.path.join(HERE, "templates")
hooks = {
"pre-init": ["minio-client"],
"remote-image": {
"minio-server": "{{ MINIO_DOCKER_IMAGE_SERVER }}",
"minio-client": "{{ MINIO_DOCKER_IMAGE_CLIENT }}",
},
}
def patches():
all_patches = {}
for path in glob(os.path.join(HERE, "patches", "*")):
with open(path) as patch_file:
name = os.path.basename(path)
content = patch_file.read()
all_patches[name] = content
return all_patches

2
plugins/minio/tutorminio/templates/minio/hooks/minio-client/pre-init
View File

@ -1,2 +0,0 @@
mc config host add minio http://minio:9000 {{ OPENEDX_AWS_ACCESS_KEY }} {{ OPENEDX_AWS_SECRET_ACCESS_KEY }} --api s3v4
mc mb --ignore-existing minio/{{ MINIO_BUCKET_NAME }} minio/{{ MINIO_FILE_UPLOAD_BUCKET_NAME }} minio/{{ MINIO_COURSE_IMPORT_EXPORT_BUCKET }}

2
plugins/notes/setup.py
View File

@ -14,7 +14,7 @@ setup(
url="https://docs.tutor.overhang.io/",
project_urls={
"Documentation": "https://docs.tutor.overhang.io/",
"Code": "https://github.com/overhangio/tutor/tree/master/plugins/minio",
"Code": "https://github.com/overhangio/tutor/tree/master/plugins/notes",
"Issue tracker": "https://github.com/overhangio/tutor/issues",
"Community": "https://discuss.overhang.io",
},

2
plugins/xqueue/setup.py
View File

@ -14,7 +14,7 @@ setup(
url="https://docs.tutor.overhang.io/",
project_urls={
"Documentation": "https://docs.tutor.overhang.io/",
"Code": "https://github.com/overhangio/tutor/tree/master/plugins/minio",
"Code": "https://github.com/overhangio/tutor/tree/master/plugins/xqueue",
"Issue tracker": "https://github.com/overhangio/tutor/issues",
"Community": "https://discuss.overhang.io",
},

4
requirements/plugins.txt
View File

@ -1,6 +1,6 @@
-e ./plugins/minio
-e ./plugins/notes
-e ./plugins/xqueue
tutor-discovery
tutor-ecommerce
tutor-figures
tutor-figures
tutor-minio

25
tests/test_env.py
View File

@ -125,12 +125,21 @@ class EnvTests(unittest.TestCase):
self.assertEqual("Hello my ID is abcd", f.read())
def test_renderer_is_reset_on_config_change(self):
config = {"PLUGINS": []}
env1 = env.Renderer.environment(config)
config["PLUGINS"].append("minio")
env2 = env.Renderer.environment(config)
with tempfile.TemporaryDirectory() as plugin_templates:
# Create one template
with open(os.path.join(plugin_templates, "myplugin.txt"), "w") as f:
f.write("some content")
self.assertNotIn(
"minio/hooks/mino-client/pre-init", env1.loader.list_templates()
)
self.assertIn("minio/hooks/minio-client/pre-init", env2.loader.list_templates())
# Load env once
config = {"PLUGINS": []}
env1 = env.Renderer.environment(config)
with unittest.mock.patch.object(
env.plugins, "iter_templates", return_value=[("myplugin", plugin_templates)]
):
# Load env a second time
config["PLUGINS"].append("myplugin")
env2 = env.Renderer.environment(config)
self.assertNotIn("myplugin.txt", env1.loader.list_templates())
self.assertIn("myplugin.txt", env2.loader.list_templates())

9
tests/test_plugins.py
View File

@ -194,6 +194,9 @@ class PluginsTests(unittest.TestCase):
config = {"PLUGINS": []}
instance1 = plugins.Plugins(config)
self.assertEqual(0, len(list(instance1.iter_enabled())))
config["PLUGINS"].append("minio")
instance2 = plugins.Plugins(config)
self.assertEqual(1, len(list(instance2.iter_enabled())))
config["PLUGINS"].append("plugin1")
with unittest.mock.patch.object(
plugins.Plugins, "iter_installed", return_value=[("plugin1", None)]
):
instance2 = plugins.Plugins(config)
self.assertEqual(1, len(list(instance2.iter_enabled())))