Move tutor-minio to dedicated plugin repo

CHANGELOG.md

@@ -4,6 +4,7 @@ Note: Breaking changes between versions are indicated by "💥".
 
 ## Latest
 
+- [Improvement] Move minio plugin outside of the tutor repo
 - [Bugfix/Improvement] Add all plugins (with data) into binary bundle (#242)
 
 ## 3.6.2 (2019-08-07)

Makefile

@@ -13,7 +13,6 @@ package: ## Build a package ready to upload to pypi
 	python3 setup.py sdist
 
 package-plugins: ## Build packages for each plugin
-	cd plugins/minio && python3 setup.py sdist --dist-dir=../../dist/
 	cd plugins/notes && python3 setup.py sdist --dist-dir=../../dist/
 	cd plugins/xqueue && python3 setup.py sdist --dist-dir=../../dist/
 
@@ -31,7 +30,6 @@ test-unit-core: ## Run unit tests on core
 	python3 -m unittest discover tests
 
 test-unit-plugins: ## Run unit tests on plugins
-	python3 -m unittest discover plugins/minio/tests
 	python3 -m unittest discover plugins/xqueue/tests
 
 test-packages: package package-plugins ## Test that packages can be uploaded to pypi

docs/plugins.rst

@@ -180,7 +180,7 @@ Existing plugins
 ----------------
 
 - `Course discovery <https://pypi.org/project/tutor-discovery>`__: Deploy an API for interacting with your course catalog
-- `Ecommerce <https://pypi.org/project/tutor-minio>`__: Sell courses and products on your Open edX platform
+- `Ecommerce <https://pypi.org/project/tutor-ecommerce>`__: Sell courses and products on your Open edX platform
 - `Figures <https://pypi.org/project/tutor-figures>`__: Visualize daily stats about course engagement
-- `MinIO <https://github.com/overhangio/tutor/tree/master/plugins/minio>`__: S3 emulator for object storage and scalable Open edX deployment.
+- `MinIO <https://pypi.org/project/tutor-minio>`__: S3 emulator for object storage and scalable Open edX deployment.
-- `Xqueue <https://github.com/overhangio/tutor/tree/master/plugins/xqueue>`__: for external grading
+- `Xqueue <https://pypi.org/project/tutor-xqueue>`__: for external grading

plugins/minio/MANIFEST.in

@@ -1,2 +0,0 @@
-recursive-include tutorminio/patches *
-recursive-include tutorminio/templates *

plugins/minio/README.rst

@@ -1,45 +0,0 @@
-Object storage for Open edX with `MinIO <https://www.minio.io/>`_
-=================================================================
-
-This is a plugin for `Tutor <https://docs.tutor.overhang.io>`_ that provides S3-like object storage for Open edX platforms. It's S3, but without the dependency on AWS. This is achieved thanks to `MinIO <https://www.minio.io/>`_, an open source project that provides object storage with an API compatible with S3.
-
-In particular, this plugin is essential for `Kubernetes deployment <https://docs.tutor.overhang.io/k8s.html>`_.
-
-Installation
-------------
-
-The plugin is currently bundled with the `binary releases of Tutor <https://github.com/overhangio/tutor/releases>`_. If you have installed Tutor from source, you will have to install this plugin from source, too::
-  
-    pip install tutor-minio
-
-Then, to enable this plugin, run::
-  
-    tutor plugins enable minio
-
-Configuration
--------------
-
-- ``MINIO_BUCKET_NAME`` (default: ``"openedx"``)
-- ``MINIO_FILE_UPLOAD_BUCKET_NAME`` (default: ``"openedxuploads"``)
-- ``MINIO_COURSE_IMPORT_EXPORT_BUCKET`` (default: ``"openedxcourseimportexport"``)
-- ``MINIO_HOST`` (default: ``"minio.{{ LMS_HOST }}"``)
-- ``MINIO_DOCKER_REGISTRY`` (default: ``"{{ DOCKER_REGISTRY }}"``)
-- ``MINIO_DOCKER_IMAGE_CLIENT`` (default: ``"minio/mc:RELEASE.2019-05-23T01-33-27Z"``)
-- ``MINIO_DOCKER_IMAGE_SERVER`` (default: ``"minio/minio:RELEASE.2019-05-23T00-29-34Z"``)
-
-These values can be modified with ``tutor config save --set PARAM_NAME=VALUE`` commands.
-
-DNS records
------------
-
-It is assumed that the ``MINIO_HOST`` DNS record points to your server. When running MinIO on your laptop, you should point your services to ``minio.localhost``::
-
-    tutor config save --set MINIO_HOST=minio.localhost
-
-Web UI
-------
-
-The MinIO web UI can be accessed at http://<MINIO_HOST>. The credentials for accessing the UI can be obtained with::
-
-  tutor config printvalue OPENEDX_AWS_ACCESS_KEY
-  tutor config printvalue OPENEDX_AWS_SECRET_ACCESS_KEY

plugins/minio/setup.py

@@ -1,40 +0,0 @@
-import io
-import os
-from setuptools import setup, find_packages
-
-here = os.path.abspath(os.path.dirname(__file__))
-
-with io.open(os.path.join(here, "README.rst"), "rt", encoding="utf8") as f:
-    readme = f.read()
-
-
-setup(
-    name="tutor-minio",
-    version="0.1.1",
-    url="https://docs.tutor.overhang.io/",
-    project_urls={
-        "Documentation": "https://docs.tutor.overhang.io/",
-        "Code": "https://github.com/overhangio/tutor/tree/master/plugins/minio",
-        "Issue tracker": "https://github.com/overhangio/tutor/issues",
-        "Community": "https://discuss.overhang.io",
-    },
-    license="AGPLv3",
-    author="Overhang.io",
-    author_email="contact@overhang.io",
-    description="A Tutor plugin for object storage in MinIO",
-    long_description=readme,
-    packages=find_packages(exclude=["tests*"]),
-    include_package_data=True,
-    python_requires=">=3.5",
-    entry_points={"tutor.plugin.v0": ["minio = tutorminio.plugin"]},
-    classifiers=[
-        "Development Status :: 3 - Alpha",
-        "Intended Audience :: Developers",
-        "License :: OSI Approved :: GNU Affero General Public License v3",
-        "Operating System :: OS Independent",
-        "Programming Language :: Python",
-        "Programming Language :: Python :: 3.5",
-        "Programming Language :: Python :: 3.6",
-        "Programming Language :: Python :: 3.7",
-    ],
-)

plugins/minio/tests/test_plugin.py

@@ -1,12 +0,0 @@
-import unittest
-
-from tutorminio import plugin
-
-
-class PluginTests(unittest.TestCase):
-    def test_patches(self):
-        patches = dict(plugin.patches())
-        self.assertIn("local-docker-compose-services", patches)
-        self.assertTrue(
-            patches["local-docker-compose-services"].startswith("# MinIO\n")
-        )

plugins/minio/tutorminio/patches/cms-env

@@ -1,2 +0,0 @@
-"FILE_UPLOAD_STORAGE_BUCKET_NAME": "{{ MINIO_FILE_UPLOAD_BUCKET_NAME }}",
-"COURSE_IMPORT_EXPORT_BUCKET": "{{ MINIO_COURSE_IMPORT_EXPORT_BUCKET }}"

plugins/minio/tutorminio/patches/https-create

@@ -1,2 +0,0 @@
-# MinIO
-certbot certonly --standalone -n --agree-tos -m admin@{{ LMS_HOST }} -d {{ MINIO_HOST }}

plugins/minio/tutorminio/patches/k8s-deployments

@@ -1,57 +0,0 @@
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: minio
-  labels:
-    app.kubernetes.io/name: minio
-spec:
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: minio
-  strategy:
-    type: Recreate
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: minio
-    spec:
-      containers:
-        - name: minio
-          image: {{ MINIO_DOCKER_REGISTRY }}{{ MINIO_DOCKER_IMAGE_SERVER }}
-          args: ["server", "--address", ":9000", "/data"]
-          env:
-            - name: MINIO_ACCESS_KEY
-              value: "{{ OPENEDX_AWS_ACCESS_KEY }}"
-            - name: MINIO_SECRET_KEY
-              value: "{{ OPENEDX_AWS_SECRET_ACCESS_KEY }}"
-          ports:
-            - containerPort: 9000
-          volumeMounts:
-            - mountPath: /data
-              name: data
-      volumes:
-        - name: data
-          persistentVolumeClaim:
-            claimName: minio
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: minio-client
-  labels:
-    app.kubernetes.io/name: minio-client
-spec:
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: minio-client
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: minio-client
-    spec:
-      containers:
-        - name: minio
-          image: {{ MINIO_DOCKER_REGISTRY }}{{ MINIO_DOCKER_IMAGE_CLIENT }}
-          command: ["sh", "-e", "-c"]
-          args: ["while true; do echo 'ready'; sleep 10; done"]

plugins/minio/tutorminio/patches/k8s-ingress-certificates

@@ -1,18 +0,0 @@
----
-apiVersion: certmanager.k8s.io/v1alpha1
-kind: Certificate
-metadata:
-  name: {{ MINIO_HOST|replace(".", "-") }}
-spec:
-  secretName: {{ MINIO_HOST }}-tls
-  issuerRef:
-    name: letsencrypt
-  commonName: {{ MINIO_HOST }}
-  dnsNames:
-  - {{ MINIO_HOST }}
-  acme:
-    config:
-    - http01:
-        ingress: web
-      domains:
-      - {{ MINIO_HOST }}

plugins/minio/tutorminio/patches/k8s-ingress-rules

@@ -1,6 +0,0 @@
-- host: {{ MINIO_HOST }}
-  http:
-    paths:
-    - backend:
-        serviceName: nginx
-        servicePort: {% if ACTIVATE_HTTPS %}443{% else %}80{% endif %}

plugins/minio/tutorminio/patches/k8s-ingress-tls-hosts

@@ -1 +0,0 @@
-- {{ MINIO_HOST }}

plugins/minio/tutorminio/patches/k8s-services

@@ -1,12 +0,0 @@
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: minio
-spec:
-  type: NodePort
-  ports:
-    - port: 9000
-      protocol: TCP
-  selector:
-    app.kubernetes.io/name: minio

plugins/minio/tutorminio/patches/k8s-volumes

@@ -1,14 +0,0 @@
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: minio
-  labels:
-    app.kubernetes.io/component: volume
-    app.kubernetes.io/name: minio
-spec:
-  accessModes:
-    - ReadWriteOnce
-  resources:
-    requests:
-      storage: 5Gi

plugins/minio/tutorminio/patches/lms-env

@@ -1,2 +0,0 @@
-"FILE_UPLOAD_STORAGE_BUCKET_NAME": "{{ MINIO_FILE_UPLOAD_BUCKET_NAME }}",
-"COURSE_IMPORT_EXPORT_BUCKET": "{{ MINIO_COURSE_IMPORT_EXPORT_BUCKET }}"

plugins/minio/tutorminio/patches/local-docker-compose-nginx-aliases

@@ -1 +0,0 @@
-"{{ MINIO_HOST }}"

plugins/minio/tutorminio/patches/local-docker-compose-services

@@ -1,17 +0,0 @@
-# MinIO
-minio:
-  image: {{ MINIO_DOCKER_REGISTRY }}{{ MINIO_DOCKER_IMAGE_SERVER }}
-  volumes:
-    - ../../data/minio:/data
-  environment:
-    MINIO_ACCESS_KEY: "{{ OPENEDX_AWS_ACCESS_KEY }}"
-    MINIO_SECRET_KEY: "{{ OPENEDX_AWS_SECRET_ACCESS_KEY }}"
-  command: server --address ":9000" /data
-  restart: unless-stopped
-
-minio-client:
-  image: {{ MINIO_DOCKER_REGISTRY }}{{ MINIO_DOCKER_IMAGE_CLIENT }}
-  restart: "no"
-  entrypoint: sh
-  depends_on:
-    - minio

plugins/minio/tutorminio/patches/nginx-extra

@@ -1,37 +0,0 @@
-# MinIO public service
-upstream minio-backend {
-    server minio:9000 fail_timeout=0;
-}
-{% if ACTIVATE_HTTPS %}
-server {
-    server_name {{ MINIO_HOST }};
-    listen 80;
-    return 301 https://$server_name$request_uri;
-}
-{% endif %}
-server {
-  {% if ACTIVATE_HTTPS %}listen 443 {{ "" if WEB_PROXY else "ssl" }};{% else %}listen 80;{% endif %}
-  server_name minio.localhost {{ MINIO_HOST }};
-
-  {% if ACTIVATE_HTTPS and not WEB_PROXY %}
-  ssl_certificate /etc/letsencrypt/live/{{ MINIO_HOST }}/fullchain.pem;
-  ssl_certificate_key /etc/letsencrypt/live/{{ MINIO_HOST }}/privkey.pem;
-  {% endif %}
-
-  # Disables server version feedback on pages and in headers
-  server_tokens off;
-  
-  client_max_body_size 0;
-
-  location / {
-    {% if not WEB_PROXY %}
-    proxy_set_header X-Forwarded-Proto $scheme;
-    proxy_set_header X-Forwarded-Port $server_port;
-    proxy_set_header X-Forwarded-For $remote_addr;
-    {% endif %}
-    proxy_set_header Host $http_host;
-    proxy_redirect off;
-
-    proxy_pass http://minio-backend;
-  }
-}

plugins/minio/tutorminio/patches/openedx-auth

@@ -1,2 +0,0 @@
-"AWS_STORAGE_BUCKET_NAME": "{{ MINIO_BUCKET_NAME }}",
-"AWS_S3_CUSTOM_DOMAIN": ""

plugins/minio/tutorminio/patches/openedx-common-settings

@@ -1,17 +0,0 @@
-AWS_S3_HOST = "{{ MINIO_HOST }}"
-AWS_S3_USE_SSL = {{ "True" if ACTIVATE_HTTPS else "False" }}
-AWS_S3_SECURE_URLS = {{ "True" if ACTIVATE_HTTPS else "False" }}
-AWS_S3_CALLING_FORMAT = "boto.s3.connection.OrdinaryCallingFormat"
-AWS_AUTO_CREATE_BUCKET = False # explicit is better than implicit
-
-# Configuring boto is required for ora2 because ora2 does not read
-# host/port/ssl settings from django. Hence this hack.
-# http://docs.pythonboto.org/en/latest/boto_config_tut.html
-import os
-os.environ["AWS_CREDENTIAL_FILE"] = "/tmp/boto.cfg"
-with open("/tmp/boto.cfg", "w") as f:
-    f.write("""[Boto]
-is_secure = {{ "True" if ACTIVATE_HTTPS else "False" }}
-[s3]
-host = {{ MINIO_HOST }}
-calling_format = boto.s3.connection.OrdinaryCallingFormat""")

plugins/minio/tutorminio/patches/openedx-lms-common-settings

@@ -1 +0,0 @@
-ORA2_FILEUPLOAD_BACKEND = "s3"

plugins/minio/tutorminio/plugin.py

@@ -1,40 +0,0 @@
-import os
-from glob import glob
-
-HERE = os.path.abspath(os.path.dirname(__file__))
-
-config = {
-    "set": {
-        "OPENEDX_AWS_ACCESS_KEY": "openedx",
-        "OPENEDX_AWS_SECRET_ACCESS_KEY": "{{ 24|random_string }}",
-    },
-    "defaults": {
-        "BUCKET_NAME": "openedx",
-        "FILE_UPLOAD_BUCKET_NAME": "openedxuploads",
-        "COURSE_IMPORT_EXPORT_BUCKET": "openedxcourseimportexport",
-        "HOST": "minio.{{ LMS_HOST }}",
-        "DOCKER_REGISTRY": "{{ DOCKER_REGISTRY }}",
-        "DOCKER_IMAGE_CLIENT": "minio/mc:RELEASE.2019-05-23T01-33-27Z",
-        "DOCKER_IMAGE_SERVER": "minio/minio:RELEASE.2019-05-23T00-29-34Z",
-    },
-}
-
-templates = os.path.join(HERE, "templates")
-
-hooks = {
-    "pre-init": ["minio-client"],
-    "remote-image": {
-        "minio-server": "{{ MINIO_DOCKER_IMAGE_SERVER }}",
-        "minio-client": "{{ MINIO_DOCKER_IMAGE_CLIENT }}",
-    },
-}
-
-
-def patches():
-    all_patches = {}
-    for path in glob(os.path.join(HERE, "patches", "*")):
-        with open(path) as patch_file:
-            name = os.path.basename(path)
-            content = patch_file.read()
-            all_patches[name] = content
-    return all_patches

plugins/minio/tutorminio/templates/minio/hooks/minio-client/pre-init

@@ -1,2 +0,0 @@
-mc config host add minio http://minio:9000 {{ OPENEDX_AWS_ACCESS_KEY }} {{ OPENEDX_AWS_SECRET_ACCESS_KEY }} --api s3v4
-mc mb --ignore-existing minio/{{ MINIO_BUCKET_NAME }} minio/{{ MINIO_FILE_UPLOAD_BUCKET_NAME }} minio/{{ MINIO_COURSE_IMPORT_EXPORT_BUCKET }}

plugins/notes/setup.py

@@ -14,7 +14,7 @@ setup(
     url="https://docs.tutor.overhang.io/",
     project_urls={
         "Documentation": "https://docs.tutor.overhang.io/",
-        "Code": "https://github.com/overhangio/tutor/tree/master/plugins/minio",
+        "Code": "https://github.com/overhangio/tutor/tree/master/plugins/notes",
         "Issue tracker": "https://github.com/overhangio/tutor/issues",
         "Community": "https://discuss.overhang.io",
     },

plugins/xqueue/setup.py

@@ -14,7 +14,7 @@ setup(
     url="https://docs.tutor.overhang.io/",
     project_urls={
         "Documentation": "https://docs.tutor.overhang.io/",
-        "Code": "https://github.com/overhangio/tutor/tree/master/plugins/minio",
+        "Code": "https://github.com/overhangio/tutor/tree/master/plugins/xqueue",
         "Issue tracker": "https://github.com/overhangio/tutor/issues",
         "Community": "https://discuss.overhang.io",
     },

requirements/plugins.txt

@@ -1,6 +1,6 @@
--e ./plugins/minio
 -e ./plugins/notes
 -e ./plugins/xqueue
 tutor-discovery
 tutor-ecommerce
-tutor-figures
+tutor-figures
+tutor-minio

tests/test_env.py

@@ -125,12 +125,21 @@ class EnvTests(unittest.TestCase):
                         self.assertEqual("Hello my ID is abcd", f.read())
 
     def test_renderer_is_reset_on_config_change(self):
-        config = {"PLUGINS": []}
+        with tempfile.TemporaryDirectory() as plugin_templates:
-        env1 = env.Renderer.environment(config)
+            # Create one template
-        config["PLUGINS"].append("minio")
+            with open(os.path.join(plugin_templates, "myplugin.txt"), "w") as f:
-        env2 = env.Renderer.environment(config)
+                f.write("some content")
 
-        self.assertNotIn(
+                # Load env once
-            "minio/hooks/mino-client/pre-init", env1.loader.list_templates()
+                config = {"PLUGINS": []}
-        )
+                env1 = env.Renderer.environment(config)
-        self.assertIn("minio/hooks/minio-client/pre-init", env2.loader.list_templates())
+
+                with unittest.mock.patch.object(
+                    env.plugins, "iter_templates", return_value=[("myplugin", plugin_templates)]
+                ):
+                    # Load env a second time
+                    config["PLUGINS"].append("myplugin")
+                    env2 = env.Renderer.environment(config)
+
+                self.assertNotIn("myplugin.txt", env1.loader.list_templates())
+                self.assertIn("myplugin.txt", env2.loader.list_templates())

tests/test_plugins.py

@@ -194,6 +194,9 @@ class PluginsTests(unittest.TestCase):
         config = {"PLUGINS": []}
         instance1 = plugins.Plugins(config)
         self.assertEqual(0, len(list(instance1.iter_enabled())))
-        config["PLUGINS"].append("minio")
+        config["PLUGINS"].append("plugin1")
-        instance2 = plugins.Plugins(config)
+        with unittest.mock.patch.object(
-        self.assertEqual(1, len(list(instance2.iter_enabled())))
+            plugins.Plugins, "iter_installed", return_value=[("plugin1", None)]
+        ):
+            instance2 = plugins.Plugins(config)
+            self.assertEqual(1, len(list(instance2.iter_enabled())))