Working Kubernetes instakk

This deployment of Kubernetes does not support all features.

Close #116.

.gitignore

@@ -2,4 +2,4 @@
 !.gitignore
 /config.json
 /data*/
-/TODO
+TODO

deploy/k8s/.gitignore

@@ -1 +1,2 @@
 /env/
+/data

deploy/k8s/Makefile

@@ -1,34 +1,52 @@
-.PHONY: config volumes services deployments
+.PHONY: env volumes services deployments
 .DEFAULT_GOAL := help
 
-all: namespace config deploy assets
+podname = kubectl get pods -n openedx --selector=app=$(1) -o name | head -1 | cut -d '/' -f 2
+podexec = kubectl exec -n openedx -it $$($(call podname,$(1))) -- $(2)
+
+all: configure deploy
+
+configure:
+	@$(MAKE) -s -C ../.. --always-make config.json
+	@$(MAKE) -s env
+
+env:
+	@$(MAKE) -s -C .. env
+	@$(MAKE) -s -C ../.. substitute TEMPLATES=$(PWD)/templates OUTPUT=$(PWD)/env
 
 namespace: ## Create the platform namespace
 	kubectl create -f namespace.yml
-
-config: ## Create configmap objects
+configmaps: ## Create configmap objects
 	kubectl create configmap nginx-config --from-file=../env/nginx --namespace=openedx
 	kubectl create configmap mysql-config --from-env-file=../env/mysql/auth.env --namespace=openedx
 	kubectl create configmap openedx-settings-lms --from-file=../env/openedx/settings/lms --namespace=openedx
 	kubectl create configmap openedx-settings-cms --from-file=../env/openedx/settings/cms --namespace=openedx
 	kubectl create configmap openedx-config --from-file=../env/openedx/config --namespace=openedx
 	kubectl create configmap openedx-scripts --from-file=../env/openedx/scripts --namespace=openedx
-
 volumes: ## Create volumes
 	kubectl create -f volumes/ --recursive=true --namespace=openedx
-deployments: ## Create deployments
-	kubectl create -f deployments/ --recursive=true --namespace=openedx
 services: ## Create services
 	kubectl create -f services/ --recursive=true --namespace=openedx
-deploy: volumes services deployments
+deployments: ## Create deployments
+	kubectl create -f deployments/ --recursive=true --namespace=openedx
+ingress: ## Create ingress
+	kubectl create -f env/ingress.yml --namespace=openedx
+deploy: namespace volumes configmaps services deployments ingress ## Deploy a platform from scratch
 
-upgrade: down deploy
+upgrade: down ## Upgrade an already running platform
+	@$(MAKE) -s namespace || true
+	@$(MAKE) -s configmaps || true
+	@$(MAKE) -s volumes || true
+	@$(MAKE) -s services || true
+	@$(MAKE) -s deployments || true
+	@$(MAKE) -s ingress || true
 
 down: ## Delete all non-persistent objects
-	kubectl delete -f services/ --recursive=true --namespace=openedx
-	kubectl delete -f deployments/ --recursive=true --namespace=openedx
+	kubectl delete -f services/ --recursive=true --namespace=openedx || true
+	kubectl delete -f deployments/ --recursive=true --namespace=openedx || true
+	kubectl delete configmap --all --namespace openedx || true
 
-delete:
+delete: ## Delete EVERYTHING! (with no confirmation at all)
 	# TODO ask question to make sure user reaaaaaaally want to do this
 	kubectl delete namespace openedx
 
@@ -37,42 +55,65 @@ delete:
 
 databases: provision-databases migrate #provision-oauth2 ## Bootstrap databases
 provision-databases: ## Create necessary databases and users
-	@$(MAKE) lms-exec COMMAND="bash /openedx/scripts/provision.sh"
+	$(call podexec,lms,bash /openedx/scripts/provision.sh)
 provision-oauth2: ## Create users for SSO between services
-	@$(MAKE) lms-exec COMMAND="bash /openedx/scripts/oauth2.sh"
-migrate: migrate-openedx #migrate-forum $(extra_migrate_targets) ## Perform all database migrations
-migrate-openedx: ## Perform database migrations on LMS/CMS
-	@$(MAKE) lms-exec COMMAND="bash -c 'dockerize -wait tcp://mysql:3306 -timeout 20s && ./manage.py lms --settings=tutor.production migrate'"
-	#@$(MAKE) cms-exec COMMAND="bash -c 'dockerize -wait tcp://mysql:3306 -timeout 20s && ./manage.py cms --settings=tutor.production migrate'"
-	#$(MAKE) reindex-courses
-#migrate-forum: ## Perform database migrations on discussion forums
-	#$(DOCKER_COMPOSE_RUN) forum bash -c "bundle exec rake search:initialize && \
-		#bundle exec rake search:rebuild_index"
-#migrate-notes: ## Perform database migrations for the Notes service
-	#$(DOCKER_COMPOSE_RUN) notes ./manage.py migrate
-#migrate-xqueue: ## Perform database migrations for the XQueue service
-	#$(DOCKER_COMPOSE_RUN) xqueue ./manage.py migrate
-#reindex-courses: ## Refresh course index so they can be found in the LMS search engine
-	#$(DOCKER_COMPOSE_RUN) cms ./manage.py cms reindex_course --all --setup
+	$(call podexec,lms,bash /openedx/scripts/oauth2.sh)
+migrate: migrate-openedx migrate-forum ## Perform all database migrations
+migrate-openedx: migrate-lms migrate-cms reindex-courses ## Perform database migrations on LMS/CMS
+migrate-lms:
+	$(call podexec,lms,bash -c 'dockerize -wait tcp://mysql:3306 -timeout 20s && ./manage.py lms migrate')
+migrate-cms:
+	$(call podexec,cms,bash -c 'dockerize -wait tcp://mysql:3306 -timeout 20s && ./manage.py cms migrate')
+migrate-forum: ## Perform database migrations on discussion forums
+	$(call podexec,forum,bash -c "bundle exec rake search:initialize && \
+		bundle exec rake search:rebuild_index")
+migrate-notes: ## Perform database migrations for the Notes service
+	$(call podexec,notes,./manage.py migrate)
+migrate-xqueue: ## Perform database migrations for the XQueue service
+	$(call podexec,xqueue,./manage.py migrate)
+reindex-courses: ## Refresh course index so they can be found in the LMS search engine
+	$(call podexec,cms,./manage.py cms reindex_course --all --setup)
 
-##################### Assets
-
-assets:
-	$(MAKE) pod-exec APP=nginx COMMAND="rm -rf /var/www/openedx/staticfiles"
-	kubectl cp openedx/openedx:/openedx/staticfiles openedx/nginx:/var/www/openedx/staticfiles
-
-##################### Various commands
+##################### Various Open edX commands
 
 lms-shell: ## Launch a shell inside an lms pod
-	@$(MAKE) lms-exec COMMAND=bash
+	$(call podexec,lms,bash)
 lms-exec: ## Execute a command inside an lms pod: make lms-exec COMMAND="..."
-	@$(MAKE) pod-exec APP=lms COMMAND=$(COMMAND)
+	$(call podexec,lms,$(COMMAND))
 pod-exec: ## Execute a command inside an arbitrary pod: make pod-exec APP=appname COMMAND="..."
-	kubectl exec -n openedx -it $$(kubectl get pods -n openedx --selector=app=$(APP) -o name | head -1 | cut -d '/' -f 2) -- $(COMMAND)
+	$(call podexec,$(APP),$(COMMAND))
+
+# TODO replace these tasks with Job objects
+import-demo-course: ## Import the demo course from edX
+	$(call podexec,cms,/bin/bash -c "\
+		git clone https://github.com/edx/edx-demo-course --branch open-release/hawthorn.2 --depth 1 ../edx-demo-course \
+		&& python ./manage.py cms import ../data ../edx-demo-course")
+
+create-staff-user: ## Create a user with admin rights: make create-staff-user USERNAME=... EMAIL=...
+	$(call podexec,lms,/bin/bash -c "\
+		./manage.py lms manage_user --superuser --staff ${USERNAME} ${EMAIL} \
+		&& ./manage.py lms changepassword ${USERNAME}")
+
+##################### Various minikube command
+
+minikube-open:
+	minikube service nginx --namespace=openedx
+
+##################### Various k8s commands
+
+k8s-proxy: ## Create a proxy to the Kubernetes API server
+	kubectl proxy
+k8s-dashboard: ## Create the dashboard
+	kubectl create -f https://raw.githubusercontent.com/kubernetes/dashboard/master/aio/deploy/recommended/kubernetes-dashboard.yaml
+k8s-admin: ## Create an admin user for accessing the Kubernetes dashboard
+	kubectl -f admin.yml
+k8s-admin-token: ## Print the admin token required to log in the dashboard
+	kubectl -n kube-system describe secret $$(kubectl -n kube-system get secret | grep admin-user | awk '{print $$1}') | grep token: | awk '{print $$2}'
+
+##################### Information
 
 ESCAPE = 
 help: ## Print this help
 	@grep -E '^([a-zA-Z_-]+:.*?## .*|######* .+)$$' Makefile \
 		| sed 's/######* \(.*\)/\n               $(ESCAPE)[1;31m\1$(ESCAPE)[0m/g' \
 		| awk 'BEGIN {FS = ":.*?## "}; {printf "\033[33m%-30s\033[0m %s\n", $$1, $$2}'
-

deploy/k8s/admin.yml

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: admin-user
+  namespace: kube-system
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: admin-user
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+- kind: ServiceAccount
+  name: admin-user
+  namespace: kube-system

deploy/k8s/deployments/cms.yml

@@ -0,0 +1,50 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cms
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: cms
+  template:
+    metadata:
+      labels:
+        app: cms
+    spec:
+      containers:
+        - name: cms
+          image: regis/openedx:hawthorn
+          env:
+          - name: SERVICE_VARIANT
+            value: cms
+          ports:
+            - containerPort: 8000
+          volumeMounts:
+            - mountPath: /openedx/edx-platform/lms/envs/tutor/
+              name: settings-lms
+            - mountPath: /openedx/edx-platform/cms/envs/tutor/
+              name: settings-cms
+            - mountPath: /openedx/config
+              name: config
+            - mountPath: /openedx/scripts
+              name: scripts
+            - mountPath: /openedx/data
+              name: data
+          #imagePullPolicy: Always
+      volumes:
+        - name: settings-lms
+          configMap:
+            name: openedx-settings-lms
+        - name: settings-cms
+          configMap:
+            name: openedx-settings-cms
+        - name: config
+          configMap:
+            name: openedx-config
+        - name: scripts
+          configMap:
+            name: openedx-scripts
+        - name: data
+          persistentVolumeClaim:
+            claimName: cms-data

deploy/k8s/deployments/forum.yml

@@ -0,0 +1,20 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: forum
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: forum
+  template:
+    metadata:
+      labels:
+        app: forum
+    spec:
+      containers:
+        - name: forum
+          image: regis/openedx-forum:hawthorn
+          ports:
+            - containerPort: 4567
+          imagePullPolicy: Always

deploy/k8s/deployments/lms.yml

@@ -14,7 +14,7 @@ spec:
     spec:
       containers:
         - name: lms
-          image: regis/openedx:latest
+          image: regis/openedx:hawthorn
           ports:
             - containerPort: 8000
           volumeMounts:
@@ -28,6 +28,7 @@ spec:
               name: scripts
             - mountPath: /openedx/data
               name: data
+          imagePullPolicy: Always
       volumes:
         - name: settings-lms
           configMap:

deploy/k8s/deployments/nginx.yml

@@ -12,12 +12,29 @@ spec:
       labels:
         app: nginx
     spec:
+      initContainers:
+        - name: clean-openedx-staticfiles
+          image: regis/openedx:hawthorn
+          command: ['rm', '-rf', '/var/www/openedx/staticfiles']
+          volumeMounts:
+            - mountPath: /var/www/openedx/
+              name: openedx-staticfiles
+          imagePullPolicy: Always
+        - name: init-openedx-staticfiles
+          image: regis/openedx:hawthorn
+          command: ['cp', '-r', '/openedx/staticfiles', '/var/www/openedx/']
+          volumeMounts:
+            - mountPath: /var/www/openedx/
+              name: openedx-staticfiles
+          imagePullPolicy: Always
       containers:
         - name: nginx
-          image: nginx:latest
+          image: nginx:1.13
           volumeMounts:
             - mountPath: /etc/nginx/conf.d/
               name: config
+            - mountPath: /var/www/openedx/
+              name: openedx-staticfiles
             - mountPath: /openedx/data/lms
               name: data
           ports:
@@ -29,6 +46,9 @@ spec:
         - name: config
           configMap:
             name: nginx-config
+        - name: openedx-staticfiles
+          persistentVolumeClaim:
+            claimName: openedx-staticfiles
         - name: data
           persistentVolumeClaim:
             claimName: lms-data

deploy/k8s/services/cms.yml

@@ -1,12 +1,11 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: openedx
+  name: cms
 spec:
   type: NodePort
   ports:
     - port: 8000
       protocol: TCP
   selector:
-    # TODO different services and names for lms/cms
-    app: openedx
+    app: cms

deploy/k8s/services/forum.yml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: forum
+spec:
+  type: NodePort
+  ports:
+    - port: 4567
+      protocol: TCP
+  selector:
+    app: forum

deploy/k8s/services/lms.yml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: lms
+spec:
+  type: NodePort
+  ports:
+    - port: 8000
+      protocol: TCP
+  selector:
+    app: lms

deploy/k8s/templates/ingress.yml

@@ -1,10 +1,13 @@
 apiVersion: extensions/v1beta1
 kind: Ingress
 metadata:
-  name: ingress
+  name: web
 spec:
   rules:
-  - host: overhang.io
+  {% set hosts = [LMS_HOST, "preview." + LMS_HOST, CMS_HOST] %}
+  {% if ACTIVATE_NOTES %}{% set hosts = hosts + ["notes." + LMS_HOST] %}{% endif %}
+  {% for host in hosts %}
+  - host: {{ host }}
     http:
       paths:
       - backend:
@@ -13,3 +16,4 @@ spec:
       - backend:
           serviceName: nginx
           servicePort: 443
+  {% endfor %}

deploy/k8s/volumes/cms-data.yml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: cms-data
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 2Gi

deploy/k8s/volumes/openedx-staticfiles.yml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: openedx-staticfiles
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1Gi

deploy/local/Makefile

@@ -165,7 +165,7 @@ import-demo-course: ## Import the demo course from edX
 		git clone https://github.com/edx/edx-demo-course --branch open-release/hawthorn.2 --depth 1 ../edx-demo-course \
 		&& python ./manage.py cms import ../data ../edx-demo-course"
 
-create-staff-user: ## Create a user with admin rights
+create-staff-user: ## Create a user with admin rights: make create-staff-user USERNAME=... EMAIL=...
 	$(DOCKER_COMPOSE_RUN_OPENEDX) lms /bin/bash -c "./manage.py lms manage_user --superuser --staff ${USERNAME} ${EMAIL} && ./manage.py lms changepassword ${USERNAME}"
 
 portainer: ## Run Portainer (https://portainer.io), a UI for container supervision

docs/k8s.rst

@@ -3,7 +3,7 @@
 Kubernetes deployment
 =====================
 
-With the same docker images we created for :ref:`single server deployment <step>` and :ref:`local development <development>`, we can launch an Open edX platform on Kubernetes. Always in 1 click, of course :)
+With the same docker images we created for :ref:`single server deployment <local>` and :ref:`local development <development>`, we can launch an Open edX platform on Kubernetes. Always in 1 click, of course :)
 
 ::
 
@@ -25,6 +25,11 @@ Start Minikube::
 
     minikube start
 
+When minikube starts, it spawns a virtual machine (VM) which you can configure in your VM manager: on most platforms, this is Virtualbox. You should configure your VM to have at least 4Gb RAM; otherwise, database migrations will crash halfway, and that's a nasty issue...
+
+.. image:: img/virtualbox-minikube-system.png
+    :alt: Virtualbox memory settings for Minikube
+
 Ingress addon must be installed::
 
     minikube addons enable ingress
@@ -33,6 +38,12 @@ At any point, access a UI to view the state of the platform with::
 
     minikube dashboard
 
+With Kubernetes, your Open edX platform will not be available at localhost or studio.localhost. Instead, you will have to access your platform with the domain names you specified for the LMS and the CMS. To do so on a local computer, you will need to add the following line to /etc/hosts::
+
+    MINIKUBEIP yourdomain.com studio.yourdomain.com preview.yourdomain.com notes.yourdomain.com
+
+where ``MINIKUBEIP`` should be replaced by the result of the command ``minikube ip``.
+
 In the following, all commands should be run inside the ``deploy/k8s`` folder::
 
     cd deploy/k8s
@@ -43,3 +54,46 @@ Quickstart
 Launch the platform on k8s in 1 click::
 
     make all
+
+All Kubernetes resources are associated to the "openedx" namespace. If you don't see anything in the Kubernetes dashboard, you are probably looking at the wrong namespace... 😉
+
+.. image:: img/k8s-dashboard.png
+    :alt: Kubernetes dashboard ("openedx" namespace)
+
+Upgrading
+---------
+
+After pulling updates from the Tutor repository, you can apply changes with::
+
+    make upgrade
+
+Accessing the Kubernetes dashboard
+----------------------------------
+
+Depending on your Kubernetes provider, you may need to create a dashboard yourself. To do so, run::
+
+    make k8s-dashboard
+
+Then, you will have to create an admin user::
+
+    make k8s-admin
+
+Print the admin token required for authentication, and copy its value::
+
+    make k8s-admin-token
+
+Create a proxy to the Kubernetes API server::
+
+    k8s-proxy
+
+Use the token to log in the dashboard at the following url: http://localhost:8001/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy/
+
+Missing features
+----------------
+
+For now, the following features from the local deployment are not supported:
+
+- HTTPS certificates
+- Xqueue
+
+Kubernetes deployment is under intense development, and these features should be implemented pretty soon. Stay tuned 🤓