diff --git a/.gitignore b/.gitignore index d55b574..aec72a3 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,4 @@ !.gitignore /config.json /data*/ -/TODO +TODO diff --git a/deploy/k8s/.gitignore b/deploy/k8s/.gitignore index 47def24..9939222 100644 --- a/deploy/k8s/.gitignore +++ b/deploy/k8s/.gitignore @@ -1 +1,2 @@ /env/ +/data diff --git a/deploy/k8s/Makefile b/deploy/k8s/Makefile index a16bb4c..ceffbce 100644 --- a/deploy/k8s/Makefile +++ b/deploy/k8s/Makefile @@ -1,34 +1,52 @@ -.PHONY: config volumes services deployments +.PHONY: env volumes services deployments .DEFAULT_GOAL := help -all: namespace config deploy assets +podname = kubectl get pods -n openedx --selector=app=$(1) -o name | head -1 | cut -d '/' -f 2 +podexec = kubectl exec -n openedx -it $$($(call podname,$(1))) -- $(2) + +all: configure deploy + +configure: + @$(MAKE) -s -C ../.. --always-make config.json + @$(MAKE) -s env + +env: + @$(MAKE) -s -C .. env + @$(MAKE) -s -C ../.. substitute TEMPLATES=$(PWD)/templates OUTPUT=$(PWD)/env namespace: ## Create the platform namespace kubectl create -f namespace.yml - -config: ## Create configmap objects +configmaps: ## Create configmap objects kubectl create configmap nginx-config --from-file=../env/nginx --namespace=openedx kubectl create configmap mysql-config --from-env-file=../env/mysql/auth.env --namespace=openedx kubectl create configmap openedx-settings-lms --from-file=../env/openedx/settings/lms --namespace=openedx kubectl create configmap openedx-settings-cms --from-file=../env/openedx/settings/cms --namespace=openedx kubectl create configmap openedx-config --from-file=../env/openedx/config --namespace=openedx kubectl create configmap openedx-scripts --from-file=../env/openedx/scripts --namespace=openedx - volumes: ## Create volumes kubectl create -f volumes/ --recursive=true --namespace=openedx -deployments: ## Create deployments - kubectl create -f deployments/ --recursive=true --namespace=openedx services: ## Create services kubectl create -f services/ --recursive=true --namespace=openedx -deploy: volumes services deployments +deployments: ## Create deployments + kubectl create -f deployments/ --recursive=true --namespace=openedx +ingress: ## Create ingress + kubectl create -f env/ingress.yml --namespace=openedx +deploy: namespace volumes configmaps services deployments ingress ## Deploy a platform from scratch -upgrade: down deploy +upgrade: down ## Upgrade an already running platform + @$(MAKE) -s namespace || true + @$(MAKE) -s configmaps || true + @$(MAKE) -s volumes || true + @$(MAKE) -s services || true + @$(MAKE) -s deployments || true + @$(MAKE) -s ingress || true down: ## Delete all non-persistent objects - kubectl delete -f services/ --recursive=true --namespace=openedx - kubectl delete -f deployments/ --recursive=true --namespace=openedx + kubectl delete -f services/ --recursive=true --namespace=openedx || true + kubectl delete -f deployments/ --recursive=true --namespace=openedx || true + kubectl delete configmap --all --namespace openedx || true -delete: +delete: ## Delete EVERYTHING! (with no confirmation at all) # TODO ask question to make sure user reaaaaaaally want to do this kubectl delete namespace openedx @@ -37,42 +55,65 @@ delete: databases: provision-databases migrate #provision-oauth2 ## Bootstrap databases provision-databases: ## Create necessary databases and users - @$(MAKE) lms-exec COMMAND="bash /openedx/scripts/provision.sh" + $(call podexec,lms,bash /openedx/scripts/provision.sh) provision-oauth2: ## Create users for SSO between services - @$(MAKE) lms-exec COMMAND="bash /openedx/scripts/oauth2.sh" -migrate: migrate-openedx #migrate-forum $(extra_migrate_targets) ## Perform all database migrations -migrate-openedx: ## Perform database migrations on LMS/CMS - @$(MAKE) lms-exec COMMAND="bash -c 'dockerize -wait tcp://mysql:3306 -timeout 20s && ./manage.py lms --settings=tutor.production migrate'" - #@$(MAKE) cms-exec COMMAND="bash -c 'dockerize -wait tcp://mysql:3306 -timeout 20s && ./manage.py cms --settings=tutor.production migrate'" - #$(MAKE) reindex-courses -#migrate-forum: ## Perform database migrations on discussion forums - #$(DOCKER_COMPOSE_RUN) forum bash -c "bundle exec rake search:initialize && \ - #bundle exec rake search:rebuild_index" -#migrate-notes: ## Perform database migrations for the Notes service - #$(DOCKER_COMPOSE_RUN) notes ./manage.py migrate -#migrate-xqueue: ## Perform database migrations for the XQueue service - #$(DOCKER_COMPOSE_RUN) xqueue ./manage.py migrate -#reindex-courses: ## Refresh course index so they can be found in the LMS search engine - #$(DOCKER_COMPOSE_RUN) cms ./manage.py cms reindex_course --all --setup + $(call podexec,lms,bash /openedx/scripts/oauth2.sh) +migrate: migrate-openedx migrate-forum ## Perform all database migrations +migrate-openedx: migrate-lms migrate-cms reindex-courses ## Perform database migrations on LMS/CMS +migrate-lms: + $(call podexec,lms,bash -c 'dockerize -wait tcp://mysql:3306 -timeout 20s && ./manage.py lms migrate') +migrate-cms: + $(call podexec,cms,bash -c 'dockerize -wait tcp://mysql:3306 -timeout 20s && ./manage.py cms migrate') +migrate-forum: ## Perform database migrations on discussion forums + $(call podexec,forum,bash -c "bundle exec rake search:initialize && \ + bundle exec rake search:rebuild_index") +migrate-notes: ## Perform database migrations for the Notes service + $(call podexec,notes,./manage.py migrate) +migrate-xqueue: ## Perform database migrations for the XQueue service + $(call podexec,xqueue,./manage.py migrate) +reindex-courses: ## Refresh course index so they can be found in the LMS search engine + $(call podexec,cms,./manage.py cms reindex_course --all --setup) -##################### Assets - -assets: - $(MAKE) pod-exec APP=nginx COMMAND="rm -rf /var/www/openedx/staticfiles" - kubectl cp openedx/openedx:/openedx/staticfiles openedx/nginx:/var/www/openedx/staticfiles - -##################### Various commands +##################### Various Open edX commands lms-shell: ## Launch a shell inside an lms pod - @$(MAKE) lms-exec COMMAND=bash + $(call podexec,lms,bash) lms-exec: ## Execute a command inside an lms pod: make lms-exec COMMAND="..." - @$(MAKE) pod-exec APP=lms COMMAND=$(COMMAND) + $(call podexec,lms,$(COMMAND)) pod-exec: ## Execute a command inside an arbitrary pod: make pod-exec APP=appname COMMAND="..." - kubectl exec -n openedx -it $$(kubectl get pods -n openedx --selector=app=$(APP) -o name | head -1 | cut -d '/' -f 2) -- $(COMMAND) + $(call podexec,$(APP),$(COMMAND)) + +# TODO replace these tasks with Job objects +import-demo-course: ## Import the demo course from edX + $(call podexec,cms,/bin/bash -c "\ + git clone https://github.com/edx/edx-demo-course --branch open-release/hawthorn.2 --depth 1 ../edx-demo-course \ + && python ./manage.py cms import ../data ../edx-demo-course") + +create-staff-user: ## Create a user with admin rights: make create-staff-user USERNAME=... EMAIL=... + $(call podexec,lms,/bin/bash -c "\ + ./manage.py lms manage_user --superuser --staff ${USERNAME} ${EMAIL} \ + && ./manage.py lms changepassword ${USERNAME}") + +##################### Various minikube command + +minikube-open: + minikube service nginx --namespace=openedx + +##################### Various k8s commands + +k8s-proxy: ## Create a proxy to the Kubernetes API server + kubectl proxy +k8s-dashboard: ## Create the dashboard + kubectl create -f https://raw.githubusercontent.com/kubernetes/dashboard/master/aio/deploy/recommended/kubernetes-dashboard.yaml +k8s-admin: ## Create an admin user for accessing the Kubernetes dashboard + kubectl -f admin.yml +k8s-admin-token: ## Print the admin token required to log in the dashboard + kubectl -n kube-system describe secret $$(kubectl -n kube-system get secret | grep admin-user | awk '{print $$1}') | grep token: | awk '{print $$2}' + +##################### Information ESCAPE =  help: ## Print this help @grep -E '^([a-zA-Z_-]+:.*?## .*|######* .+)$$' Makefile \ | sed 's/######* \(.*\)/\n $(ESCAPE)[1;31m\1$(ESCAPE)[0m/g' \ | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[33m%-30s\033[0m %s\n", $$1, $$2}' - diff --git a/deploy/k8s/admin.yml b/deploy/k8s/admin.yml new file mode 100644 index 0000000..cac3641 --- /dev/null +++ b/deploy/k8s/admin.yml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: admin-user + namespace: kube-system + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: admin-user +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- kind: ServiceAccount + name: admin-user + namespace: kube-system diff --git a/deploy/k8s/deployments/cms.yml b/deploy/k8s/deployments/cms.yml new file mode 100644 index 0000000..9cbee8a --- /dev/null +++ b/deploy/k8s/deployments/cms.yml @@ -0,0 +1,50 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cms +spec: + replicas: 1 + selector: + matchLabels: + app: cms + template: + metadata: + labels: + app: cms + spec: + containers: + - name: cms + image: regis/openedx:hawthorn + env: + - name: SERVICE_VARIANT + value: cms + ports: + - containerPort: 8000 + volumeMounts: + - mountPath: /openedx/edx-platform/lms/envs/tutor/ + name: settings-lms + - mountPath: /openedx/edx-platform/cms/envs/tutor/ + name: settings-cms + - mountPath: /openedx/config + name: config + - mountPath: /openedx/scripts + name: scripts + - mountPath: /openedx/data + name: data + #imagePullPolicy: Always + volumes: + - name: settings-lms + configMap: + name: openedx-settings-lms + - name: settings-cms + configMap: + name: openedx-settings-cms + - name: config + configMap: + name: openedx-config + - name: scripts + configMap: + name: openedx-scripts + - name: data + persistentVolumeClaim: + claimName: cms-data diff --git a/deploy/k8s/deployments/forum.yml b/deploy/k8s/deployments/forum.yml new file mode 100644 index 0000000..625f947 --- /dev/null +++ b/deploy/k8s/deployments/forum.yml @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: forum +spec: + replicas: 1 + selector: + matchLabels: + app: forum + template: + metadata: + labels: + app: forum + spec: + containers: + - name: forum + image: regis/openedx-forum:hawthorn + ports: + - containerPort: 4567 + imagePullPolicy: Always diff --git a/deploy/k8s/deployments/lms.yml b/deploy/k8s/deployments/lms.yml index 03dd41b..7b3d23f 100644 --- a/deploy/k8s/deployments/lms.yml +++ b/deploy/k8s/deployments/lms.yml @@ -14,7 +14,7 @@ spec: spec: containers: - name: lms - image: regis/openedx:latest + image: regis/openedx:hawthorn ports: - containerPort: 8000 volumeMounts: @@ -28,6 +28,7 @@ spec: name: scripts - mountPath: /openedx/data name: data + imagePullPolicy: Always volumes: - name: settings-lms configMap: diff --git a/deploy/k8s/deployments/nginx.yml b/deploy/k8s/deployments/nginx.yml index 27bdb20..a065352 100644 --- a/deploy/k8s/deployments/nginx.yml +++ b/deploy/k8s/deployments/nginx.yml @@ -12,12 +12,29 @@ spec: labels: app: nginx spec: + initContainers: + - name: clean-openedx-staticfiles + image: regis/openedx:hawthorn + command: ['rm', '-rf', '/var/www/openedx/staticfiles'] + volumeMounts: + - mountPath: /var/www/openedx/ + name: openedx-staticfiles + imagePullPolicy: Always + - name: init-openedx-staticfiles + image: regis/openedx:hawthorn + command: ['cp', '-r', '/openedx/staticfiles', '/var/www/openedx/'] + volumeMounts: + - mountPath: /var/www/openedx/ + name: openedx-staticfiles + imagePullPolicy: Always containers: - name: nginx - image: nginx:latest + image: nginx:1.13 volumeMounts: - mountPath: /etc/nginx/conf.d/ name: config + - mountPath: /var/www/openedx/ + name: openedx-staticfiles - mountPath: /openedx/data/lms name: data ports: @@ -29,6 +46,9 @@ spec: - name: config configMap: name: nginx-config + - name: openedx-staticfiles + persistentVolumeClaim: + claimName: openedx-staticfiles - name: data persistentVolumeClaim: claimName: lms-data diff --git a/deploy/k8s/services/openedx.yml b/deploy/k8s/services/cms.yml similarity index 58% rename from deploy/k8s/services/openedx.yml rename to deploy/k8s/services/cms.yml index e24e2f7..62e3ecd 100644 --- a/deploy/k8s/services/openedx.yml +++ b/deploy/k8s/services/cms.yml @@ -1,12 +1,11 @@ apiVersion: v1 kind: Service metadata: - name: openedx + name: cms spec: type: NodePort ports: - port: 8000 protocol: TCP selector: - # TODO different services and names for lms/cms - app: openedx + app: cms diff --git a/deploy/k8s/services/forum.yml b/deploy/k8s/services/forum.yml new file mode 100644 index 0000000..b3791fc --- /dev/null +++ b/deploy/k8s/services/forum.yml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: forum +spec: + type: NodePort + ports: + - port: 4567 + protocol: TCP + selector: + app: forum diff --git a/deploy/k8s/services/lms.yml b/deploy/k8s/services/lms.yml new file mode 100644 index 0000000..23d8c52 --- /dev/null +++ b/deploy/k8s/services/lms.yml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: lms +spec: + type: NodePort + ports: + - port: 8000 + protocol: TCP + selector: + app: lms diff --git a/deploy/k8s/templates/ingress.yml b/deploy/k8s/templates/ingress.yml index 5b69d04..c4dad2e 100644 --- a/deploy/k8s/templates/ingress.yml +++ b/deploy/k8s/templates/ingress.yml @@ -1,10 +1,13 @@ apiVersion: extensions/v1beta1 kind: Ingress metadata: - name: ingress + name: web spec: rules: - - host: overhang.io + {% set hosts = [LMS_HOST, "preview." + LMS_HOST, CMS_HOST] %} + {% if ACTIVATE_NOTES %}{% set hosts = hosts + ["notes." + LMS_HOST] %}{% endif %} + {% for host in hosts %} + - host: {{ host }} http: paths: - backend: @@ -13,3 +16,4 @@ spec: - backend: serviceName: nginx servicePort: 443 + {% endfor %} diff --git a/deploy/k8s/volumes/cms-data.yml b/deploy/k8s/volumes/cms-data.yml new file mode 100644 index 0000000..69a78ff --- /dev/null +++ b/deploy/k8s/volumes/cms-data.yml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: cms-data +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi diff --git a/deploy/k8s/volumes/openedx-staticfiles.yml b/deploy/k8s/volumes/openedx-staticfiles.yml new file mode 100644 index 0000000..754d0ea --- /dev/null +++ b/deploy/k8s/volumes/openedx-staticfiles.yml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: openedx-staticfiles +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi diff --git a/deploy/local/Makefile b/deploy/local/Makefile index 3fb2487..971fe9b 100644 --- a/deploy/local/Makefile +++ b/deploy/local/Makefile @@ -165,7 +165,7 @@ import-demo-course: ## Import the demo course from edX git clone https://github.com/edx/edx-demo-course --branch open-release/hawthorn.2 --depth 1 ../edx-demo-course \ && python ./manage.py cms import ../data ../edx-demo-course" -create-staff-user: ## Create a user with admin rights +create-staff-user: ## Create a user with admin rights: make create-staff-user USERNAME=... EMAIL=... $(DOCKER_COMPOSE_RUN_OPENEDX) lms /bin/bash -c "./manage.py lms manage_user --superuser --staff ${USERNAME} ${EMAIL} && ./manage.py lms changepassword ${USERNAME}" portainer: ## Run Portainer (https://portainer.io), a UI for container supervision diff --git a/docs/img/k8s-dashboard.png b/docs/img/k8s-dashboard.png new file mode 100644 index 0000000..72b96e8 Binary files /dev/null and b/docs/img/k8s-dashboard.png differ diff --git a/docs/img/virtualbox-minikube-system.png b/docs/img/virtualbox-minikube-system.png new file mode 100644 index 0000000..cfbbf3d Binary files /dev/null and b/docs/img/virtualbox-minikube-system.png differ diff --git a/docs/k8s.rst b/docs/k8s.rst index 189c779..f0db67e 100644 --- a/docs/k8s.rst +++ b/docs/k8s.rst @@ -3,7 +3,7 @@ Kubernetes deployment ===================== -With the same docker images we created for :ref:`single server deployment ` and :ref:`local development `, we can launch an Open edX platform on Kubernetes. Always in 1 click, of course :) +With the same docker images we created for :ref:`single server deployment ` and :ref:`local development `, we can launch an Open edX platform on Kubernetes. Always in 1 click, of course :) :: @@ -25,6 +25,11 @@ Start Minikube:: minikube start +When minikube starts, it spawns a virtual machine (VM) which you can configure in your VM manager: on most platforms, this is Virtualbox. You should configure your VM to have at least 4Gb RAM; otherwise, database migrations will crash halfway, and that's a nasty issue... + +.. image:: img/virtualbox-minikube-system.png + :alt: Virtualbox memory settings for Minikube + Ingress addon must be installed:: minikube addons enable ingress @@ -33,6 +38,12 @@ At any point, access a UI to view the state of the platform with:: minikube dashboard +With Kubernetes, your Open edX platform will not be available at localhost or studio.localhost. Instead, you will have to access your platform with the domain names you specified for the LMS and the CMS. To do so on a local computer, you will need to add the following line to /etc/hosts:: + + MINIKUBEIP yourdomain.com studio.yourdomain.com preview.yourdomain.com notes.yourdomain.com + +where ``MINIKUBEIP`` should be replaced by the result of the command ``minikube ip``. + In the following, all commands should be run inside the ``deploy/k8s`` folder:: cd deploy/k8s @@ -43,3 +54,46 @@ Quickstart Launch the platform on k8s in 1 click:: make all + +All Kubernetes resources are associated to the "openedx" namespace. If you don't see anything in the Kubernetes dashboard, you are probably looking at the wrong namespace... 😉 + +.. image:: img/k8s-dashboard.png + :alt: Kubernetes dashboard ("openedx" namespace) + +Upgrading +--------- + +After pulling updates from the Tutor repository, you can apply changes with:: + + make upgrade + +Accessing the Kubernetes dashboard +---------------------------------- + +Depending on your Kubernetes provider, you may need to create a dashboard yourself. To do so, run:: + + make k8s-dashboard + +Then, you will have to create an admin user:: + + make k8s-admin + +Print the admin token required for authentication, and copy its value:: + + make k8s-admin-token + +Create a proxy to the Kubernetes API server:: + + k8s-proxy + +Use the token to log in the dashboard at the following url: http://localhost:8001/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy/ + +Missing features +---------------- + +For now, the following features from the local deployment are not supported: + +- HTTPS certificates +- Xqueue + +Kubernetes deployment is under intense development, and these features should be implemented pretty soon. Stay tuned 🤓