From c903ab2b12fc19e9902df085d7e976382123b778 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Mon, 3 Dec 2018 19:59:09 +0100 Subject: [PATCH] Migrate openedx-docker project to Tutor :woman_teacher: The project gets a new name and some proper documentation. Build/Deploy are now properly separated. --- .gitignore | 8 +- .travis.yml | 7 +- Makefile | 268 ++---------- README.md | 414 +----------------- android/.gitignore | 2 + android/Makefile | 41 ++ .../templates}/edx.properties | 0 .../templates}/gradle.properties | 0 .../templates}/universal.yaml | 0 build/.gitignore | 1 + build/Makefile | 59 +++ {android => build/android}/Dockerfile | 0 {android => build/android}/edx.properties | 0 build/configurator/Dockerfile | 14 + .../configurator/bin/configurator | 70 ++- .../configurator}/bin/docker-entrypoint.sh | 1 - {forum => build/forum}/Dockerfile | 0 {notes => build/notes}/Dockerfile | 0 {openedx => build/openedx}/Dockerfile | 59 ++- .../openedx}/bin/docker-entrypoint.sh | 0 {openedx => build/openedx}/bin/openedx-assets | 0 .../openedx}/requirements/.gitignore | 0 .../openedx}/requirements/README | 0 .../openedx/settings}/cms/__init__.py | 0 .../openedx}/settings/cms/assets.py | 0 .../openedx/settings}/lms/__init__.py | 0 .../openedx}/settings/lms/assets.py | 0 {openedx => build/openedx}/themes/.gitignore | 0 {xqueue => build/xqueue}/Dockerfile | 0 configurator/Dockerfile | 16 - deploy/.gitignore | 1 + deploy/Makefile | 22 + deploy/singleserver/.gitignore | 3 + deploy/singleserver/Makefile | 195 +++++++++ .../singleserver/scripts.yml | 0 .../singleserver}/templates/Makefile.env | 1 - .../singleserver/templates/docker-compose.yml | 93 ++-- .../templates/letsencrypt/certonly.sh | 0 .../templates/mysql/auth.env | 0 .../templates/nginx/cms.conf | 15 +- .../templates/nginx/extra.conf | 0 .../templates/nginx/lms.conf | 15 +- .../templates/notes/universal.py | 0 .../templates/openedx/cms.auth.json | 0 .../templates/openedx/cms.env.json | 0 .../templates/openedx/lms.auth.json | 0 .../templates/openedx/lms.env.json | 0 .../templates/openedx/oauth2.sh | 0 .../templates/openedx/provision.sh | 0 .../templates/openedx/stats | 0 .../openedx/universal}/cms/__init__.py | 0 .../openedx/universal/cms/development.py | 0 .../openedx/universal/cms/production.py | 0 .../openedx/universal}/lms/__init__.py | 0 .../openedx/universal/lms/development.py | 0 .../openedx/universal/lms/production.py | 0 .../templates/xqueue/universal.py | 0 docker-compose-android.yml | 10 - docker-compose-notes.yml | 18 - docker-compose-portainer.yml | 12 - docker-compose-xqueue.yml | 27 -- docs/.gitignore | 1 + docs/Makefile | 19 + docs/conf.py | 173 ++++++++ docs/customise.rst | 84 ++++ docs/dev.rst | 70 +++ docs/index.rst | 77 ++++ docs/missing.rst | 8 + docs/options.rst | 72 +++ docs/requirements.rst | 15 + docs/step.rst | 112 +++++ docs/troubleshooting.rst | 73 +++ 72 files changed, 1208 insertions(+), 868 deletions(-) create mode 100644 android/.gitignore create mode 100644 android/Makefile rename {configurator/templates/android => android/templates}/edx.properties (100%) rename {configurator/templates/android => android/templates}/gradle.properties (100%) rename {configurator/templates/android => android/templates}/universal.yaml (100%) create mode 100644 build/.gitignore create mode 100644 build/Makefile rename {android => build/android}/Dockerfile (100%) rename {android => build/android}/edx.properties (100%) create mode 100644 build/configurator/Dockerfile rename configurator/bin/configure.py => build/configurator/bin/configurator (76%) rename {configurator => build/configurator}/bin/docker-entrypoint.sh (88%) rename {forum => build/forum}/Dockerfile (100%) rename {notes => build/notes}/Dockerfile (100%) rename {openedx => build/openedx}/Dockerfile (61%) rename {openedx => build/openedx}/bin/docker-entrypoint.sh (100%) rename {openedx => build/openedx}/bin/openedx-assets (100%) rename {openedx => build/openedx}/requirements/.gitignore (100%) rename {openedx => build/openedx}/requirements/README (100%) rename {configurator/templates/openedx/universal => build/openedx/settings}/cms/__init__.py (100%) rename {openedx => build/openedx}/settings/cms/assets.py (100%) rename {configurator/templates/openedx/universal => build/openedx/settings}/lms/__init__.py (100%) rename {openedx => build/openedx}/settings/lms/assets.py (100%) rename {openedx => build/openedx}/themes/.gitignore (100%) rename {xqueue => build/xqueue}/Dockerfile (100%) delete mode 100644 configurator/Dockerfile create mode 100644 deploy/.gitignore create mode 100644 deploy/Makefile create mode 100644 deploy/singleserver/.gitignore create mode 100644 deploy/singleserver/Makefile rename docker-compose-scripts.yml => deploy/singleserver/scripts.yml (100%) rename {configurator => deploy/singleserver}/templates/Makefile.env (72%) rename docker-compose.yml => deploy/singleserver/templates/docker-compose.yml (59%) rename {configurator => deploy}/templates/letsencrypt/certonly.sh (100%) rename {configurator => deploy}/templates/mysql/auth.env (100%) rename {configurator => deploy}/templates/nginx/cms.conf (83%) rename {configurator => deploy}/templates/nginx/extra.conf (100%) rename {configurator => deploy}/templates/nginx/lms.conf (87%) rename {configurator => deploy}/templates/notes/universal.py (100%) rename {configurator => deploy}/templates/openedx/cms.auth.json (100%) rename {configurator => deploy}/templates/openedx/cms.env.json (100%) rename {configurator => deploy}/templates/openedx/lms.auth.json (100%) rename {configurator => deploy}/templates/openedx/lms.env.json (100%) rename {configurator => deploy}/templates/openedx/oauth2.sh (100%) rename {configurator => deploy}/templates/openedx/provision.sh (100%) rename {configurator => deploy}/templates/openedx/stats (100%) rename {openedx/settings => deploy/templates/openedx/universal}/cms/__init__.py (100%) rename {configurator => deploy}/templates/openedx/universal/cms/development.py (100%) rename {configurator => deploy}/templates/openedx/universal/cms/production.py (100%) rename {openedx/settings => deploy/templates/openedx/universal}/lms/__init__.py (100%) rename {configurator => deploy}/templates/openedx/universal/lms/development.py (100%) rename {configurator => deploy}/templates/openedx/universal/lms/production.py (100%) rename {configurator => deploy}/templates/xqueue/universal.py (100%) delete mode 100644 docker-compose-android.yml delete mode 100644 docker-compose-notes.yml delete mode 100644 docker-compose-portainer.yml delete mode 100644 docker-compose-xqueue.yml create mode 100644 docs/.gitignore create mode 100644 docs/Makefile create mode 100644 docs/conf.py create mode 100644 docs/customise.rst create mode 100644 docs/dev.rst create mode 100644 docs/index.rst create mode 100644 docs/missing.rst create mode 100644 docs/options.rst create mode 100644 docs/requirements.rst create mode 100644 docs/step.rst create mode 100644 docs/troubleshooting.rst diff --git a/.gitignore b/.gitignore index 615722b..d55b574 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,5 @@ .*.swp -config/ -data-*/ -TODO -openedx/requirements/private.txt -.env !.gitignore +/config.json +/data*/ +/TODO diff --git a/.travis.yml b/.travis.yml index 5a02a12..01b9c16 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,13 +2,10 @@ language: minimal services: - docker script: - - make configure SILENT=1 CONFIGURE_OPTS="-e SETTING_ACTIVATE_NOTES=1 -e SETTING_ACTIVATE_XQUEUE=1" - - make build - - make databases - - make assets + - make travis deploy: provider: script - script: docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && make push + script: cd build/ && docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && make push on: all_branches: true condition: $TRAVIS_BRANCH =~ ^master|release\/.*$ diff --git a/Makefile b/Makefile index 78fe08a..6ac9b54 100644 --- a/Makefile +++ b/Makefile @@ -1,257 +1,43 @@ -.PHONY: all android configure build update migrate run +.PHONY: android build .DEFAULT_GOAL := help - -PWD ?= $$(pwd) +PWD = $$(pwd) USERID ?= $$(id -u) -EDX_PLATFORM_SETTINGS ?= universal.production -DOCKER_COMPOSE = docker-compose -f docker-compose.yml --include $(PWD)/config/Makefile.env -post_configure_targets = -ifneq ($(DISABLE_STATS), 1) - post_configure_targets += stats -endif -ifeq ($(ACTIVATE_HTTPS), 1) - post_configure_targets += https-certificate -endif -extra_migrate_targets = -ifeq ($(ACTIVATE_XQUEUE), 1) - extra_migrate_targets += migrate-xqueue - DOCKER_COMPOSE += -f docker-compose-xqueue.yml -endif -ifeq ($(ACTIVATE_NOTES), 1) - extra_migrate_targets += migrate-notes - DOCKER_COMPOSE += -f docker-compose-notes.yml -endif -ifeq ($(ACTIVATE_PORTAINER), 1) - DOCKER_COMPOSE += -f docker-compose-portainer.yml -endif +build: ## Build all docker images + cd build/ && make build -DOCKER_COMPOSE_RUN = $(DOCKER_COMPOSE) run --rm -DOCKER_COMPOSE_RUN_OPENEDX = $(DOCKER_COMPOSE_RUN) -e SETTINGS=$(EDX_PLATFORM_SETTINGS) \ - --volume="$(PWD)/openedx/themes:/openedx/themes" -ifneq ($(EDX_PLATFORM_PATH),) - DOCKER_COMPOSE_RUN_OPENEDX += -e USERID=$(USERID) --volume="$(EDX_PLATFORM_PATH):/openedx/edx-platform" -endif - -DOCKER_COMPOSE_RUN_LMS = $(DOCKER_COMPOSE_RUN_OPENEDX) -p 8000:8000 lms -DOCKER_COMPOSE_RUN_CMS = $(DOCKER_COMPOSE_RUN_OPENEDX) -p 8001:8001 cms - -##################### Running Open edX - -# other targets are not listed as requirements in order to reload the env file -all: configure ## Configure and run a full-featured platform - @$(MAKE) post_configure - @$(MAKE) update - @$(MAKE) databases - @$(MAKE) assets - @$(MAKE) daemonize - @echo "All set \o/ You can access the LMS at http://localhost and the CMS at http://studio.localhost" - -run: ## Run the complete platform - $(DOCKER_COMPOSE) up -up: run - -daemonize: ## Run the complete platform, with daemonization - $(DOCKER_COMPOSE) up -d - @echo "Daemon is up and running" -daemon: daemonize - -stop: ## Stop all services - $(DOCKER_COMPOSE) rm --stop --force - -##################### Configuration - -configure: build-configurator ## Configure the environment prior to running the platform - docker run --rm -it --volume="$(PWD)/config:/openedx/config" \ - -e USERID=$(USERID) -e SILENT=$(SILENT) $(CONFIGURE_OPTS) \ +configure: + @$(MAKE) -s -C build/ build-configurator 1> /dev/null + @docker run --rm -it \ + --volume="$(PWD):/openedx/config/" \ + -e USERID=$(USERID) -e SILENT=$(SILENT) \ regis/openedx-configurator:hawthorn -post_configure: $(post_configure_targets) +singleserver: ## Configure and run a ready-to-go Open edX platform + cd deploy/ && make all -##################### Database +android: ## Configure and build a development Android app + cd android/ && make all -databases: provision-databases migrate provision-oauth2 ## Bootstrap databases +travis: + cd build && make build + cd deploy/singleserver \ + && make configure SILENT=1 CONFIGURE_OPTS="-e SETTING_ACTIVATE_NOTES=1 -e SETTING_ACTIVATE_XQUEUE=1" \ + && make databases \ + && make assets -provision-databases: ## Create necessary databases and users - $(DOCKER_COMPOSE_RUN) lms /openedx/config/provision.sh -provision-oauth2: ## Create users for SSO between services - $(DOCKER_COMPOSE_RUN) lms /openedx/config/oauth2.sh +upgrade-to-tutor: + @(stat config/config.json > /dev/null 2>&1 && (\ + echo "You are running an older version of Tutor. Now migrating to the latest version" \ + && echo "Moving config/config.json to ./config.json" && mv config/config.json config.json \ + && echo "Moving config/ to deploy/env/" && mv config/ deploy/env/ \ + && ((ls openedx/themes/* > /dev/null 2>&1 && echo "Moving openedx/themes/* to build/openedx/themes/" && mv openedx/themes/* build/openedx/themes/) || true) \ + && echo "Done migrating to tutor. This command will not be run again."\ + )) || true -migrate: migrate-openedx migrate-forum $(extra_migrate_targets) ## Perform all database migrations -migrate-openedx: ## Perform database migrations on LMS/CMS - $(DOCKER_COMPOSE_RUN) lms bash -c "dockerize -wait tcp://mysql:3306 -timeout 20s && ./manage.py lms migrate" - $(DOCKER_COMPOSE_RUN) cms bash -c "dockerize -wait tcp://mysql:3306 -timeout 20s && ./manage.py cms migrate" - $(MAKE) reindex-courses -migrate-forum: ## Perform database migrations on discussion forums - $(DOCKER_COMPOSE_RUN) forum bash -c "bundle exec rake search:initialize && \ - bundle exec rake search:rebuild_index" -migrate-notes: ## Perform database migrations for the Notes service - $(DOCKER_COMPOSE_RUN) notes ./manage.py migrate -migrate-xqueue: ## Perform database migrations for the XQueue service - $(DOCKER_COMPOSE_RUN) xqueue ./manage.py migrate -reindex-courses: ## Refresh course index so they can be found in the LMS search engine - $(DOCKER_COMPOSE_RUN) cms ./manage.py cms reindex_course --all --setup - -##################### Static assets - -# To collect assets we don't rely on the "paver update_assets" command because -# webpack collection incorrectly sets the NODE_ENV variable when using custom -# settings. Thus, each step must be performed separately. This should be fixed -# in the next edx-platform release thanks to https://github.com/edx/edx-platform/pull/18430/ - -assets: ## Generate production-ready static assets - docker-compose -f docker-compose-scripts.yml run --rm \ - --volume=$(PWD)/data/openedx:/tmp/openedx/ openedx bash -c \ - "rm -rf /tmp/openedx/staticfiles \ - && cp -r /openedx/staticfiles /tmp/openedx" -assets-development: ## Generate static assets for local development - $(DOCKER_COMPOSE_RUN_OPENEDX) --no-deps lms bash -c "openedx-assets build --env=dev" -assets-development-lms: - $(DOCKER_COMPOSE_RUN_OPENEDX) --no-deps lms bash -c "openedx-assets build --env=dev --system lms" -assets-development-cms: - $(DOCKER_COMPOSE_RUN_OPENEDX) --no-deps lms bash -c "openedx-assets build --env=dev --system cms" -watch-themes: ## Watch for changes in your themes and build development assets - $(DOCKER_COMPOSE_RUN_OPENEDX) --no-deps lms openedx-assets watch-themes --env dev - -##################### Information - -# Obtained by running "echo '\033' in a shell ESCAPE =  help: ## Print this help @grep -E '^([a-zA-Z_-]+:.*?## .*|######* .+)$$' Makefile \ | sed 's/######* \(.*\)/\n $(ESCAPE)[1;31m\1$(ESCAPE)[0m/g' \ | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[33m%-30s\033[0m %s\n", $$1, $$2}' - -info: ## Print some information about the current install, for debugging - uname -a - @echo "-------------------------" - git rev-parse HEAD - @echo "-------------------------" - docker version - @echo "-------------------------" - docker-compose --version - @echo "-------------------------" - echo $$EDX_PLATFORM_PATH - echo $$EDX_PLATFORM_SETTINGS - - -#################### Logging - -logs: ## Print all logs from a service since it started. E.g: "make logs service=lms", "make logs service=nginx" - $(DOCKER_COMPOSE) logs $(service) -tail: ## Similar to "tail" on the logs of a service. E.g: "make tail service=lms", "make tail service=nginx" - $(DOCKER_COMPOSE) logs --tail=10 $(service) -tail-follow: ## Similar to "tail -f" on the logs of a service. E.g: "make tail-follow service=lms", "make tail-follow service=nginx" - $(DOCKER_COMPOSE) logs --tail=10 -f $(service) - -#################### Docker image building & updating - -update: ## Download most recent images - $(DOCKER_COMPOSE) pull - -build: build-openedx build-configurator build-forum build-notes build-xqueue build-android ## Build all docker images - -openedx_build_args = -ifdef EDX_PLATFORM_REPOSITORY - openedx_build_args += --build-arg="EDX_PLATFORM_REPOSITORY=$(EDX_PLATFORM_REPOSITORY)" -endif -ifdef EDX_PLATFORM_VERSION - openedx_build_args += --build-arg="EDX_PLATFORM_VERSION=$(EDX_PLATFORM_VERSION)" -endif - -build-openedx: ## Build the Open edX docker image - docker build -t regis/openedx:latest -t regis/openedx:hawthorn $(openedx_build_args) openedx/ -build-configurator: ## Build the configurator docker image - docker build -t regis/openedx-configurator:latest -t regis/openedx-configurator:hawthorn configurator/ -build-forum: ## Build the forum docker image - docker build -t regis/openedx-forum:latest -t regis/openedx-forum:hawthorn forum/ -build-notes: ## Build the Notes docker image - docker build -t regis/openedx-notes:latest -t regis/openedx-notes:hawthorn notes/ -build-xqueue: ## Build the Xqueue docker image - docker build -t regis/openedx-xqueue:latest -t regis/openedx-xqueue:hawthorn xqueue/ -build-android: ## Build the docker image for Android - docker build -t regis/openedx-android:latest android/ - -################### Pushing images to docker hub - -push: push-openedx push-configurator push-forum push-notes push-xqueue push-android ## Push all images to dockerhub -push-openedx: ## Push Open edX images to dockerhub - docker push regis/openedx:hawthorn - docker push regis/openedx:latest -push-configurator: ## Push configurator image to dockerhub - docker push regis/openedx-configurator:hawthorn - docker push regis/openedx-configurator:latest -push-forum: ## Push forum image to dockerhub - docker push regis/openedx-forum:hawthorn - docker push regis/openedx-forum:latest -push-notes: ## Push notes image to dockerhub - docker push regis/openedx-notes:hawthorn - docker push regis/openedx-notes:latest -push-xqueue: ## Push Xqueue image to dockerhub - docker push regis/openedx-xqueue:hawthorn - docker push regis/openedx-xqueue:latest -push-android: ## Push the Android image to dockerhub - docker push regis/openedx-android:latest - -dockerhub: build push ## Build and push all images to dockerhub - -##################### Development - -lms: ## Open a bash shell in the LMS - $(DOCKER_COMPOSE_RUN_LMS) bash -cms: ## Open a bash shell in the CMS - $(DOCKER_COMPOSE_RUN_CMS) bash - -lms-python: ## Open a python shell in the LMS - $(DOCKER_COMPOSE_RUN_OPENEDX) lms ./manage.py lms shell -lms-shell: lms-python -lms-runserver: ## Run a local webserver, useful for debugging - $(DOCKER_COMPOSE_RUN_LMS) ./manage.py lms runserver 0.0.0.0:8000 -cms-python: ## Open a python shell in the CMS - $(DOCKER_COMPOSE_RUN_OPENEDX) cms ./manage.py cms shell -cms-shell: cms-python -cms-runserver: ## Run a local webserver, useful for debugging - $(DOCKER_COMPOSE_RUN_CMS) ./manage.py cms runserver 0.0.0.0:8001 - -restart-openedx: ## Restart lms, cms, and workers - docker-compose restart lms lms_worker cms cms_worker - -##################### SSL/TLS (HTTPS certificates) - -https_command = docker run --rm -it \ - --volume="$(PWD)/config/letsencrypt/:/openedx/letsencrypt/config/" \ - --volume="$(PWD)/data/letsencrypt/:/etc/letsencrypt/" \ - -p "80:80" -certbot_image = certbot/certbot:latest - -https-certificate: ## Generate https certificates - $(https_command) --entrypoint "/openedx/letsencrypt/config/certonly.sh" $(certbot_image) - -https-certificate-renew: ## Renew https certificates - $(https_command) $(certbot_image) renew - -#################### Android application - -android: ## Build the Android app, for development - @docker-compose -f docker-compose-android.yml run --rm android - @echo "Your APK file is ready: ./data/android/$(shell ls data/android/*.apk)" - -android-release: ## Build the final Android app (beta) - # Note that this requires that you edit ./config/android/gradle.properties - docker-compose -f docker-compose-android.yml run --rm android ./gradlew assembleProdRelease - -##################### Additional commands - -stats: ## Collect anonymous information about the platform - @docker run --rm -it --volume="$(PWD)/config:/openedx/config" \ - regis/openedx-configurator:hawthorn /openedx/config/openedx/stats 2> /dev/null|| true - -import-demo-course: ## Import the demo course from edX - $(DOCKER_COMPOSE_RUN_OPENEDX) cms /bin/bash -c " \ - git clone https://github.com/edx/edx-demo-course --branch open-release/hawthorn.2 --depth 1 ../edx-demo-course \ - && python ./manage.py cms import ../data ../edx-demo-course" - -create-staff-user: ## Create a user with admin rights - $(DOCKER_COMPOSE_RUN_OPENEDX) lms /bin/bash -c "./manage.py lms manage_user --superuser --staff ${USERNAME} ${EMAIL} && ./manage.py lms changepassword ${USERNAME}" diff --git a/README.md b/README.md index 44f8da8..1d9e217 100644 --- a/README.md +++ b/README.md @@ -1,419 +1,15 @@ -# Open edX 1-click install for everyone +# Tutor: Open edX 1-click install for everyone + ![Build status](https://img.shields.io/travis/regisb/openedx-docker.svg) ![GitHub issues](https://img.shields.io/github/issues/regisb/openedx-docker.svg) ![GitHub closed issues](https://img.shields.io/github/issues-closed/regisb/openedx-docker.svg?colorB=brightgreen) -This is a one-click install of [Open edX](https://openedx.org), both for production and local development, inside docker containers. As a bonus, this also builds a mobile Android app for your platform. +Tutor is a one-click install of [Open edX](https://openedx.org), both for production and local development, inside docker containers. [![asciicast](https://asciinema.org/a/6DowVk4iJf3AJ2m8xlXDWJKh3.png)](https://asciinema.org/a/6DowVk4iJf3AJ2m8xlXDWJKh3) -## Getting started +# Quickstart git clone https://github.com/regisb/openedx-docker cd openedx-docker/ - make all - -## That's it? - -Yes :) When running `make all`, you will be asked some questions about the configuration of your Open edX platform. Then, all the components for a functional Open edX platform will be downloaded and assembled to and you will have both an LMS and a CMS running behind a web server on port 80, ready for production. You should be able to access your platform at the address you gave during the configuration phase. - -All of this without touching your host environment! You don't even need root access. - -To be honest, I really don't like 1-click installs :-p They tend to hide much of the important details. So I strongly recommend you read the more detailed instructions below to understand what is going on exactly and to troubleshoot potential issues. Also, instructions are given to setup a local development environment. - -This might seem too simple to be true, but there's no magic -- just good packaging of already existing Open edX code. The code for building the Docker images is 100% available and fits in less than 1000 lines of code, in this repository. - -## Optional features - -Some optional features may be activated or deactivated during the interactive configuration step. These features change configuration files (during the `configure` step) as well as make targets. - -### SSL/TLS certificates for HTTPS access - -By activating this feature, a free SSL/TLS certificate from the [Let's Encrypt](https://letsencrypt.org/) certificate authority will be created for your platform. With this feature, **your platform will no longer be accessible in HTTP**. Calls to http urls will be redirected to https url. - -The following DNS records must exist and point to your server: - - LMS_HOST (e.g: myopenedx.com) - preview.LMS_HOST (e.g: preview.myopenedx.com) - CMS_HOST (e.g: studio.myopenedx.com) - -Thus, **this feature will (probably) not work in development** because the DNS records will (probably) not point to your development machine. - -To download the certificate manually, run: - - make https-certificate - -To renew the certificate, run this command once per month: - - make https-certificate-renew - -### Student notes - -With [notes](https://edx.readthedocs.io/projects/open-edx-building-and-running-a-course/en/open-release-hawthorn.master/exercises_tools/notes.html?highlight=notes), students can annotate portions of the courseware. - -![Notes in action](https://edx.readthedocs.io/projects/open-edx-building-and-running-a-course/en/open-release-hawthorn.master/_images/SFD_SN_bodyexample.png) - -You should beware that the `notes.` domain name should be activated and point to your server. For instance, if your LMS is hosted at [myopenedx.com](), the notes service should be found at [notes.myopenedx.com](). Student browsers will access this domain name to fetch their notes. - -### Xqueue - -[Xqueue](https://github.com/edx/xqueue) is for grading problems with external services. If you don't know what it is, you probably don't need it. - -Note: in previous releases of openedx-docker, xqueue was run for all platforms. It is now an optional feature. - -### Docker container web UI with [Portainer](https://portainer.io/) - -Portainer is a web UI for managing docker containers. It lets you view your entire Open edX platform at a glace. Try it! It's really cool. - -![Portainer demo](https://portainer.io/images/screenshots/portainer.gif) - -After launching your platfom, the web UI will be available at [http://portainer.localhost](http://portainer.localhost) and http://portainer.YOUR_LMS_HOST. You will be asked to define a password for the admin user. Then, select a "Local environment" to work on and hit "Connect". You're done! Select the "local" group to view all running containers. Amon many other things, you'll be able to view the logs for each container, which is really useful. - -### Android app (beta) - -The Android app for your platform can be easily built in just one command: - - make android - -If all goes well, the debuggable APK for your platform should then be available in ./data/android. To obtain a release APK, you will need to obtain credentials from the app store and add them to `config/android/gradle.properties`. Then run: - - make android-release - -Building the Android app for an Open edX platform is currently labeled as a **beta feature** because it was not fully tested yet. In particular, there is no easy mechanism for overriding the edX assets in the mobile app. This is still a work-in-progress. - -### Stats - -By default, the install script will collect some information about your install and send it to a private server. The only transmitted information are the LMS domain name and the ID of the install. To disable stats collection, define the following environment variable: - - export DISABLE_STATS=1 - -If you decide to disable stats, please send me a message to tell me about your platform! - -## Requirements - -The only prerequisite for running this is a working docker install. You will need both docker and docker-compose. Follow the instructions from the official documentation: - -- [Docker](https://docs.docker.com/engine/installation/) -- [Docker compose](https://docs.docker.com/compose/install/) - -Note that the production web server container will bind to port 80, so if you already have a web server running (Apache or Nginx, for instance), you should stop it. - -You should be able to run Open edX on any platform that supports Docker and Python, including Mac OS and Windows. For now, only Ubuntu 16.04 was tested but we have no reason to believe the install would not work on a different OS. - -At a minimum, the server running the containers should have 4 Gb of RAM; otherwise, the deployment procedure will crash during migrations (see the [troubleshooting](#troubleshooting) section). - -Also, the host running the containers should be a 64 bit platform. (images are not built for i386 systems) - -## Step-by-step install - -### Configure - - make configure - -This is the only non-automatic step in the install process. You will be asked various questions about your Open edX platform and appropriate configuration files will be generated. If you would like to automate this step then you should run `make configure` interactively once. After that, you will have a `config.json` file at the root of the repository. Just upload it to wherever you want to run Open edX and then run `make configure SILENT=1` instead of `make configure`. All values from `config.json` will be automatically loaded. - -### Download - - make update - -You will need to download the docker images from [Docker Hub](https://hub.docker.com/r/regis/openedx/). Depending on your bandwidth, this might take a long time. Minor image updates will be incremental, and thus much faster. - -### Database creation, migrations and collection of static assets - - make databases - make assets - -These commands should be run just once. They will create the required databases tables, apply database migrations and make sure that static assets, such as images, stylesheets and Javascript dependencies, can be served by the nginx container. - -If migrations are stopped with a `Killed` message, this certainly means the docker containers don't have enough RAM. See the [troubleshooting](#troubleshooting) section. - -### Running Open edX - - make run - -This will launch the various docker containers required for your Open edX platform. The LMS and the Studio will then be reachable at the domain name you specified during the configuration step. You can also access them at [http://localhost](http://localhost) and [http://studio.localhost](http://studio.localhost). - -## Additional commands - -All available commands can be listed by running: - - make help - -### Creating a new user with staff and admin rights - -You will most certainly need to create a user to administer the platform. Just run: - - make create-staff-user USERNAME=yourusername EMAIL=user@email.com - -You will asked to set the user password interactively. - -### Importing the demo course - -On a fresh install, your platform will not have a single course. To import the [Open edX demo course](https://github.com/edx/edx-demo-course ), run: - - make import-demo-course - -### Daemonizing - -In production, you will probably want to daemonize the services. Instead of `make run`, run: - - make daemonize - -And then, to stop all services: - - make stop - -### Updating the course search index - -The course search index can be updated with: - - make reindex-courses - -Run this command periodically to ensure that course search results are always up-to-date. - -### Logging - -To view the logs from all containers use the [`docker-compose logs`](https://docs.docker.com/compose/reference/logs/) command: - - docker-compose logs -f - -To view the logs from just one container, for instance the web server: - - docker-compose logs -f nginx - -The last commands produce the logs since the creation of the containers, which can be a lot. Similar to a `tail -f`, you can run: - - docker-compose logs --tail=0 -f - -### Debugging - -Open a bash shell in the lms or the cms: - - make lms - make cms - -Open a python shell in the lms or the cms: - - make lms-python - make cms-python - -## For developers - -In addition to running Open edX in production, you can use the docker containers for local development. This means you can hack on Open edX without setting up a Virtual Machine. Essentially, this replaces the devstack provided by edX. - -To begin with, define development settings: - - export EDX_PLATFORM_SETTINGS=universal.development - -### Run a local webserver - - make lms-runserver - make cms-runserver - -### Open a bash shell - - make lms - make cms - -### Debug edx-platform - -If you have one, you can point to a local version of [edx-platform](https://github.com/edx/edx-platform/) on your host machine: - - export EDX_PLATFORM_PATH=/path/to/your/edx-platform - -Note that you should use an absolute path here, not a relative path (e.g: `/path/to/edx-platform` and not `../edx-platform`). - -All development commands will then automatically mount your local repo. For instance, you can add a `import pdb; pdb.set_trace()` breakpoint anywhere in your code and run: - - make lms-runserver - -Note: containers are built on the Hawthorn release. If you are working on a different version of Open edX, you will have to rebuild the images with the right `EDX_PLATFORM_VERSION` argument. You may also want to change the `EDX_PLATFORM_REPOSITORY` argument to point to your own fork of edx-platform. - -With a customised edx-platform repo, you must be careful to have settings that are compatible with the docker environment. You are encouraged to copy the `universal.development` settings files to our own repo: - - cp -r config/openedx/universal/lms/ /path/to/edx-platform/lms/envs/universal - cp -r config/openedx/universal/cms/ /path/to/edx-platform/cms/envs/universal - -You can then run your platform with the `universal.development` settings. - -### Develop customised themes - -Run a local webserver: - - make lms-runserver - -Watch the themes folders for changes: - - make watch-themes - -Make changes to `openedx/themes/yourtheme`: the theme assets should be automatically recompiled and visible at http://localhost:8000. - -### Assets management - -Assets building and collecting is made more difficult by the fact that development settings are [incorrectly loaded in Hawthorn](https://github.com/edx/edx-platform/pull/18430/files). This should be fixed in the next Open edX release. Meanwhile, do not run `paver update_assets` while in development mode. When working locally on a theme, build assets by running in the container: - - openedx-assets build - -This command will take quite some time to run. You can speed up this process by running only part of the full build. Run `openedx-assets -h` for more information. - -## Customising the `openedx` docker image - -The LMS and the CMS all run from the `openedx` docker image. The base image is downloaded from [Docker Hub](https://hub.docker.com/r/regis/openedx/) when we run `make update` (or `make all`). But you can also customise and build the image yourself. The base image is built with: - - make build-openedx - -The following sections describe how to modify various aspects of the docker image. After you have built your own image, you can run it as usual: - - make run - -### Custom themes - -Comprehensive theming is enabled by default. Put your themes in `openedx/themes`: - - openedx/themes/ - mycustomtheme1/ - cms/ - ... - lms/ - ... - mycustomtheme2/ - ... - -Then you must rebuild the openedx Docker image: - - make build-openedx - -Make sure the assets can be served by the web server: - - make assets - -Finally, follow the [Open edX documentation to enable your themes](https://edx.readthedocs.io/projects/edx-installing-configuring-and-running/en/latest/configuration/changing_appearance/theming/enable_themes.html#apply-a-theme-to-a-site). - -### Extra xblocks and requirements - -Additional requirements can be added to the `openedx/requirements/private.txt` file. For instance: - - echo "git+https://github.com/open-craft/xblock-poll.git" >> openedx/requirements/private.txt - -Then, the `openedx` docker image must be rebuilt: - - make build-openedx - -To install xblocks from a private repository that requires authentication, you must first clone the repository inside the `openedx/requirements` folder on the host: - - git clone git@github.com:me/myprivaterepo.git ./openedx/requirements/myprivaterepo - -Then, declare your extra requirements with the `-e` flag in `openedx/requirements/private.txt` : - - echo "-e ./myprivaterepo" >> openedx/requirements/private.txt - -### Forked version of edx-platform - -You may want to run your own flavor of edx-platform instead of the [official version](https://github.com/edx/edx-platform/). To do so, you will have to re-build the openedx image with the proper environment variables pointing to your repository and version: - - EDX_PLATFORM_REPOSITORY=https://mygitrepo/edx-platform.git EDX_PLATFORM_VERSION=my-tag-or-branch make build-openedx - -You can then restart the services which will now be running your forked version of edx-platform: - - make restart-openedx - -Note that your release must be a fork of Hawthorn in order to work. Otherwise, you may have important compatibility issues with other services. - -### Running a different Docker image instead of [regis/openedx](https://hub.docker.com/r/regis/openedx/) - -This is for people who have an account on [hub.docker.com](https://hub.docker.com) or a private image registry. You can build your image and push it to your repo. Then add the following content to the `.env` file: - - OPENEDX_DOCKER_IMAGE=myusername/myimage:mytag - -Your own image will be used next time you run `make run`. - -Note that the `make build` and `make push` command will no longer work as you expect and that you are responsible for building and pushing the image yourself. - -## Maintainers - -The images are built, tagged and uploaded to Docker Hub in one command: - - make dockerhub - -## Help/Troubleshooting - -### "Cannot start service nginx: driver failed programming external connectivity" - -The containerized Nginx needs to listen to ports 80 and 443 on the host. If there is already a webserver, such as Apache or Nginx, running on the host, the nginx container will not be able to start. There are two solutions: - -1. Stop Apache or Nginx on the host: - - sudo systemctl stop apache2 - sudo systemctl stop nginx - -However, you might now want to do that if you need a webserver for running non-Open edX related applications. In such cases... - -2. Run the nginx container on different ports: you can create a `.env` file in the `openedx-docker` directory in which you indicate different ports. For instance: - - cat .env - NGINX_HTTP_PORT=81 - NGINX_HTTPS_PORT=444 - -In this example, the nginx container ports would be mapped to 81 and 444, instead of 80 and 443. - -You should note that with the latter solution, it is your responsibility to configure the webserver on the host as a proxy to the nginx container. See [this](https://github.com/regisb/openedx-docker/issues/69#issuecomment-425916825) for http, and [this](https://github.com/regisb/openedx-docker/issues/90#issuecomment-437687294) for https. - -### Help! The Docker containers are eating all my RAM/CPU/CHEESE - -You can identify which containers are consuming most resources by running: - - docker stats - -### "Running migrations... Killed!" - -The LMS and CMS containers require at least 4 GB RAM, in particular to run the Open edX SQL migrations. On Docker for Mac, by default, containers are allocated at most 2 GB of RAM. On Mac OS, if the `make all` command dies after displaying "Running migrations", you most probably need to increase the allocated RAM. [Follow these instructions from the official Docker documentation](https://docs.docker.com/docker-for-mac/#advanced). - - -### `Build failed running pavelib.servers.lms: Subprocess return code: 1` - - python manage.py lms --settings=development print_setting STATIC_ROOT 2>/dev/null - ... - Build failed running pavelib.servers.lms: Subprocess return code: 1`" - -This might occur when you run a `paver` command. `/dev/null` eats the actual error, so you will have to run the command manually. Run `make lms` (or `make cms`) to open a bash session and then: - - python manage.py lms --settings=development print_setting STATIC_ROOT - -Of course, you should replace `development` with your own settings. The error produced should help you better understand what is happening. - -### `ValueError: Unable to configure handler 'local'` - - ValueError: Unable to configure handler 'local': [Errno 2] No such file or directory - -This will occur if you try to run a development environment without patching the LOGGING configuration, as indicated in the [development](#for-developers) section above. Maybe you correctly patched the development settings, but they are not taken into account? For instance, you might have correctly defined the `EDX_PLATFORM_SETTINGS` environment variable, but `paver` uses the `devstack` settings (which does not patch the LOGGING variable). This is because calling `paver lms --settings=development` or `paver cms --settings=development` ignores the `--settings` argument. Yes, it might be considered an edx-platform bug... Instead, you should run the `update_assets` and `runserver` commands, as explained above. - -### "`TypeError: get_logger_config() got an unexpected keyword argument 'debug'`" - -This might occur when you try to run the latest version of `edx-platform`, and not a version close to `gingko.master`. It is no longer necessary to patch the `LOGGING` configuration in the latest `edx-platform` releases, as indicated in the [development](#for-developers) section above, so you should remove the call to `get_logger_config` altogether from your development settings. - -### The chosen default language does not display properly - -By default, Open edX comes with a [very limited set](https://github.com/edx/edx-platform/blob/master/conf/locale/config.yaml) of translation/localization files. To complement these languages, we add locales from the [openedx-i18n project](https://github.com/regisb/openedx-i18n/blob/master/edx-platform/locale/config-extra.yaml). But not all supported locales are downloaded. In some cases, the chosen default language will not display properly because if was not packaged in either edx-platform or openedx-i18n. If you feel like your language should be packaged, please [open an issue on the openedx-i18n project](https://github.com/regisb/openedx-i18n/issues). - -## Disclaimers & Warnings - -This project is the follow-up of my work on an [install from scratch of Open edX](https://github.com/regisb/openedx-install). It does not rely on any hack or complex deployment script. In particular, we do not use the Open edX [Ansible deployment playbooks](https://github.com/edx/configuration/). That means that the folks at edX.org are *not* responsible for troubleshooting issues of this project. Please don't bother Ned ;-) - -Do you have a problem? - -1. Carefully read the README, and in particular the [troubleshooting](#troubleshooting) section -2. Search for your problem among [open and closed Github issues](https://github.com/regisb/openedx-docker/issues?utf8=%E2%9C%93&q=is%3Aissue) -3. If necessary, open an [issue on Github](https://github.com/regisb/openedx-docker/issues/new). - - -## Known missing features: [discovery service](https://github.com/edx/course-discovery/), [ecommerce](https://github.com/edx/ecommerce), [analytics](https://github.com/edx/edx-analytics-pipeline) - -Those extra services were considered low priority while developing this project. However, most of them should not be too hard to add to a standard install. If you need one or more of these services, feel free to let me know by opening an issue. - -## Contributing - -Pull requests will be happily examined! However, we should be careful to keep the project lean and simple: both to use and to modify. Optional features should not make the user experience more complex. Instead, documentation on how to add the feature is preferred. - -## License - -This work is licensed under the terms of the [MIT License](https://github.com/regisb/openedx-docker/blob/master/LICENSE.txt). + make singleserver diff --git a/android/.gitignore b/android/.gitignore new file mode 100644 index 0000000..9374b02 --- /dev/null +++ b/android/.gitignore @@ -0,0 +1,2 @@ +/config/ +/data/ diff --git a/android/Makefile b/android/Makefile new file mode 100644 index 0000000..80b5686 --- /dev/null +++ b/android/Makefile @@ -0,0 +1,41 @@ +PWD ?= $$(pwd) +.DEFAULT_GOAL := help + +all: environment update android ## Configure and build a development Android app + +configure: + @$(MAKE) -C .. configure + +env: configure ## Generate the environment + @docker run --rm -it \ + --volume="$(PWD)/../:/openedx/config/" \ + --volume="$(PWD)/templates:/openedx/templates" \ + --volume="$(PWD)/config:/openedx/output" \ + -e USERID=$(USERID) \ + regis/openedx-configurator:hawthorn \ + configurator substitute /openedx/templates/ /openedx/output/ + +update: ## Download most recent Android image + docker pull regis/openedx-android:latest + +android: ## Build the Android app, for development + docker run --rm -it \ + --volume=$(PWD)/config/:/openedx/config/ \ + --volume=$(PWD)/data/:/openedx/data \ + regis/openedx-android:latest + @echo "Your development APK file is ready in $(PWD)/data/" + +android-release: ## Build the final Android app (beta) + # Note that this requires that you edit ./config/android/gradle.properties + docker run --rm -it \ + --volume=$(PWD)/config/:/openedx/config/ \ + --volume=$(PWD)/data/:/openedx/data \ + regis/openedx-android:latest \ + ./gradlew assembleProdRelease + @echo "Your production APK file is ready in $(PWD)/data/" + +ESCAPE =  +help: ## Print this help + @grep -E '^([a-zA-Z_-]+:.*?## .*|######* .+)$$' Makefile \ + | sed 's/######* \(.*\)/\n $(ESCAPE)[1;31m\1$(ESCAPE)[0m/g' \ + | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[33m%-30s\033[0m %s\n", $$1, $$2}' diff --git a/configurator/templates/android/edx.properties b/android/templates/edx.properties similarity index 100% rename from configurator/templates/android/edx.properties rename to android/templates/edx.properties diff --git a/configurator/templates/android/gradle.properties b/android/templates/gradle.properties similarity index 100% rename from configurator/templates/android/gradle.properties rename to android/templates/gradle.properties diff --git a/configurator/templates/android/universal.yaml b/android/templates/universal.yaml similarity index 100% rename from configurator/templates/android/universal.yaml rename to android/templates/universal.yaml diff --git a/build/.gitignore b/build/.gitignore new file mode 100644 index 0000000..70fdd94 --- /dev/null +++ b/build/.gitignore @@ -0,0 +1 @@ +openedx/requirements/private.txt diff --git a/build/Makefile b/build/Makefile new file mode 100644 index 0000000..db0c865 --- /dev/null +++ b/build/Makefile @@ -0,0 +1,59 @@ +#################### Docker image building +.DEFAULT_GOAL := help + +build: build-openedx build-configurator build-forum build-notes build-xqueue build-android ## Build all docker images + +openedx_build_args = +ifdef EDX_PLATFORM_REPOSITORY + openedx_build_args += --build-arg="EDX_PLATFORM_REPOSITORY=$(EDX_PLATFORM_REPOSITORY)" +endif +ifdef EDX_PLATFORM_VERSION + openedx_build_args += --build-arg="EDX_PLATFORM_VERSION=$(EDX_PLATFORM_VERSION)" +endif +ifdef THEMES + openedx_build_args += --build-arg="THEMES=$(THEMES)" +endif + +build-openedx: ## Build the Open edX docker image + docker build -t regis/openedx:latest -t regis/openedx:hawthorn $(openedx_build_args) openedx/ +build-configurator: ## Build the configurator docker image + docker build -t regis/openedx-configurator:latest -t regis/openedx-configurator:hawthorn configurator/ +build-forum: ## Build the forum docker image + docker build -t regis/openedx-forum:latest -t regis/openedx-forum:hawthorn forum/ +build-notes: ## Build the Notes docker image + docker build -t regis/openedx-notes:latest -t regis/openedx-notes:hawthorn notes/ +build-xqueue: ## Build the Xqueue docker image + docker build -t regis/openedx-xqueue:latest -t regis/openedx-xqueue:hawthorn xqueue/ +build-android: ## Build the docker image for Android + docker build -t regis/openedx-android:latest android/ + +################### Pushing images to docker hub + +push: push-openedx push-configurator push-forum push-notes push-xqueue push-android ## Push all images to dockerhub +push-openedx: ## Push Open edX images to dockerhub + docker push regis/openedx:hawthorn + docker push regis/openedx:latest +push-configurator: ## Push configurator image to dockerhub + docker push regis/openedx-configurator:hawthorn + docker push regis/openedx-configurator:latest +push-forum: ## Push forum image to dockerhub + docker push regis/openedx-forum:hawthorn + docker push regis/openedx-forum:latest +push-notes: ## Push notes image to dockerhub + docker push regis/openedx-notes:hawthorn + docker push regis/openedx-notes:latest +push-xqueue: ## Push Xqueue image to dockerhub + docker push regis/openedx-xqueue:hawthorn + docker push regis/openedx-xqueue:latest +push-android: ## Push the Android image to dockerhub + docker push regis/openedx-android:latest + +dockerhub: build push ## Build and push all images to dockerhub + +##################### Information + +ESCAPE =  +help: ## Print this help + @grep -E '^([a-zA-Z_-]+:.*?## .*|######* .+)$$' Makefile \ + | sed 's/######* \(.*\)/\n $(ESCAPE)[1;31m\1$(ESCAPE)[0m/g' \ + | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[33m%-30s\033[0m %s\n", $$1, $$2}' diff --git a/android/Dockerfile b/build/android/Dockerfile similarity index 100% rename from android/Dockerfile rename to build/android/Dockerfile diff --git a/android/edx.properties b/build/android/edx.properties similarity index 100% rename from android/edx.properties rename to build/android/edx.properties diff --git a/build/configurator/Dockerfile b/build/configurator/Dockerfile new file mode 100644 index 0000000..8da8b03 --- /dev/null +++ b/build/configurator/Dockerfile @@ -0,0 +1,14 @@ +FROM ubuntu:18.04 + +RUN apt update && \ + apt install -y python3 python3-pip curl +RUN pip3 install jinja2 + +RUN mkdir /openedx /openedx/config /openedx/templates +COPY ./bin/configurator /usr/local/bin +COPY ./bin/docker-entrypoint.sh /usr/local/bin +WORKDIR /openedx/ + +ENTRYPOINT ["docker-entrypoint.sh"] +CMD configurator -c /openedx/config/config.json interactive && \ + configurator -c /openedx/config/config.json substitute /openedx/templates/ /openedx/output/ diff --git a/configurator/bin/configure.py b/build/configurator/bin/configurator similarity index 76% rename from configurator/bin/configure.py rename to build/configurator/bin/configurator index 9569aa7..f5f7e45 100755 --- a/configurator/bin/configure.py +++ b/build/configurator/bin/configurator @@ -24,17 +24,17 @@ class Configurator: """ self.__values = OrderedDict() self.__default_values = default_overrides - try: - self.__input = raw_input - except NameError: + if os.environ.get('SILENT'): + self.__input = None + else: self.__input = input + print("====================================\n" + " Interactive configuration \n" + "====================================") def as_dict(self): return self.__values - def mute(self): - self.__input = None - def get_default_value(self, name, default): setting_name = 'SETTING_' + name.upper() if os.environ.get(setting_name): @@ -45,12 +45,9 @@ class Configurator: def add(self, name, default, question=""): default = self.get_default_value(name, default) - if not self.__input or not question: return self.set(name, default) - - question += " (default: \"{}\"): ".format(default) - return self.set(name, self.__input(question) or default) + return self.set(name, self.ask(question, default)) def add_bool(self, name, default, question=""): default = self.get_default_value(name, default) @@ -60,9 +57,9 @@ class Configurator: default = False if not self.__input or not question: return self.set(name, default) - question += " [Y/n] " if default else " [y/N] " + question += " (y/n)" while True: - answer = self.__input(question) + answer = self.ask(question, 'y' if default else 'n') if answer is None or answer == '': return self.set(name, default) if answer.lower() in ['y', 'yes']: @@ -74,13 +71,15 @@ class Configurator: default = self.get_default_value(name, default) if not self.__input or not question: return self.set(name, default) - question += " (default: \"{}\"): ".format(default) while True: - answer = self.__input(question) or default + answer = self.ask(question, default) if answer in choices: return self.set(name, answer) print("Invalid value. Choices are: {}".format(", ".join(choices))) + def ask(self, question, default): + return self.__input('\1\2\x1b[35m> {} [{}] \x1b[39;49;00m'.format(question, default)) or default + def get(self, name): return self.__values.get(name) @@ -96,12 +95,6 @@ def main(): subparsers = parser.add_subparsers() parser_interactive = subparsers.add_parser('interactive') - parser_interactive.add_argument('-s', '--silent', action='store_true', - help=( - "Be silent and accept all default values. " - "This is good for debugging and automation, but " - "probably not what you want" - )) parser_interactive.set_defaults(func=interactive) parser_substitute = subparsers.add_parser('substitute') @@ -112,31 +105,36 @@ def main(): args = parser.parse_args() args.func(args) -def load_config(args): - if os.path.exists(args.config): - with open(args.config) as f: +def load_config(path): + if os.path.exists(path): + with open(path) as f: return json.load(f) return {} def interactive(args): - print("\n====================================") - print(" Interactive configuration ") - print("====================================") + interactive_configuration(args.config) - configurator = Configurator(**load_config(args)) - if args.silent or os.environ.get('SILENT'): - configurator.mute() +def interactive_configuration(config_path): + configurator = Configurator(**load_config(config_path)) configurator.add( - 'LMS_HOST', 'www.myopenedx.com', "Your website domain name for students (LMS)." + 'LMS_HOST', 'www.myopenedx.com', "Your website domain name for students (LMS)" ).add( - 'CMS_HOST', 'studio.' + configurator.get('LMS_HOST'), "Your website domain name for teachers (CMS)." + 'CMS_HOST', 'studio.' + configurator.get('LMS_HOST'), "Your website domain name for teachers (CMS)" ).add( 'PLATFORM_NAME', "My Open edX", "Your platform name/title" ).add( 'CONTACT_EMAIL', 'contact@' + configurator.get('LMS_HOST'), "Your public contact email address", ).add_choice( 'LANGUAGE_CODE', 'en', - ['en', 'am', 'ar', 'az', 'bg-bg', 'bn-bd', 'bn-in', 'bs', 'ca', 'ca@valencia', 'cs', 'cy', 'da', 'de-de', 'el', 'en-uk', 'en@lolcat', 'en@pirate', 'es-419', 'es-ar', 'es-ec', 'es-es', 'es-mx', 'es-pe', 'et-ee', 'eu-es', 'fa', 'fa-ir', 'fi-fi', 'fil', 'fr', 'gl', 'gu', 'he', 'hi', 'hr', 'hu', 'hy-am', 'id', 'it-it', 'ja-jp', 'kk-kz', 'km-kh', 'kn', 'ko-kr', 'lt-lt', 'ml', 'mn', 'mr', 'ms', 'nb', 'ne', 'nl-nl', 'or', 'pl', 'pt-br', 'pt-pt', 'ro', 'ru', 'si', 'sk', 'sl', 'sq', 'sr', 'sv', 'sw', 'ta', 'te', 'th', 'tr-tr', 'uk', 'ur', 'vi', 'uz', 'zh-cn', 'zh-hk', 'zh-tw'], + ['en', 'am', 'ar', 'az', 'bg-bg', 'bn-bd', 'bn-in', 'bs', 'ca', + 'ca@valencia', 'cs', 'cy', 'da', 'de-de', 'el', 'en-uk', 'en@lolcat', + 'en@pirate', 'es-419', 'es-ar', 'es-ec', 'es-es', 'es-mx', 'es-pe', + 'et-ee', 'eu-es', 'fa', 'fa-ir', 'fi-fi', 'fil', 'fr', 'gl', 'gu', + 'he', 'hi', 'hr', 'hu', 'hy-am', 'id', 'it-it', 'ja-jp', 'kk-kz', + 'km-kh', 'kn', 'ko-kr', 'lt-lt', 'ml', 'mn', 'mr', 'ms', 'nb', 'ne', + 'nl-nl', 'or', 'pl', 'pt-br', 'pt-pt', 'ro', 'ru', 'si', 'sk', 'sl', + 'sq', 'sr', 'sv', 'sw', 'ta', 'te', 'th', 'tr-tr', 'uk', 'ur', 'vi', + 'uz', 'zh-cn', 'zh-hk', 'zh-tw'], "The default language code for the platform" ).add( 'SECRET_KEY', random_string(24) @@ -183,14 +181,12 @@ def interactive(args): ) # Save values - with open(args.config, 'w') as f: + with open(config_path, 'w') as f: json.dump(configurator.as_dict(), f, sort_keys=True, indent=4) - print("\nConfiguration values were saved to ", args.config) def substitute(args): - config = load_config(args) - + config = load_config(args.config) for root, _, filenames in os.walk(args.src): for filename in filenames: if filename.startswith('.'): @@ -218,8 +214,6 @@ def substitute_file(config, src, dst): # Set same permissions as original file os.chmod(dst, os.stat(src).st_mode) - print("Generated file {} from template {}".format(dst, src)) - def random_string(length): return "".join([random.choice(string.ascii_letters + string.digits) for _ in range(length)]) diff --git a/configurator/bin/docker-entrypoint.sh b/build/configurator/bin/docker-entrypoint.sh similarity index 88% rename from configurator/bin/docker-entrypoint.sh rename to build/configurator/bin/docker-entrypoint.sh index 67f4b65..a4445bc 100755 --- a/configurator/bin/docker-entrypoint.sh +++ b/build/configurator/bin/docker-entrypoint.sh @@ -4,7 +4,6 @@ USERID=${USERID:=0} ## Configure user with a different USERID if requested. if [ "$USERID" -ne 0 ] then - echo "creating new user 'openedx' with UID $USERID" useradd --home-dir /openedx -u $USERID openedx # Change file permissions diff --git a/forum/Dockerfile b/build/forum/Dockerfile similarity index 100% rename from forum/Dockerfile rename to build/forum/Dockerfile diff --git a/notes/Dockerfile b/build/notes/Dockerfile similarity index 100% rename from notes/Dockerfile rename to build/notes/Dockerfile diff --git a/openedx/Dockerfile b/build/openedx/Dockerfile similarity index 61% rename from openedx/Dockerfile rename to build/openedx/Dockerfile index 0d47703..cd62602 100644 --- a/openedx/Dockerfile +++ b/build/openedx/Dockerfile @@ -4,46 +4,39 @@ FROM ubuntu:16.04 # Install system requirements RUN apt update && \ - apt upgrade -y && \ - # Global requirements - apt install -y language-pack-en git python-virtualenv build-essential software-properties-common curl git-core libxml2-dev libxslt1-dev python-pip libmysqlclient-dev python-apt python-dev libxmlsec1-dev libfreetype6-dev swig gcc g++ && \ - # openedx requirements - apt install -y gettext gfortran graphviz graphviz-dev libffi-dev libfreetype6-dev libgeos-dev libjpeg8-dev liblapack-dev libpng12-dev libsqlite3-dev libxml2-dev libxmlsec1-dev libxslt1-dev nodejs npm ntp pkg-config && \ - # Our requirements - apt install -y mysql-client - -# Install symlink so that we have access to 'node' binary without virtualenv. -# This replaces the "nodeenv" install. -RUN apt install -y nodejs-legacy + # Global requirements + apt install -y language-pack-en git python-virtualenv build-essential software-properties-common curl git-core libxml2-dev libxslt1-dev python-pip libmysqlclient-dev python-apt python-dev libxmlsec1-dev libfreetype6-dev swig gcc g++ \ + # openedx requirements + gettext gfortran graphviz graphviz-dev libffi-dev libfreetype6-dev libgeos-dev libjpeg8-dev liblapack-dev libpng12-dev libsqlite3-dev libxml2-dev libxmlsec1-dev libxslt1-dev nodejs npm ntp pkg-config \ + # Our requirements + mysql-client \ + # Install symlink so that we have access to 'node' binary without virtualenv. + # This replaces the "nodeenv" install. + nodejs-legacy \ + && rm -rf /var/lib/apt/lists/* # Dockerize will be useful to wait for mysql DB availability -ENV DOCKERIZE_VERSION v0.6.1 +ARG DOCKERIZE_VERSION=v0.6.1 RUN curl -L -o /tmp/dockerize.tar.gz https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \ && tar -C /usr/local/bin -xzvf /tmp/dockerize.tar.gz \ && rm /tmp/dockerize.tar.gz -# Static assets will reside in /openedx/data, themes in /openedx/themes and -# edx-platform will be checked-out in /openedx/ -RUN mkdir /openedx /openedx/data /openedx/themes /openedx/edx-platform -WORKDIR /openedx/edx-platform - ## Checkout edx-platform code ARG EDX_PLATFORM_REPOSITORY=https://github.com/edx/edx-platform.git ARG EDX_PLATFORM_VERSION=open-release/hawthorn.2 -RUN git clone $EDX_PLATFORM_REPOSITORY --branch $EDX_PLATFORM_VERSION --depth 1 . +RUN mkdir -p /openedx/edx-platform && \ + git clone $EDX_PLATFORM_REPOSITORY --branch $EDX_PLATFORM_VERSION --depth 1 /openedx/edx-platform +WORKDIR /openedx/edx-platform # Download extra locales to /openedx/locale RUN cd /tmp \ && curl -L -o openedx-i18n.tar.gz https://github.com/regisb/openedx-i18n/archive/hawthorn.tar.gz \ && tar xzf /tmp/openedx-i18n.tar.gz \ - && mv openedx-i18n-hawthorn/edx-platform/locale/ /openedx/ \ + && mv openedx-i18n-hawthorn/edx-platform/locale/ /openedx/locale/ \ && rm -rf openedx-i18n* -# Copy convenient scripts -COPY ./bin/docker-entrypoint.sh /usr/local/bin/ - # Install python requirements (clone source repos in a separate dir, otherwise -# will be overwritten when we mount edx-platform) +# they will be overwritten when we mount edx-platform) ENV NO_PYTHON_UNINSTALL 1 RUN pip install --src ../venv/src -r requirements/edx/base.txt RUN pip install --src ../venv/src -r requirements/edx/development.txt @@ -59,7 +52,7 @@ ENV PATH ./node_modules/.bin:${PATH} # Install private requirements: this is useful for installing custom xblocks. # In particular, to install xblocks from a private repository, clone the -# respositories to ./requirements on the host and add `-e ./myxblock/` to +# repositories to ./requirements on the host and add `-e ./myxblock/` to # ./requirements/private.txt. COPY ./requirements/ /openedx/requirements RUN touch /openedx/requirements/private.txt \ @@ -68,15 +61,15 @@ RUN touch /openedx/requirements/private.txt \ # Link configuration files to common /openedx/config folder, which should later # be mounted as a volume. Note that this image will not be functional until # config files have been mounted inside the container -RUN mkdir -p /openedx/config/universal/lms /openedx/config/universal/cms \ - && ln -s /openedx/config/universal/lms/ /openedx/edx-platform/lms/envs/universal \ - && ln -s /openedx/config/universal/cms/ /openedx/edx-platform/cms/envs/universal \ - && ln -s /openedx/config/lms.env.json /openedx/ \ - && ln -s /openedx/config/cms.env.json /openedx/ \ - && ln -s /openedx/config/lms.auth.json /openedx/ \ - && ln -s /openedx/config/cms.auth.json /openedx/ -COPY settings/lms/*.py /openedx/config/universal/lms/ -COPY settings/cms/*.py /openedx/config/universal/cms/ +RUN mkdir -p /openedx/env/universal/lms /openedx/env/universal/cms \ + && ln -s /openedx/env/universal/lms/ /openedx/edx-platform/lms/envs/universal \ + && ln -s /openedx/env/universal/cms/ /openedx/edx-platform/cms/envs/universal \ + && ln -s /openedx/env/lms.env.json /openedx/ \ + && ln -s /openedx/env/cms.env.json /openedx/ \ + && ln -s /openedx/env/lms.auth.json /openedx/ \ + && ln -s /openedx/env/cms.auth.json /openedx/ +COPY settings/lms/*.py /openedx/env/universal/lms/ +COPY settings/cms/*.py /openedx/env/universal/cms/ # Copy scripts COPY ./bin /openedx/bin diff --git a/openedx/bin/docker-entrypoint.sh b/build/openedx/bin/docker-entrypoint.sh similarity index 100% rename from openedx/bin/docker-entrypoint.sh rename to build/openedx/bin/docker-entrypoint.sh diff --git a/openedx/bin/openedx-assets b/build/openedx/bin/openedx-assets similarity index 100% rename from openedx/bin/openedx-assets rename to build/openedx/bin/openedx-assets diff --git a/openedx/requirements/.gitignore b/build/openedx/requirements/.gitignore similarity index 100% rename from openedx/requirements/.gitignore rename to build/openedx/requirements/.gitignore diff --git a/openedx/requirements/README b/build/openedx/requirements/README similarity index 100% rename from openedx/requirements/README rename to build/openedx/requirements/README diff --git a/configurator/templates/openedx/universal/cms/__init__.py b/build/openedx/settings/cms/__init__.py similarity index 100% rename from configurator/templates/openedx/universal/cms/__init__.py rename to build/openedx/settings/cms/__init__.py diff --git a/openedx/settings/cms/assets.py b/build/openedx/settings/cms/assets.py similarity index 100% rename from openedx/settings/cms/assets.py rename to build/openedx/settings/cms/assets.py diff --git a/configurator/templates/openedx/universal/lms/__init__.py b/build/openedx/settings/lms/__init__.py similarity index 100% rename from configurator/templates/openedx/universal/lms/__init__.py rename to build/openedx/settings/lms/__init__.py diff --git a/openedx/settings/lms/assets.py b/build/openedx/settings/lms/assets.py similarity index 100% rename from openedx/settings/lms/assets.py rename to build/openedx/settings/lms/assets.py diff --git a/openedx/themes/.gitignore b/build/openedx/themes/.gitignore similarity index 100% rename from openedx/themes/.gitignore rename to build/openedx/themes/.gitignore diff --git a/xqueue/Dockerfile b/build/xqueue/Dockerfile similarity index 100% rename from xqueue/Dockerfile rename to build/xqueue/Dockerfile diff --git a/configurator/Dockerfile b/configurator/Dockerfile deleted file mode 100644 index f80f9da..0000000 --- a/configurator/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM ubuntu:18.04 - -RUN apt update && \ - apt install -y python3 python3-pip curl -RUN pip3 install jinja2 - -RUN mkdir /openedx -VOLUME /openedx/config -COPY ./bin/configure.py /openedx/configure.py -COPY ./bin/docker-entrypoint.sh /openedx/docker-entrypoint.sh -COPY ./templates /openedx/templates -WORKDIR /openedx - -ENTRYPOINT ["./docker-entrypoint.sh"] -CMD ./configure.py -c /openedx/config/config.json interactive && \ - ./configure.py -c /openedx/config/config.json substitute ./templates/ /openedx/config diff --git a/deploy/.gitignore b/deploy/.gitignore new file mode 100644 index 0000000..47def24 --- /dev/null +++ b/deploy/.gitignore @@ -0,0 +1 @@ +/env/ diff --git a/deploy/Makefile b/deploy/Makefile new file mode 100644 index 0000000..717d091 --- /dev/null +++ b/deploy/Makefile @@ -0,0 +1,22 @@ +.PHONY: env +PWD = $$(pwd) + +configure: + @$(MAKE) -s -C .. configure + +env: configure ## Create the platform environment + @docker run --rm -it \ + --volume="$(PWD)/../:/openedx/config/" \ + --volume="$(PWD)/templates/apps:/openedx/templates" \ + --volume="$(PWD)/env:/openedx/output" \ + -e USERID=$(USERID) -e SILENT=$(SILENT) $(CONFIGURE_OPTS) \ + regis/openedx-configurator:hawthorn \ + configurator substitute /openedx/templates/ /openedx/output/ + @docker run --rm -it \ + --volume="$(PWD)/../:/openedx/config/" \ + --volume="$(PWD)/templates/project:/openedx/templates" \ + --volume="$(PWD):/openedx/output" \ + -e USERID=$(USERID) -e SILENT=$(SILENT) $(CONFIGURE_OPTS) \ + regis/openedx-configurator:hawthorn \ + configurator substitute /openedx/templates/ /openedx/output/ + diff --git a/deploy/singleserver/.gitignore b/deploy/singleserver/.gitignore new file mode 100644 index 0000000..d66b519 --- /dev/null +++ b/deploy/singleserver/.gitignore @@ -0,0 +1,3 @@ +/Makefile.env +/docker-compose.yml +/.env diff --git a/deploy/singleserver/Makefile b/deploy/singleserver/Makefile new file mode 100644 index 0000000..9d5a61e --- /dev/null +++ b/deploy/singleserver/Makefile @@ -0,0 +1,195 @@ +.PHONY: env +.DEFAULT_GOAL := help + +PWD = $$(pwd) +USERID ?= $$(id -u) +EDX_PLATFORM_SETTINGS ?= universal.production +-include $(PWD)/Makefile.env + +DOCKER_COMPOSE_RUN = docker-compose run --rm +DOCKER_COMPOSE_RUN_OPENEDX = $(DOCKER_COMPOSE_RUN) -e SETTINGS=$(EDX_PLATFORM_SETTINGS) \ + --volume="$(PWD)/../../build/openedx/themes:/openedx/themes" +ifneq ($(EDX_PLATFORM_PATH),) + DOCKER_COMPOSE_RUN_OPENEDX += -e USERID=$(USERID) --volume="$(EDX_PLATFORM_PATH):/openedx/edx-platform" +endif + +DOCKER_COMPOSE_RUN_LMS = $(DOCKER_COMPOSE_RUN_OPENEDX) -p 8000:8000 lms +DOCKER_COMPOSE_RUN_CMS = $(DOCKER_COMPOSE_RUN_OPENEDX) -p 8001:8001 cms + +##################### Running Open edX + +all: env ## Configure and run a full-featured platform + @$(MAKE) stats + @$(MAKE) https-certificate + @$(MAKE) update + @$(MAKE) databases + @$(MAKE) assets + @$(MAKE) daemonize + @echo "All set \o/ You can access the LMS at http://localhost and the CMS at http://studio.localhost" + +run: ## Run the complete platform + docker-compose up +up: run + +daemonize: ## Run the complete platform, with daemonization + docker-compose up -d + @echo "Daemon is up and running" +daemon: daemonize + +stop: ## Stop all services + docker-compose rm --stop --force + +##################### Docker image management + +update: ## Download most recent images + docker-compose pull + +##################### Configuration + +configure: + @$(MAKE) -s -C .. configure + +env: + @$(MAKE) -s -C .. env + @docker run --rm -it \ + --volume="$(PWD)/../../:/openedx/config/" \ + --volume="$(PWD)/templates/:/openedx/templates" \ + --volume="$(PWD):/openedx/output" \ + -e USERID=$(USERID) \ + regis/openedx-configurator:hawthorn \ + configurator substitute /openedx/templates/ /openedx/output/ + +##################### Database + +databases: provision-databases migrate provision-oauth2 ## Bootstrap databases + +provision-databases: ## Create necessary databases and users + $(DOCKER_COMPOSE_RUN) lms /openedx/env/provision.sh +provision-oauth2: ## Create users for SSO between services + $(DOCKER_COMPOSE_RUN) lms /openedx/env/oauth2.sh + +migrate: migrate-openedx migrate-forum ## Perform all database migrations + @if [ "$(ACTIVATE_XQUEUE)" = "1" ] ; then \ + $(MAKE) migrate-xqueue ;\ + fi + @if [ "$(ACTIVATE_NOTES)" = "1" ] ; then \ + $(MAKE) migrate-notes ; \ + fi +migrate-openedx: ## Perform database migrations on LMS/CMS + $(DOCKER_COMPOSE_RUN) lms bash -c "dockerize -wait tcp://mysql:3306 -timeout 20s && ./manage.py lms migrate" + $(DOCKER_COMPOSE_RUN) cms bash -c "dockerize -wait tcp://mysql:3306 -timeout 20s && ./manage.py cms migrate" + $(MAKE) reindex-courses +migrate-forum: ## Perform database migrations on discussion forums + $(DOCKER_COMPOSE_RUN) forum bash -c "bundle exec rake search:initialize && \ + bundle exec rake search:rebuild_index" +migrate-notes: ## Perform database migrations for the Notes service + $(DOCKER_COMPOSE_RUN) notes ./manage.py migrate +migrate-xqueue: ## Perform database migrations for the XQueue service + $(DOCKER_COMPOSE_RUN) xqueue ./manage.py migrate + +reindex-courses: ## Refresh course index so they can be found in the LMS search engine + $(DOCKER_COMPOSE_RUN) cms ./manage.py cms reindex_course --all --setup + +##################### Static assets + +# To collect assets we don't rely on the "paver update_assets" command because +# webpack collection incorrectly sets the NODE_ENV variable when using custom +# settings. Thus, each step must be performed separately. This should be fixed +# in the next edx-platform release thanks to https://github.com/edx/edx-platform/pull/18430/ + +assets: ## Generate production-ready static assets + docker-compose -f scripts.yml run --rm \ + --volume=$(PWD)/../../data/openedx:/tmp/openedx/ openedx bash -c \ + "rm -rf /tmp/openedx/staticfiles \ + && cp -r /openedx/staticfiles /tmp/openedx" +assets-development: ## Generate static assets for local development + $(DOCKER_COMPOSE_RUN_OPENEDX) --no-deps lms bash -c "openedx-assets build --env=dev" +assets-development-lms: + $(DOCKER_COMPOSE_RUN_OPENEDX) --no-deps lms bash -c "openedx-assets build --env=dev --system lms" +assets-development-cms: + $(DOCKER_COMPOSE_RUN_OPENEDX) --no-deps lms bash -c "openedx-assets build --env=dev --system cms" +watch-themes: ## Watch for changes in your themes and build development assets + $(DOCKER_COMPOSE_RUN_OPENEDX) --no-deps lms openedx-assets watch-themes --env dev + +#################### Logging + +logs: ## Print all logs from a service since it started. E.g: "make logs service=lms", "make logs service=nginx" + docker-compose logs $(service) +tail: ## Similar to "tail" on the logs of a service. E.g: "make tail service=lms", "make tail service=nginx" + docker-compose logs --tail=10 $(service) +tail-follow: ## Similar to "tail -f" on the logs of a service. E.g: "make tail-follow service=lms", "make tail-follow service=nginx" + docker-compose logs --tail=10 -f $(service) + +##################### Development + +lms: ## Open a bash shell in the LMS + $(DOCKER_COMPOSE_RUN_LMS) bash +cms: ## Open a bash shell in the CMS + $(DOCKER_COMPOSE_RUN_CMS) bash + +lms-python: ## Open a python shell in the LMS + $(DOCKER_COMPOSE_RUN_OPENEDX) lms ./manage.py lms shell +lms-shell: lms-python +lms-runserver: ## Run a local webserver, useful for debugging + $(DOCKER_COMPOSE_RUN_LMS) ./manage.py lms runserver 0.0.0.0:8000 +cms-python: ## Open a python shell in the CMS + $(DOCKER_COMPOSE_RUN_OPENEDX) cms ./manage.py cms shell +cms-shell: cms-python +cms-runserver: ## Run a local webserver, useful for debugging + $(DOCKER_COMPOSE_RUN_CMS) ./manage.py cms runserver 0.0.0.0:8001 + +restart-openedx: ## Restart lms, cms, and workers + docker-compose restart lms lms_worker cms cms_worker + +##################### SSL/TLS (HTTPS certificates) + +https_command = docker run --rm -it \ + --volume="$(PWD)/../env/letsencrypt/:/openedx/letsencrypt/env/" \ + --volume="$(PWD)/data/letsencrypt/:/etc/letsencrypt/" \ + -p "80:80" +certbot_image = certbot/certbot:latest + +https-certificate: ## Generate https certificates + @if [ "$(ACTIVATE_HTTPS)" = "1" ] ; then \ + $(https_command) --entrypoint "/openedx/letsencrypt/env/certonly.sh" $(certbot_image) + fi + +https-certificate-renew: ## Renew https certificates + $(https_command) $(certbot_image) renew + +##################### Additional commands + +stats: ## Collect anonymous information about the platform + @if [ "$(DISABLE_STATS)" != "1" ] ; then \ + docker run --rm -it --volume="$(PWD)/../env:/openedx/env" \ + regis/openedx-configurator:hawthorn /openedx/env/openedx/stats 2> /dev/null || true ; \ + fi + +import-demo-course: ## Import the demo course from edX + $(DOCKER_COMPOSE_RUN_OPENEDX) cms /bin/bash -c " \ + git clone https://github.com/edx/edx-demo-course --branch open-release/hawthorn.2 --depth 1 ../edx-demo-course \ + && python ./manage.py cms import ../data ../edx-demo-course" + +create-staff-user: ## Create a user with admin rights + $(DOCKER_COMPOSE_RUN_OPENEDX) lms /bin/bash -c "./manage.py lms manage_user --superuser --staff ${USERNAME} ${EMAIL} && ./manage.py lms changepassword ${USERNAME}" + +##################### Information + +# Obtained by running "echo '\033' in a shell +ESCAPE =  +help: ## Print this help + @grep -E '^([a-zA-Z_-]+:.*?## .*|######* .+)$$' Makefile \ + | sed 's/######* \(.*\)/\n $(ESCAPE)[1;31m\1$(ESCAPE)[0m/g' \ + | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[33m%-30s\033[0m %s\n", $$1, $$2}' + +info: ## Print some information about the current install, for debugging + uname -a + @echo "-------------------------" + git rev-parse HEAD + @echo "-------------------------" + docker version + @echo "-------------------------" + docker-compose --version + @echo "-------------------------" + echo $$EDX_PLATFORM_PATH + echo $$EDX_PLATFORM_SETTINGS diff --git a/docker-compose-scripts.yml b/deploy/singleserver/scripts.yml similarity index 100% rename from docker-compose-scripts.yml rename to deploy/singleserver/scripts.yml diff --git a/configurator/templates/Makefile.env b/deploy/singleserver/templates/Makefile.env similarity index 72% rename from configurator/templates/Makefile.env rename to deploy/singleserver/templates/Makefile.env index d0a5cb4..5bf0189 100644 --- a/configurator/templates/Makefile.env +++ b/deploy/singleserver/templates/Makefile.env @@ -1,4 +1,3 @@ ACTIVATE_HTTPS ?= {{ 1 if ACTIVATE_HTTPS else 0 }} ACTIVATE_XQUEUE ?= {{ 1 if ACTIVATE_XQUEUE else 0 }} ACTIVATE_NOTES ?= {{ 1 if ACTIVATE_NOTES else 0 }} -ACTIVATE_PORTAINER ?= {{ 1 if ACTIVATE_PORTAINER else 0 }} diff --git a/docker-compose.yml b/deploy/singleserver/templates/docker-compose.yml similarity index 59% rename from docker-compose.yml rename to deploy/singleserver/templates/docker-compose.yml index df61af0..e4f108d 100644 --- a/docker-compose.yml +++ b/deploy/singleserver/templates/docker-compose.yml @@ -13,15 +13,15 @@ services: command: mongod --smallfiles --nojournal --storageEngine wiredTiger restart: unless-stopped volumes: - - ./data/mongodb:/data/db + - ../../data/mongodb:/data/db mysql: image: mysql:5.6.36 command: mysqld --character-set-server=utf8 --collation-server=utf8_general_ci restart: unless-stopped volumes: - - ./data/mysql:/var/lib/mysql - env_file: ./config/mysql/auth.env + - ../../data/mysql:/var/lib/mysql + env_file: ../env/mysql/auth.env elasticsearch: image: elasticsearch:1.5.2 @@ -35,7 +35,7 @@ services: hard: -1 restart: unless-stopped volumes: - - ./data/elasticsearch:/usr/share/elasticsearch/data + - ../../data/elasticsearch:/usr/share/elasticsearch/data nginx: image: nginx:1.13 @@ -44,14 +44,14 @@ services: - "${NGINX_HTTP_PORT:-80}:80" - "${NGINX_HTTPS_PORT:-443}:443" volumes: - - ./config/nginx:/etc/nginx/conf.d/ - - ./data/openedx:/var/www/openedx:ro - - ./data/letsencrypt:/etc/letsencrypt/:ro + - ../env/nginx:/etc/nginx/conf.d/ + - ../../data/openedx:/var/www/openedx:ro + - ../../data/letsencrypt:/etc/letsencrypt/:ro rabbitmq: image: rabbitmq:3.6.10 volumes: - - ./data/rabbitmq:/var/lib/rabbitmq + - ../../data/rabbitmq:/var/lib/rabbitmq restart: unless-stopped # Simple SMTP server @@ -63,8 +63,6 @@ services: forum: image: regis/openedx-forum:hawthorn - build: - context: ./forum environment: API_KEY: "forumapikey" SEARCH_SERVER: "http://elasticsearch:9200" @@ -78,14 +76,12 @@ services: lms: image: ${OPENEDX_DOCKER_IMAGE:-regis/openedx:hawthorn} - build: - context: ./openedx environment: SERVICE_VARIANT: lms restart: unless-stopped volumes: - - ./config/openedx:/openedx/config - - ./data/lms:/openedx/data + - ../env/openedx:/openedx/env + - ../../data/lms:/openedx/data depends_on: - elasticsearch - forum @@ -97,14 +93,12 @@ services: cms: image: ${OPENEDX_DOCKER_IMAGE:-regis/openedx:hawthorn} - build: - context: ./openedx environment: SERVICE_VARIANT: cms restart: unless-stopped volumes: - - ./config/openedx:/openedx/config - - ./data/cms:/openedx/data + - ../env/openedx:/openedx/env + - ../../data/cms:/openedx/data depends_on: - memcached - mongodb @@ -117,30 +111,77 @@ services: # We could probably create one service per queue here. For small instances, it is not necessary. lms_worker: image: ${OPENEDX_DOCKER_IMAGE:-regis/openedx:hawthorn} - build: - context: ./openedx environment: SERVICE_VARIANT: lms C_FORCE_ROOT: "1" # run celery tasks as root #nofear command: ./manage.py lms celery worker --loglevel=info --hostname=edx.lms.core.default.%%h --maxtasksperchild 100 restart: unless-stopped volumes: - - ./config/openedx:/openedx/config - - ./data/lms:/openedx/data + - ../env/openedx:/openedx/env + - ../../data/lms:/openedx/data depends_on: - lms cms_worker: image: ${OPENEDX_DOCKER_IMAGE:-regis/openedx:hawthorn} - build: - context: ./openedx environment: SERVICE_VARIANT: cms C_FORCE_ROOT: "1" # run celery tasks as root #nofear command: ./manage.py cms celery worker --loglevel=info --hostname=edx.cms.core.default.%%h --maxtasksperchild 100 restart: unless-stopped volumes: - - ./config/openedx:/openedx/config - - ./data/cms:/openedx/data + - ../env/openedx:/openedx/env + - ../../data/cms:/openedx/data depends_on: - cms + + {% if ACTIVATE_NOTES %} + ############# Notes: backend store for edX Student Notes + notes: + image: regis/openedx-notes:hawthorn + networks: + default: + aliases: + - notes.openedx + volumes: + - ../env/notes:/openedx/env + - ../../data/notes:/openedx/data + restart: unless-stopped + depends_on: + - mysql + {% endif %} + + {% if ACTIVATE_XQUEUE %} + ############# Xqueue: external grading of Open edX problems + xqueue: + image: regis/openedx-xqueue:hawthorn + volumes: + - ../env/xqueue:/openedx/env + - ../../data/xqueue:/openedx/data + restart: unless-stopped + depends_on: + - mysql + + xqueue_consumer: + image: regis/openedx-xqueue:hawthorn + volumes: + - ../env/xqueue:/openedx/env + - ../../data/xqueue:/openedx/data + restart: unless-stopped + # Run 12 workers per queue + command: ./manage.py run_consumer 12 + depends_on: + - mysql + {% endif %} + + {% if ACTIVATE_PORTAINER %} + ############# Portainer: container supervision with a web UI + portainer: + image: portainer/portainer:latest + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ../../data/portainer:/data + restart: unless-stopped + depends_on: + - nginx + {% endif %} diff --git a/configurator/templates/letsencrypt/certonly.sh b/deploy/templates/letsencrypt/certonly.sh similarity index 100% rename from configurator/templates/letsencrypt/certonly.sh rename to deploy/templates/letsencrypt/certonly.sh diff --git a/configurator/templates/mysql/auth.env b/deploy/templates/mysql/auth.env similarity index 100% rename from configurator/templates/mysql/auth.env rename to deploy/templates/mysql/auth.env diff --git a/configurator/templates/nginx/cms.conf b/deploy/templates/nginx/cms.conf similarity index 83% rename from configurator/templates/nginx/cms.conf rename to deploy/templates/nginx/cms.conf index 99d6b61..0d525e8 100644 --- a/configurator/templates/nginx/cms.conf +++ b/deploy/templates/nginx/cms.conf @@ -15,18 +15,14 @@ server { ssl_certificate_key /etc/letsencrypt/live/{{ LMS_HOST }}/privkey.pem; {% endif %} - # Prevent invalid display courseware in IE 10+ with high privacy settings - add_header P3P 'CP="Open edX does not have a P3P policy."'; - - # Nginx does not support nested condition or or conditions so - # there is an unfortunate mix of conditonals here. - client_max_body_size 100M; rewrite ^(.*)/favicon.ico$ /static/images/favicon.ico last; # Disables server version feedback on pages and in headers server_tokens off; + # Prevent invalid display courseware in IE 10+ with high privacy settings + add_header P3P 'CP="Open edX does not have a P3P policy."'; location @proxy_to_cms_app { proxy_set_header X-Forwarded-Proto $scheme; @@ -58,17 +54,10 @@ server { # Set django-pipelined files to maximum cache time location ~ "/static/(?P.*\.[0-9a-f]{12}\..*)" { expires max; - # Without this try_files, files that have been run through - # django-pipeline return 404s try_files /staticfiles/$collected /course_static/$collected =404; } - - # Set django-pipelined files for studio to maximum cache time location ~ "/static/(?P[0-9a-f]{7}/.*)" { expires max; - - # Without this try_files, files that have been run through - # django-pipeline return 404s try_files /staticfiles/$collected /course_static/$collected =404; } diff --git a/configurator/templates/nginx/extra.conf b/deploy/templates/nginx/extra.conf similarity index 100% rename from configurator/templates/nginx/extra.conf rename to deploy/templates/nginx/extra.conf diff --git a/configurator/templates/nginx/lms.conf b/deploy/templates/nginx/lms.conf similarity index 87% rename from configurator/templates/nginx/lms.conf rename to deploy/templates/nginx/lms.conf index 65bb94b..295dbc1 100644 --- a/configurator/templates/nginx/lms.conf +++ b/deploy/templates/nginx/lms.conf @@ -15,18 +15,14 @@ server { ssl_certificate_key /etc/letsencrypt/live/{{ LMS_HOST }}/privkey.pem; {% endif %} - # Prevent invalid display courseware in IE 10+ with high privacy settings - add_header P3P 'CP="Open edX does not have a P3P policy."'; - - # Nginx does not support nested condition or or conditions so - # there is an unfortunate mix of conditonals here. - client_max_body_size 4M; rewrite ^(.*)/favicon.ico$ /static/images/favicon.ico last; # Disables server version feedback on pages and in headers server_tokens off; + # Prevent invalid display courseware in IE 10+ with high privacy settings + add_header P3P 'CP="Open edX does not have a P3P policy."'; location @proxy_to_lms_app { proxy_set_header X-Forwarded-Proto $scheme; @@ -80,17 +76,10 @@ server { # Set django-pipelined files to maximum cache time location ~ "/static/(?P.*\.[0-9a-f]{12}\..*)" { expires max; - # Without this try_files, files that have been run through - # django-pipeline return 404s try_files /staticfiles/$collected /course_static/$collected =404; } - - # Set django-pipelined files for studio to maximum cache time location ~ "/static/(?P[0-9a-f]{7}/.*)" { expires max; - - # Without this try_files, files that have been run through - # django-pipeline return 404s try_files /staticfiles/$collected /course_static/$collected =404; } diff --git a/configurator/templates/notes/universal.py b/deploy/templates/notes/universal.py similarity index 100% rename from configurator/templates/notes/universal.py rename to deploy/templates/notes/universal.py diff --git a/configurator/templates/openedx/cms.auth.json b/deploy/templates/openedx/cms.auth.json similarity index 100% rename from configurator/templates/openedx/cms.auth.json rename to deploy/templates/openedx/cms.auth.json diff --git a/configurator/templates/openedx/cms.env.json b/deploy/templates/openedx/cms.env.json similarity index 100% rename from configurator/templates/openedx/cms.env.json rename to deploy/templates/openedx/cms.env.json diff --git a/configurator/templates/openedx/lms.auth.json b/deploy/templates/openedx/lms.auth.json similarity index 100% rename from configurator/templates/openedx/lms.auth.json rename to deploy/templates/openedx/lms.auth.json diff --git a/configurator/templates/openedx/lms.env.json b/deploy/templates/openedx/lms.env.json similarity index 100% rename from configurator/templates/openedx/lms.env.json rename to deploy/templates/openedx/lms.env.json diff --git a/configurator/templates/openedx/oauth2.sh b/deploy/templates/openedx/oauth2.sh similarity index 100% rename from configurator/templates/openedx/oauth2.sh rename to deploy/templates/openedx/oauth2.sh diff --git a/configurator/templates/openedx/provision.sh b/deploy/templates/openedx/provision.sh similarity index 100% rename from configurator/templates/openedx/provision.sh rename to deploy/templates/openedx/provision.sh diff --git a/configurator/templates/openedx/stats b/deploy/templates/openedx/stats similarity index 100% rename from configurator/templates/openedx/stats rename to deploy/templates/openedx/stats diff --git a/openedx/settings/cms/__init__.py b/deploy/templates/openedx/universal/cms/__init__.py similarity index 100% rename from openedx/settings/cms/__init__.py rename to deploy/templates/openedx/universal/cms/__init__.py diff --git a/configurator/templates/openedx/universal/cms/development.py b/deploy/templates/openedx/universal/cms/development.py similarity index 100% rename from configurator/templates/openedx/universal/cms/development.py rename to deploy/templates/openedx/universal/cms/development.py diff --git a/configurator/templates/openedx/universal/cms/production.py b/deploy/templates/openedx/universal/cms/production.py similarity index 100% rename from configurator/templates/openedx/universal/cms/production.py rename to deploy/templates/openedx/universal/cms/production.py diff --git a/openedx/settings/lms/__init__.py b/deploy/templates/openedx/universal/lms/__init__.py similarity index 100% rename from openedx/settings/lms/__init__.py rename to deploy/templates/openedx/universal/lms/__init__.py diff --git a/configurator/templates/openedx/universal/lms/development.py b/deploy/templates/openedx/universal/lms/development.py similarity index 100% rename from configurator/templates/openedx/universal/lms/development.py rename to deploy/templates/openedx/universal/lms/development.py diff --git a/configurator/templates/openedx/universal/lms/production.py b/deploy/templates/openedx/universal/lms/production.py similarity index 100% rename from configurator/templates/openedx/universal/lms/production.py rename to deploy/templates/openedx/universal/lms/production.py diff --git a/configurator/templates/xqueue/universal.py b/deploy/templates/xqueue/universal.py similarity index 100% rename from configurator/templates/xqueue/universal.py rename to deploy/templates/xqueue/universal.py diff --git a/docker-compose-android.yml b/docker-compose-android.yml deleted file mode 100644 index 6f64d75..0000000 --- a/docker-compose-android.yml +++ /dev/null @@ -1,10 +0,0 @@ -version: "3" -services: - - android: - image: regis/openedx-android:latest - build: - context: ./android - volumes: - - ./config/android/:/openedx/config/ - - ./data/android/:/openedx/data diff --git a/docker-compose-notes.yml b/docker-compose-notes.yml deleted file mode 100644 index da4ca92..0000000 --- a/docker-compose-notes.yml +++ /dev/null @@ -1,18 +0,0 @@ -version: "3" -services: - - ############# Notes: backend store for edX Student Notes - notes: - image: regis/openedx-notes:hawthorn - build: - context: ./notes - networks: - default: - aliases: - - notes.openedx - volumes: - - ./config/notes:/openedx/config - - ./data/notes:/openedx/data - restart: unless-stopped - depends_on: - - mysql diff --git a/docker-compose-portainer.yml b/docker-compose-portainer.yml deleted file mode 100644 index 3cebc8d..0000000 --- a/docker-compose-portainer.yml +++ /dev/null @@ -1,12 +0,0 @@ -version: "3" -services: - - ############# Portainer: container supervision with a web UI - portainer: - image: portainer/portainer:latest - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - ./data/portainer:/data - restart: unless-stopped - depends_on: - - nginx diff --git a/docker-compose-xqueue.yml b/docker-compose-xqueue.yml deleted file mode 100644 index 792e2a5..0000000 --- a/docker-compose-xqueue.yml +++ /dev/null @@ -1,27 +0,0 @@ -version: "3" -services: - - ############# Xqueue: external grading of Open edX problems - xqueue: - image: regis/openedx-xqueue:hawthorn - build: - context: ./xqueue - volumes: - - ./config/xqueue:/openedx/config - - ./data/xqueue:/openedx/data - restart: unless-stopped - depends_on: - - mysql - - xqueue_consumer: - image: regis/openedx-xqueue:hawthorn - build: - context: ./xqueue - volumes: - - ./config/xqueue:/openedx/config - - ./data/xqueue:/openedx/data - restart: unless-stopped - # Run 12 workers per queue - command: ./manage.py run_consumer 12 - depends_on: - - mysql diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 0000000..69fa449 --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1 @@ +_build/ diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..298ea9e --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,19 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..ebbaf3c --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,173 @@ +# -*- coding: utf-8 -*- +# +# Configuration file for the Sphinx documentation builder. +# +# This file does only contain a selection of the most common options. For a +# full list see the documentation: +# http://www.sphinx-doc.org/en/master/config + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- Project information ----------------------------------------------------- + +project = 'Tutor' +copyright = '2018, Régis Behmo' +author = 'Régis Behmo' + +# The short X.Y version +version = '' +# The full version, including alpha/beta/rc tags +release = '' + + +# -- General configuration --------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = None + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'sphinx_rtd_theme' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + + +# -- Options for HTMLHelp output --------------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = 'tutordoc' + + +# -- Options for LaTeX output ------------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'tutor.tex', 'Tutor Documentation', + 'Régis Behmo', 'manual'), +] + + +# -- Options for manual page output ------------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'tutor', 'Tutor Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ---------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'tutor', 'Tutor Documentation', + author, 'tutor', 'One line description of project.', + 'Miscellaneous'), +] + + +# -- Options for Epub output ------------------------------------------------- + +# Bibliographic Dublin Core info. +epub_title = project + +# The unique identifier of the text. This can be a ISBN number +# or the project homepage. +# +# epub_identifier = '' + +# A unique identification for the text. +# +# epub_uid = '' + +# A list of files that should not be packed into the epub file. +epub_exclude_files = ['search.html'] diff --git a/docs/customise.rst b/docs/customise.rst new file mode 100644 index 0000000..219aad5 --- /dev/null +++ b/docs/customise.rst @@ -0,0 +1,84 @@ +Customising the ``openedx`` docker image +======================================== + +The LMS and the CMS all run from the ``openedx`` docker image. The base image is downloaded from `Docker Hub `_ when we run ``make update`` (or ``make all``). But you can also customise and build the image yourself. The base image is built with:: + + make build-openedx + +The following sections describe how to modify various aspects of the docker image. After you have built your own image, you can run it as usual:: + + make run + +Custom themes +------------- + +Comprehensive theming is enabled by default. Put your themes in ``openedx/themes/``:: + + openedx/themes/ + mycustomtheme1/ + cms/ + ... + lms/ + ... + mycustomtheme2/ + ... + +Then you must rebuild the openedx Docker image:: + + make build-openedx + +Make sure the assets can be served by the web server:: + + make assets + +Finally, follow the `Open edX documentation to enable your themes `_. + +Extra xblocks and requirements +------------------------------ + +Additional requirements can be added to the ``openedx/requirements/private.txt`` file. For instance:: + + echo "git+https://github.com/open-craft/xblock-poll.git" >> openedx/requirements/private.txt + +Then, the ``openedx`` docker image must be rebuilt:: + + make build-openedx + +To install xblocks from a private repository that requires authentication, you must first clone the repository inside the ``openedx/requirements`` folder on the host:: + + git clone git@github.com:me/myprivaterepo.git ./openedx/requirements/myprivaterepo + +Then, declare your extra requirements with the ``-e`` flag in ``openedx/requirements/private.txt``:: + + echo "-e ./myprivaterepo" >> openedx/requirements/private.txt + +Forked version of edx-platform +------------------------------ + +You may want to run your own flavor of edx-platform instead of the `official version `_. To do so, you will have to re-build the openedx image with the proper environment variables pointing to your repository and version:: + + EDX_PLATFORM_REPOSITORY=https://mygitrepo/edx-platform.git EDX_PLATFORM_VERSION=my-tag-or-branch make build-openedx + +You can then restart the services which will now be running your forked version of edx-platform:: + + make restart-openedx + +Note that your release must be a fork of Hawthorn in order to work. Otherwise, you may have important compatibility issues with other services. + +Running a different Docker image instead of `regis/openedx `_ +------------------------------------------------------------------------------------------------------ + +This is for people who have an account on `hub.docker.com `_ or a private image registry. You can build your image and push it to your repo. Then add the following content to the ``.env`` file:: + + OPENEDX_DOCKER_IMAGE=myusername/myimage:mytag + +Your own image will be used next time you run ``make run``. + +Note that the ``make build`` and ``make push`` command will no longer work as you expect and that you are responsible for building and pushing the image yourself. + +Maintainers +----------- + +The images are built, tagged and uploaded to Docker Hub in one command:: + + make dockerhub diff --git a/docs/dev.rst b/docs/dev.rst new file mode 100644 index 0000000..cf0b719 --- /dev/null +++ b/docs/dev.rst @@ -0,0 +1,70 @@ +.. _development: + +Open edX development +==================== + +In addition to running Open edX in production, you can use the docker containers for local development. This means you can hack on Open edX without setting up a Virtual Machine. Essentially, this replaces the devstack provided by edX. + +To begin with, define development settings:: + + export EDX_PLATFORM_SETTINGS=universal.development + +Run a local webserver +--------------------- + +:: + + make lms-runserver + make cms-runserver + +Open a bash shell +----------------- + +:: + + make lms + make cms + +Debug edx-platform +------------------ + +If you have one, you can point to a local version of `edx-platform `_ on your host machine:: + + export EDX_PLATFORM_PATH=/path/to/your/edx-platform + +Note that you should use an absolute path here, not a relative path (e.g: ``/path/to/edx-platform`` and not ``../edx-platform``). + +All development commands will then automatically mount your local repo. For instance, you can add a ``import pdb; pdb.set_trace()`` breakpoint anywhere in your code and run:: + + make lms-runserver + +Note: containers are built on the Hawthorn release. If you are working on a different version of Open edX, you will have to rebuild the images with the right ``EDX_PLATFORM_VERSION`` argument. You may also want to change the ``EDX_PLATFORM_REPOSITORY`` argument to point to your own fork of edx-platform. + +With a customised edx-platform repo, you must be careful to have settings that are compatible with the docker environment. You are encouraged to copy the ``universal.development`` settings files to our own repo: + + cp -r config/openedx/universal/lms/ /path/to/edx-platform/lms/envs/universal + cp -r config/openedx/universal/cms/ /path/to/edx-platform/cms/envs/universal + +You can then run your platform with the ``universal.development`` settings. + +Develop customised themes +------------------------- + +Run a local webserver:: + + make lms-runserver + +Watch the themes folders for changes:: + + make watch-themes + +Make changes to ``openedx/themes/yourtheme``: the theme assets should be automatically recompiled and visible at http://localhost:8000. + +Assets management +----------------- + +Assets building and collecting is made more difficult by the fact that development settings are `incorrectly loaded in Hawthorn `_. This should be fixed in the next Open edX release. Meanwhile, do not run ``paver update_assets`` while in development mode. When working locally on a theme, build assets by running in the container:: + + openedx-assets build + +This command will take quite some time to run. You can speed up this process by running only part of the full build. Run ``openedx-assets -h`` for more information. diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..39f3bbf --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,77 @@ +.. Tutor documentation master file, created by + sphinx-quickstart on Mon Dec 3 21:36:51 2018. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Tutor: Open edX 1-click install for everyone +============================================ + +.. image:: https://img.shields.io/travis/regisb/openedx-docker.svg + :alt: Build status + :target: https://travis-ci.org/regisb/openedx-docker + +.. image:: https://img.shields.io/github/issues/regisb/openedx-docker.svg + :alt: GitHub issues + :target: https://github.com/regisb/openedx-docker/issues + +.. image:: https://img.shields.io/github/issues-closed/regisb/openedx-docker.svg?colorB=brightgreen + :alt: GitHub closed issues + :target: https://github.com/regisb/openedx-docker/issues?q=is%3Aclosed + +Tutor is a one-click install of `Open edX `_, both for production and local development, inside docker containers. + +.. image:: https://asciinema.org/a/6DowVk4iJf3AJ2m8xlXDWJKh3.png + :alt: asciicast + :target: https://asciinema.org/a/6DowVk4iJf3AJ2m8xlXDWJKh3 + +---------------------------------- + +**Quickstart**:: + + git clone https://github.com/regisb/openedx-docker + cd openedx-docker/ + make all + +**That's it?** + +Yes :) When running `make all`, you will be asked a few questions about the configuration of your Open edX platform. Then, all the components for a functional Open edX platform will be downloaded and assembled to and you will have both an LMS and a CMS running behind a web server on port 80, ready for production. You should be able to access your platform at the address you gave during the configuration phase. + +All of this without touching your host environment! You don't even need root access. + +To be honest, I really don't like 1-click installs :-p They tend to hide much of the important details. So I strongly recommend you read the more detailed instructions below to understand what is going on exactly and to troubleshoot potential issues. Also, instructions are given to setup a local development environment. + +This might seem too simple to be true, but there's no magic -- just good packaging of already existing Open edX code. The code for building the Docker images is 100% available and fits in less than 1000 lines of code, in this repository. + +.. toctree:: + :maxdepth: 2 + :caption: User guide + + requirements + step + options + customise + dev + troubleshooting + missing + +Disclaimers & Warnings +---------------------- + +This project is the follow-up of my work on an `install from scratch of Open edX `_. It does not rely on any hack or complex deployment script. In particular, we do not use the Open edX `Ansible deployment playbooks `_. That means that the folks at edX.org are *not* responsible for troubleshooting issues of this project. Please don't bother Ned ;-) + +Do you have a problem? + +1. Carefully read the README, and in particular the :ref:`troubleshooting` section +2. Search for your problem in the `open and closed Github issues `_. +3. If necessary, open an `issue on Github `_. + + +Contributing +------------ + +Pull requests will be happily examined! However, we should be careful to keep the project lean and simple: both to use and to modify. Optional features should not make the user experience more complex. Instead, documentation on how to add the feature is preferred. + +License +------- + +This work is licensed under the terms of the `MIT License `_. diff --git a/docs/missing.rst b/docs/missing.rst new file mode 100644 index 0000000..26babcd --- /dev/null +++ b/docs/missing.rst @@ -0,0 +1,8 @@ +Missing features +================ + +- `discovery service `_ +- `ecommerce `_ +- `analytics `_ + +Those extra services were considered low priority while developing this project. However, most of them should not be too hard to add to a standard install. If you need one or more of these services, feel free to let me know by opening an issue. diff --git a/docs/options.rst b/docs/options.rst new file mode 100644 index 0000000..1419e85 --- /dev/null +++ b/docs/options.rst @@ -0,0 +1,72 @@ +Optional features +================= + +Some optional features may be activated or deactivated during the interactive configuration step. These features change configuration files (during the ``configure`` step) as well as make targets. + +SSL/TLS certificates for HTTPS access +------------------------------------- + +By activating this feature, a free SSL/TLS certificate from the `Let's Encrypt `_ certificate authority will be created for your platform. With this feature, **your platform will no longer be accessible in HTTP**. Calls to http urls will be redirected to https url. + +The following DNS records must exist and point to your server:: + + LMS_HOST (e.g: myopenedx.com) + preview.LMS_HOST (e.g: preview.myopenedx.com) + CMS_HOST (e.g: studio.myopenedx.com) + +Thus, **this feature will (probably) not work in development** because the DNS records will (probably) not point to your development machine. + +To download the certificate manually, run:: + + make https-certificate + +To renew the certificate, run this command once per month:: + + make https-certificate-renew + +Student notes +------------- + +With `notes `_, students can annotate portions of the courseware. + +.. image:: https://edx.readthedocs.io/projects/open-edx-building-and-running-a-course/en/open-release-hawthorn.master/_images/SFD_SN_bodyexample.png + :alt: Notes in action + +You should beware that the ``notes.`` domain name should be activated and point to your server. For instance, if your LMS is hosted at http://myopenedx.com, the notes service should be found at http://notes.myopenedx.com. + +Xqueue +------ + +`Xqueue `_ is for grading problems with external services. If you don't know what it is, you probably don't need it. + +Docker container web UI with `Portainer `_ +----------------------------------------------------------------- + +Portainer is a web UI for managing docker containers. It lets you view your entire Open edX platform at a glace. Try it! It's really cool. + +.. image:: https://portainer.io/images/screenshots/portainer.gif + :alt: Portainer demo + +After launching your platfom, the web UI will be available at `http://portainer.localhost `_ and http://portainer.YOUR_LMS_HOST. You will be asked to define a password for the admin user. Then, select a "Local environment" to work on and hit "Connect". You're done! Select the "local" group to view all running containers. Amon many other things, you'll be able to view the logs for each container, which is really useful. + +Android app (beta) +------------------ + +The Android app for your platform can be easily built in just one command:: + + make android + +If all goes well, the debuggable APK for your platform should then be available in ./data/android. To obtain a release APK, you will need to obtain credentials from the app store and add them to ``config/android/gradle.properties``. Then run:: + + make android-release + +Building the Android app for an Open edX platform is currently labeled as a **beta feature** because it was not fully tested yet. In particular, there is no easy mechanism for overriding the edX assets in the mobile app. This is still a work-in-progress. + +Stats +----- + +By default, the install script will collect some information about your install and send it to a private server. The only transmitted information are the LMS domain name and the ID of the install. To disable stats collection, define the following environment variable:: + + export DISABLE_STATS=1 + +If you decide to disable stats, please send me a message to tell me about your platform! diff --git a/docs/requirements.rst b/docs/requirements.rst new file mode 100644 index 0000000..eab3919 --- /dev/null +++ b/docs/requirements.rst @@ -0,0 +1,15 @@ +Requirements +============ + +The only prerequisite for running this is a working docker install. You will need both docker and docker-compose. Follow the instructions from the official documentation: + +- `Docker `_ +- `Docker compose `_ + +Note that the production web server container will bind to port 80, so if you already have a web server running (Apache or Nginx, for instance), you should stop it. + +You should be able to run Open edX on any platform that supports Docker and Python, including Mac OS and Windows. For now, only Ubuntu 16.04 was tested but we have no reason to believe the install would not work on a different OS. + +At a minimum, the server running the containers should have 4 Gb of RAM; otherwise, the deployment procedure will crash during migrations (see the :ref:`troubleshooting` section). + +Also, the host running the containers should be a 64 bit platform. (images are not built for i386 systems) diff --git a/docs/step.rst b/docs/step.rst new file mode 100644 index 0000000..62d16d9 --- /dev/null +++ b/docs/step.rst @@ -0,0 +1,112 @@ +Step-by-step install +==================== + +Configure +--------- + +:: + + make configure + +This is the only non-automatic step in the install process. You will be asked various questions about your Open edX platform and appropriate configuration files will be generated. If you would like to automate this step then you should run ``make configure`` interactively once. After that, you will have a ``config.json`` file at the root of the repository. Just upload it to wherever you want to run Open edX and then run ``make configure SILENT=1`` instead of ``make configure``. All values from ``config.json`` will be automatically loaded. + +Download +-------- + +:: + + make update + +You will need to download the docker images from `Docker Hub `_. Depending on your bandwidth, this might take a long time. Minor image updates will be incremental, and thus much faster. + +Database creation, migrations and collection of static assets +------------------------------------------------------------- + +:: + + make databases + make assets + +These commands should be run just once. They will create the required databases tables, apply database migrations and make sure that static assets, such as images, stylesheets and Javascript dependencies, can be served by the nginx container. + +If migrations are stopped with a ``Killed`` message, this certainly means the docker containers don't have enough RAM. See the :ref:`troubleshooting` section. + +Running Open edX +---------------- + +:: + + make run + +This will launch the various docker containers required for your Open edX platform. The LMS and the Studio will then be reachable at the domain name you specified during the configuration step. You can also access them at http://localhost and http://studio.localhost. + +Additional commands +------------------- + +All available commands can be listed by running:: + + make help + +Creating a new user with staff and admin rights +----------------------------------------------- + +You will most certainly need to create a user to administer the platform. Just run:: + + make create-staff-user USERNAME=yourusername EMAIL=user@email.com + +You will asked to set the user password interactively. + +Importing the demo course +------------------------- + +On a fresh install, your platform will not have a single course. To import the `Open edX demo course `, run:: + + make import-demo-course + +Daemonizing +----------- + +In production, you will probably want to daemonize the services. Instead of ``make run``, run:: + + make daemonize + +And then, to stop all services:: + + make stop + +Updating the course search index +-------------------------------- + +The course search index can be updated with:: + + make reindex-courses + +Run this command periodically to ensure that course search results are always up-to-date. + +Logging +------- + +To view the logs from all containers use the `docker-compose logs `_ command:: + + docker-compose logs -f + +To view the logs from just one container, for instance the web server:: + + docker-compose logs -f nginx + +The last commands produce the logs since the creation of the containers, which can be a lot. Similar to a ``tail -f``, you can run:: + + docker-compose logs --tail=0 -f + +Debugging +--------- + +Open a bash shell in the lms or the cms:: + + make lms + make cms + +Open a python shell in the lms or the cms:: + + make lms-python + make cms-python diff --git a/docs/troubleshooting.rst b/docs/troubleshooting.rst new file mode 100644 index 0000000..f2d3e81 --- /dev/null +++ b/docs/troubleshooting.rst @@ -0,0 +1,73 @@ +.. _troubleshooting: + +Troubleshooting +=============== + +"Cannot start service nginx: driver failed programming external connectivity" +----------------------------------------------------------------------------- + +The containerized Nginx needs to listen to ports 80 and 443 on the host. If there is already a webserver, such as Apache or Nginx, running on the host, the nginx container will not be able to start. There are two solutions: + +1. Stop Apache or Nginx on the host:: + + sudo systemctl stop apache2 + sudo systemctl stop nginx + +However, you might now want to do that if you need a webserver for running non-Open edX related applications. In such cases... + +2. Run the nginx container on different ports: you can create a ``.env`` file in the ``openedx-docker`` directory in which you indicate different ports. For instance:: + + cat .env + NGINX_HTTP_PORT=81 + NGINX_HTTPS_PORT=444 + +In this example, the nginx container ports would be mapped to 81 and 444, instead of 80 and 443. + +You should note that with the latter solution, it is your responsibility to configure the webserver on the host as a proxy to the nginx container. See `this `_ for http, and `this `_ for https. + +Help! The Docker containers are eating all my RAM/CPU/CHEESE +------------------------------------------------------------ + +You can identify which containers are consuming most resources by running:: + + docker stats + +"Running migrations... Killed!" +------------------------------- + +The LMS and CMS containers require at least 4 GB RAM, in particular to run the Open edX SQL migrations. On Docker for Mac, by default, containers are allocated at most 2 GB of RAM. On Mac OS, if the ``make all`` command dies after displaying "Running migrations", you most probably need to increase the allocated RAM. Follow `these instructions from the official Docker documentation `_. + + +``Build failed running pavelib.servers.lms: Subprocess return code: 1`` +----------------------------------------------------------------------- + +:: + + python manage.py lms --settings=development print_setting STATIC_ROOT 2>/dev/null + ... + Build failed running pavelib.servers.lms: Subprocess return code: 1`" + +This might occur when you run a ``paver`` command. ``/dev/null`` eats the actual error, so you will have to run the command manually. Run ``make lms`` (or ``make cms``) to open a bash session and then:: + + python manage.py lms --settings=development print_setting STATIC_ROOT + +Of course, you should replace `development` with your own settings. The error produced should help you better understand what is happening. + +``ValueError: Unable to configure handler 'local'`` +--------------------------------------------------- + +:: + + ValueError: Unable to configure handler 'local': [Errno 2] No such file or directory + +This will occur if you try to run a development environment without patching the LOGGING configuration, as indicated in the `development_` section. Maybe you correctly patched the development settings, but they are not taken into account? For instance, you might have correctly defined the ``EDX_PLATFORM_SETTINGS`` environment variable, but ``paver`` uses the ``devstack`` settings (which does not patch the ``LOGGING`` variable). This is because calling ``paver lms --settings=development`` or ``paver cms --settings=development`` ignores the ``--settings`` argument. Yes, it might be considered an edx-platform bug... Instead, you should run the ``update_assets`` and ``runserver`` commands, as explained above. + +"``TypeError: get_logger_config() got an unexpected keyword argument 'debug'``" +------------------------------------------------------------------------------- + +This might occur when you try to run the latest version of ``edx-platform``, and not a version close to ``hawthorn.master``. It is no longer necessary to patch the ``LOGGING`` configuration in the latest ``edx-platform`` releases, as indicated in the `development_` section, so you should remove the call to ``get_logger_config`` altogether from your development settings. + +The chosen default language does not display properly +----------------------------------------------------- + +By default, Open edX comes with a `limited set ` of translation/localization files. To complement these languages, we add locales from the `openedx-i18n project `_. But not all supported locales are downloaded. In some cases, the chosen default language will not display properly because if was not packaged in either edx-platform or openedx-i18n. If you feel like your language should be packaged, please `open an issue on the openedx-i18n project `_.