diff --git a/.gitignore b/.gitignore index 4100e1d..b0f2efb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,6 @@ .*.swp -config.json -openedx/config/*.env.json -openedx/config/*.auth.json -config/rendered/nginx/*.conf +config/openedx/*.json +config/nginx/*.conf mysql/config/database mysql/config/username mysql/config/password diff --git a/Makefile b/Makefile index 16bc2eb..a49ce3c 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ -.PHONY: all configure build migrate assets up daemon +.PHONY: all configure build update migrate assets up daemon USERID ?= $$(id -u) DOCKER_COMPOSE_RUN = docker-compose run --rm -e USERID=$(USERID) -EDX_PLATFORM_SETTINGS ?= production +EDX_PLATFORM_SETTINGS ?= universal.production DOCKER_COMPOSE_RUN += -e SETTINGS=$(EDX_PLATFORM_SETTINGS) ifneq ($(EDX_PLATFORM_PATH),) DOCKER_COMPOSE_RUN += --volume="$(EDX_PLATFORM_PATH):/openedx/edx-platform" @@ -11,23 +11,23 @@ endif DOCKER_COMPOSE_RUN_LMS = $(DOCKER_COMPOSE_RUN) -p 8000:8000 lms DOCKER_COMPOSE_RUN_CMS = $(DOCKER_COMPOSE_RUN) -p 8001:8001 cms -all: configure build migrate assets daemon +all: configure update migrate assets daemon ##################### Bootstrapping configure: ./configure -build: - docker-compose build +update: + docker-compose pull migrate: $(DOCKER_COMPOSE_RUN_LMS) bash -c "wait-for-greenlight.sh && ./manage.py lms migrate" $(DOCKER_COMPOSE_RUN_CMS) bash -c "wait-for-greenlight.sh && ./manage.py cms migrate" assets: - $(DOCKER_COMPOSE_RUN_LMS) paver update_assets lms --settings=$(EDX_PLATFORM_SETTINGS) - $(DOCKER_COMPOSE_RUN_CMS) paver update_assets cms --settings=$(EDX_PLATFORM_SETTINGS) + $(DOCKER_COMPOSE_RUN) lms paver update_assets lms --settings=$(EDX_PLATFORM_SETTINGS) + $(DOCKER_COMPOSE_RUN) cms paver update_assets cms --settings=$(EDX_PLATFORM_SETTINGS) ##################### Running @@ -44,10 +44,10 @@ stop: ##################### Extra import-demo-course: - $(DOCKER_COMPOSE_RUN_CMS) /bin/bash -c "git clone https://github.com/edx/edx-demo-course ../edx-demo-course && git -C ../edx-demo-course checkout open-release/ginkgo.master && python ./manage.py cms import ../data ../edx-demo-course" + $(DOCKER_COMPOSE_RUN) cms /bin/bash -c "git clone https://github.com/edx/edx-demo-course ../edx-demo-course && git -C ../edx-demo-course checkout open-release/ginkgo.master && python ./manage.py cms import ../data ../edx-demo-course" create-staff-user: - $(DOCKER_COMPOSE_RUN_LMS) /bin/bash -c "./manage.py lms manage_user --superuser --staff ${USERNAME} ${EMAIL} && ./manage.py lms changepassword ${USERNAME}" + $(DOCKER_COMPOSE_RUN) lms /bin/bash -c "./manage.py lms manage_user --superuser --staff ${USERNAME} ${EMAIL} && ./manage.py lms changepassword ${USERNAME}" ##################### Development @@ -58,6 +58,22 @@ cms: $(DOCKER_COMPOSE_RUN_CMS) bash lms-shell: - $(DOCKER_COMPOSE_RUN_LMS) ./manage.py lms shell + $(DOCKER_COMPOSE_RUN) lms ./manage.py lms shell cms-shell: - $(DOCKER_COMPOSE_RUN_LMS) ./manage.py cms shell + $(DOCKER_COMPOSE_RUN) cms ./manage.py cms shell + + +#################### Deploying to docker hub + +build: + docker-compose build + +tag: + docker tag openedx overhangio/openedx:ginkgo + docker tag openedx overhangio/openedx:latest + +push: + docker push overhangio/openedx:ginkgo + docker push overhangio/openedx:latest + +dockerhub: build tag push diff --git a/README.md b/README.md index 14b75e8..e273b31 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ All you have to do is [download the content of this repository](https://codeload make all -You will be asked some questions about the configuration of your Open edX platform. The build step will take some time, but then you will have both an LMS and a CMS running behind a web server on port 80, ready for production. You should be able to access your platform at the address you gave during the configuration phase. +You will be asked some questions about the configuration of your Open edX platform. You will have to download a ~900 Mb docker image, but then you will have both an LMS and a CMS running behind a web server on port 80, ready for production. You should be able to access your platform at the address you gave during the configuration phase. To be honest, I really don't like 1-click installs :-p They tend to hide much of the important details. So I strongly recommend you read the more detailed instructions below to understand what is going on exactly and to troubleshoot potential issues. Also, instructions are given to setup a local development environment. @@ -41,11 +41,11 @@ At a minimum, the server running the containers should have 4 Gb of RAM. This is the only non-automatic step in the install process. You will be asked various questions about your Open edX platform and appropriate configuration files will be generated. If you would like to automate this step then you should run `make configure` interactively once. After that, you will have a `config.json` file at the root of the repository. Just upload it to wherever you want to run Open edX and then run `./configure --silent` instead of `make configure`. All values from `config.json` will be automatically loaded. -### Build +### Download - make build # go get a coffee + make update -Building the images may require a long time, depending on your bandwidth, as you will have to checkout the `edx-platform` repository (> 1 Gb) as well as the python dependencies. Once the images have been built, this step can be repeated very quickly. +You will need to download the docker images from [Docker Hub](https://hub.docker.com/r/overhangio/openedx/). Depending on your bandwidth, this might take a long time. Minor image updates will be incremental, and thus much faster. ### Migrations and assets @@ -113,10 +113,13 @@ Open a python shell in the lms or the cms: make lms-shell make cms-shell + ## For developers In addition to running Open edX in production, you can use the docker containers for local development. This means you can hack on Open edX without setting up a Virtual Machine. Essentially, this replaces the devstack provided by edX. +(Note: containers are built on the Ginkgo release. If you are working on a different version of Open edX, you will have to rebuild the images with a different `EDX_PLATFORM_VERSION` argument. You may also want to change the `EDX_PLATFORM_REPOSITORY` argument to point to your own fork of edx-platform.) + First, configure your project such that the LMS and the CMS can be accessed locally: make configure @@ -125,12 +128,24 @@ First, configure your project such that the LMS and the CMS can be accessed loca Your website domain name for teachers (CMS) (default: "studio.myopenedx.com"): localhost:8001 ... -Then, build the images and prepare the database: +### Standard devstack - make build - make migrate +Define development settings (on the host): -Point to your local install of [edx-platform](https://github.com/edx/edx-platform/) on your host machine: + export EDX_PLATFORM_SETTINGS=universal.development + +Then open an LMS shell: + + make lms + +You can then collect assets and run a local web server, as usual: + + paver update_assets lms --settings=universal.development + ./manage.py lms runserver 0.0.0.0:8000 + +### Custom devstack + +If you have one, you can point to a local version of [edx-platform](https://github.com/edx/edx-platform/) on your host machine: export EDX_PLATFORM_PATH=/path/to/your/edx-platform @@ -138,34 +153,9 @@ Note that you should use an absolute path here, not a relative path (e.g: `/path Point to your settings file: - export EDX_PLATFORM_SETTINGS=development + export EDX_PLATFORM_SETTINGS=mysettings.py -In this example, you should have a `development.py` file in `edx-platform/lms/envs` and `edx-platform/cms/envs`. Here is a minimal settings file: - - from .devstack import * - - # Load module store settings from config files - update_module_store_settings(MODULESTORE, doc_store_settings=DOC_STORE_CONFIG) - - # Set uploaded media file path - MEDIA_ROOT = "/openedx/data/uploads/" - - # Deactivate forums - FEATURES['ENABLE_DISCUSSION_SERVICE'] = False - - # Activate dev_env for logging, otherwise rsyslog is required (but it is - # not available in docker). - LOGGING = get_logger_config(LOG_DIR, - logging_env=ENV_TOKENS['LOGGING_ENV'], - debug=False, - dev_env=True, - service_variant=SERVICE_VARIANT) - - # Create folders if necessary - import os - for folder in [LOG_DIR, MEDIA_ROOT, STATIC_ROOT_BASE]: - if not os.path.exists(folder): - os.makedirs(folder) +In this example, you should have a `mysettings.py` file in `edx-platform/lms/envs` and `edx-platform/cms/envs`. Development settings file for docker are a bit different from stock devstack settings. For valid development settings files, check [`config/openedx/universal/lms/development.py`](https://github.com/regisb/openedx-docker/blob/master/config/openedx/universal/lms/development.py) and [`config/openedx/universal/cms/development.py`](https://github.com/regisb/openedx-docker/blob/master/config/openedx/universal/cms/development.py) You are ready to go! Run: @@ -175,13 +165,17 @@ Or: make cms -This will open a shell in the LMS container. You can then run just any command you are used to. For example, collect assets and run a local server: +This will open a shell in the LMS (or CMS) container. You can then run just any command you are used to. For example, collect assets and run a local server: - paver update_assets lms --settings=development + paver update_assets lms --settings=mysettings ./manage.py lms runserver 0.0.0.0:8000 -Note that the containers are built on the Ginkgo release. If you are working on a different version of Open edX, you will have to rebuild the images with a different `EDX_PLATFORM_VERSION` argument. You may also want to change the `EDX_PLATFORM_REPOSITORY` argument to point to your own fork of edx-platform. +## Maintainers +The images are built, tagged and uploaded to Docker Hub in one command: + + make dockerhub + ## Troubleshooting ### "Running migrations... Killed!" diff --git a/config/templates/nginx/cms.conf.templ b/config/nginx/templates/cms.conf.templ similarity index 100% rename from config/templates/nginx/cms.conf.templ rename to config/nginx/templates/cms.conf.templ diff --git a/config/templates/nginx/lms.conf.templ b/config/nginx/templates/lms.conf.templ similarity index 100% rename from config/templates/nginx/lms.conf.templ rename to config/nginx/templates/lms.conf.templ diff --git a/openedx/config/templates/cms.auth.json.templ b/config/openedx/templates/cms.auth.json.templ similarity index 100% rename from openedx/config/templates/cms.auth.json.templ rename to config/openedx/templates/cms.auth.json.templ diff --git a/openedx/config/templates/cms.env.json.templ b/config/openedx/templates/cms.env.json.templ similarity index 100% rename from openedx/config/templates/cms.env.json.templ rename to config/openedx/templates/cms.env.json.templ diff --git a/openedx/config/templates/lms.auth.json.templ b/config/openedx/templates/lms.auth.json.templ similarity index 100% rename from openedx/config/templates/lms.auth.json.templ rename to config/openedx/templates/lms.auth.json.templ diff --git a/openedx/config/templates/lms.env.json.templ b/config/openedx/templates/lms.env.json.templ similarity index 100% rename from openedx/config/templates/lms.env.json.templ rename to config/openedx/templates/lms.env.json.templ diff --git a/config/rendered/nginx/.gitignore b/config/openedx/universal/cms/__init__.py similarity index 100% rename from config/rendered/nginx/.gitignore rename to config/openedx/universal/cms/__init__.py diff --git a/config/openedx/universal/cms/development.py b/config/openedx/universal/cms/development.py new file mode 100644 index 0000000..18dd59a --- /dev/null +++ b/config/openedx/universal/cms/development.py @@ -0,0 +1,24 @@ +from ..devstack import * + +# Load module store settings from config files +update_module_store_settings(MODULESTORE, doc_store_settings=DOC_STORE_CONFIG) + +# Set uploaded media file path +MEDIA_ROOT = "/openedx/data/uploads/" + +# Deactivate forums +FEATURES['ENABLE_DISCUSSION_SERVICE'] = False + +# Activate dev_env for logging, otherwise rsyslog is required (but it is +# not available in docker). +LOGGING = get_logger_config(LOG_DIR, + logging_env=ENV_TOKENS['LOGGING_ENV'], + debug=False, + dev_env=True, + service_variant=SERVICE_VARIANT) + +# Create folders if necessary +import os +for folder in [LOG_DIR, MEDIA_ROOT, STATIC_ROOT_BASE]: + if not os.path.exists(folder): + os.makedirs(folder) diff --git a/openedx/config/production_common.py b/config/openedx/universal/cms/production.py similarity index 73% rename from openedx/config/production_common.py rename to config/openedx/universal/cms/production.py index 509b974..febefca 100644 --- a/openedx/config/production_common.py +++ b/config/openedx/universal/cms/production.py @@ -1,5 +1,5 @@ import os -from .aws import * +from ..aws import * update_module_store_settings(MODULESTORE, doc_store_settings=DOC_STORE_CONFIG) @@ -18,3 +18,11 @@ LOGGING = get_logger_config(LOG_DIR, for folder in [LOG_DIR, MEDIA_ROOT, STATIC_ROOT_BASE]: if not os.path.exists(folder): os.makedirs(folder) + +ALLOWED_HOSTS = [ + ENV_TOKENS.get('CMS_BASE'), +] + +DEFAULT_FROM_EMAIL = 'registration@' + ENV_TOKENS['LMS_BASE'] +DEFAULT_FEEDBACK_EMAIL = 'feedback@' + ENV_TOKENS['LMS_BASE'] +SERVER_EMAIL = 'devops@' + ENV_TOKENS['LMS_BASE'] diff --git a/config/openedx/universal/lms/__init__.py b/config/openedx/universal/lms/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/config/openedx/universal/lms/development.py b/config/openedx/universal/lms/development.py new file mode 100644 index 0000000..18dd59a --- /dev/null +++ b/config/openedx/universal/lms/development.py @@ -0,0 +1,24 @@ +from ..devstack import * + +# Load module store settings from config files +update_module_store_settings(MODULESTORE, doc_store_settings=DOC_STORE_CONFIG) + +# Set uploaded media file path +MEDIA_ROOT = "/openedx/data/uploads/" + +# Deactivate forums +FEATURES['ENABLE_DISCUSSION_SERVICE'] = False + +# Activate dev_env for logging, otherwise rsyslog is required (but it is +# not available in docker). +LOGGING = get_logger_config(LOG_DIR, + logging_env=ENV_TOKENS['LOGGING_ENV'], + debug=False, + dev_env=True, + service_variant=SERVICE_VARIANT) + +# Create folders if necessary +import os +for folder in [LOG_DIR, MEDIA_ROOT, STATIC_ROOT_BASE]: + if not os.path.exists(folder): + os.makedirs(folder) diff --git a/openedx/config/production_lms.py b/config/openedx/universal/lms/production.py similarity index 52% rename from openedx/config/production_lms.py rename to config/openedx/universal/lms/production.py index e5f87be..45b4032 100644 --- a/openedx/config/production_lms.py +++ b/config/openedx/universal/lms/production.py @@ -1,4 +1,23 @@ -from .production_common import * +import os +from ..aws import * + +update_module_store_settings(MODULESTORE, doc_store_settings=DOC_STORE_CONFIG) + +MEDIA_ROOT = "/openedx/data/uploads/" +FEATURES['ENABLE_DISCUSSION_SERVICE'] = False + +# We need to activate dev_env for logging, otherwise rsyslog is required (but +# it is not available in docker). +LOGGING = get_logger_config(LOG_DIR, + logging_env=ENV_TOKENS['LOGGING_ENV'], + debug=False, + dev_env=True, + service_variant=SERVICE_VARIANT) + +# Create folders if necessary +for folder in [LOG_DIR, MEDIA_ROOT, STATIC_ROOT_BASE]: + if not os.path.exists(folder): + os.makedirs(folder) ALLOWED_HOSTS = [ ENV_TOKENS.get('LMS_BASE'), diff --git a/configure b/configure index 0dd7f7e..1f64d38 100755 --- a/configure +++ b/configure @@ -91,8 +91,13 @@ def template_class(user_delimiter='$'): def main(): + # Hack to handle older config.json files that used to be in a different location + # TODO remove me + if os.path.exists("config.json"): + os.rename("config.json", os.path.join("config", "openedx", "config.json")) + parser = argparse.ArgumentParser("Config file generator for Open edX") - parser.add_argument('-c', '--config', default="config.json", + parser.add_argument('-c', '--config', default=os.path.join("config", "openedx", "config.json"), help="Load default values from this file. Config values will be saved there.") parser.add_argument('-s', '--silent', action='store_true', help=( @@ -133,23 +138,23 @@ def main(): # Open edX substitute( - os.path.join('openedx', 'config', 'templates', 'lms.env.json.templ'), - os.path.join('openedx', 'config', 'lms.env.json'), + os.path.join('config', 'openedx', 'templates', 'lms.env.json.templ'), + os.path.join('config', 'openedx', 'lms.env.json'), **configurator.as_dict() ) substitute( - os.path.join('openedx', 'config', 'templates', 'cms.env.json.templ'), - os.path.join('openedx', 'config', 'cms.env.json'), + os.path.join('config', 'openedx', 'templates', 'cms.env.json.templ'), + os.path.join('config', 'openedx', 'cms.env.json'), **configurator.as_dict() ) substitute( - os.path.join('openedx', 'config', 'templates', 'lms.auth.json.templ'), - os.path.join('openedx', 'config', 'lms.auth.json'), + os.path.join('config', 'openedx', 'templates', 'lms.auth.json.templ'), + os.path.join('config', 'openedx', 'lms.auth.json'), **configurator.as_dict() ) substitute( - os.path.join('openedx', 'config', 'templates', 'cms.auth.json.templ'), - os.path.join('openedx', 'config', 'cms.auth.json'), + os.path.join('config', 'openedx', 'templates', 'cms.auth.json.templ'), + os.path.join('config', 'openedx', 'cms.auth.json'), **configurator.as_dict() ) @@ -174,17 +179,17 @@ def main(): # We need a different delimiter in nginx config files, because the '$' sign # is widely used there substitute( - os.path.join('config', 'templates', 'nginx', 'lms.conf.templ'), - os.path.join('config', 'rendered', 'nginx', 'lms.conf'), + os.path.join('config', 'nginx', 'templates', 'lms.conf.templ'), + os.path.join('config', 'nginx', 'lms.conf'), delimiter='£', **configurator.as_dict() ) substitute( - os.path.join('config', 'templates', 'nginx', 'cms.conf.templ'), - os.path.join('config', 'rendered', 'nginx', 'cms.conf'), + os.path.join('config', 'nginx', 'templates', 'cms.conf.templ'), + os.path.join('config', 'nginx', 'cms.conf'), delimiter='£', **configurator.as_dict() ) - print("\nConfiguration files were successfuly generated. You may now build the app containers.") + print("\nConfiguration files were successfuly generated. You may now run the app containers.") if __name__ == '__main__': diff --git a/docker-compose.yml b/docker-compose.yml index 0f09386..243b3c3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -52,13 +52,14 @@ services: ############# LMS and CMS lms: + image: overhangio/openedx:ginkgo build: context: ./openedx environment: SERVICE_VARIANT: lms restart: unless-stopped volumes: - - ./openedx/config:/openedx/config + - ./config/openedx:/openedx/config - ./data/lms:/openedx/data depends_on: - memcached @@ -68,13 +69,14 @@ services: - smtp cms: + image: overhangio/openedx:ginkgo build: context: ./openedx environment: SERVICE_VARIANT: cms restart: unless-stopped volumes: - - ./openedx/config:/openedx/config + - ./config/openedx:/openedx/config - ./data/cms:/openedx/data depends_on: - memcached @@ -87,6 +89,7 @@ services: # We could probably create one service per queue here. For small instances, it is not necessary. lms_worker: + image: overhangio/openedx:ginkgo build: context: ./openedx environment: @@ -96,12 +99,13 @@ services: environment: C_FORCE_ROOT: "1" # run celery tasks as root #nofear volumes: - - ./openedx/config:/openedx/config + - ./config/openedx:/openedx/config - ./data/lms_worker:/openedx/data depends_on: - lms cms_worker: + image: overhangio/openedx:ginkgo build: context: ./openedx environment: @@ -111,7 +115,7 @@ services: environment: C_FORCE_ROOT: "1" # run celery tasks as root #nofear volumes: - - ./openedx/config:/openedx/config + - ./config/openedx:/openedx/config - ./data/cms_worker:/openedx/data depends_on: - cms diff --git a/openedx/Dockerfile b/openedx/Dockerfile index e423c7b..b78bf08 100644 --- a/openedx/Dockerfile +++ b/openedx/Dockerfile @@ -38,13 +38,12 @@ RUN pip install --src ../venv/src -r requirements/edx/paver.txt # Install nodejs requirements RUN npm install -# Copy configuration files in common folder (which can later be mounted as a volume) +# Link configuration files to common /openedx/config folder, which should later +# be mounted as a volume. Note that this image will not be functional until +# config files have been mounted inside the container RUN mkdir /openedx/config -COPY ./config/* /openedx/config/ -RUN ln -s /openedx/config/production_lms.py /openedx/edx-platform/lms/envs/production.py \ - && ln -s /openedx/config/production_cms.py /openedx/edx-platform/cms/envs/production.py \ - && ln -s /openedx/config/production_common.py /openedx/edx-platform/lms/envs/production_common.py \ - && ln -s /openedx/config/production_common.py /openedx/edx-platform/cms/envs/production_common.py +RUN ln -s /openedx/config/universal/lms/ /openedx/edx-platform/lms/envs/universal \ + && ln -s /openedx/config/universal/cms/ /openedx/edx-platform/cms/envs/universal RUN ln -s /openedx/config/lms.env.json /openedx/ \ && ln -s /openedx/config/cms.env.json /openedx/ \ && ln -s /openedx/config/lms.auth.json /openedx/ \ @@ -56,7 +55,7 @@ COPY ./bin/docker-entrypoint.sh /usr/local/bin/ # service variant is "lms" or "cms" ENV SERVICE_VARIANT lms -ENV SETTINGS production +ENV SETTINGS universal.production # Entrypoint will fix permissiosn of all files and run commands as openedx ENTRYPOINT ["docker-entrypoint.sh"] diff --git a/openedx/config/production_cms.py b/openedx/config/production_cms.py deleted file mode 100644 index 24af94d..0000000 --- a/openedx/config/production_cms.py +++ /dev/null @@ -1,9 +0,0 @@ -from .production_common import * - -ALLOWED_HOSTS = [ - ENV_TOKENS.get('CMS_BASE'), -] - -DEFAULT_FROM_EMAIL = 'registration@' + ENV_TOKENS['LMS_BASE'] -DEFAULT_FEEDBACK_EMAIL = 'feedback@' + ENV_TOKENS['LMS_BASE'] -SERVER_EMAIL = 'devops@' + ENV_TOKENS['LMS_BASE']