diff --git a/.gitignore b/.gitignore index 9925492..615722b 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ data-*/ TODO openedx/requirements/private.txt .env +!.gitignore diff --git a/.travis.yml b/.travis.yml index 509f02d..5a02a12 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ script: - make configure SILENT=1 CONFIGURE_OPTS="-e SETTING_ACTIVATE_NOTES=1 -e SETTING_ACTIVATE_XQUEUE=1" - make build - make databases - #- make assets # too time-consuming + - make assets deploy: provider: script script: docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && make push diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d065db..2e6c340 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ # Changelog +- 2018-12-07 [Improvement] Bundle theme and production static assets in the openedx docker image - 2018-12-02 [Feature] Download extra locales from [openedx-i18n](https://github.com/regisb/openedx-i18n/) to the Open edX Docker image - 2018-11-28 [Feature] Easily change openedx docker image - 2018-11-28 [Feature] Enable comprehensive theming! diff --git a/Makefile b/Makefile index 079942b..e7ba608 100644 --- a/Makefile +++ b/Makefile @@ -104,19 +104,14 @@ reindex-courses: ## Refresh course index so they can be found in the LMS search #assets-cms: ## Collect static assets for the CMS # $(DOCKER_COMPOSE_RUN_OPENEDX) cms -e NO_PREREQ_INSTALL=True cms paver update_assets cms --settings=$(EDX_PLATFORM_SETTINGS) -assets: assets-lms assets-cms ## Generate production-ready static assets assets-development: assets-development-lms assets-development-cms ## Generate static assets for local development -assets-lms: - $(DOCKER_COMPOSE_RUN_OPENEDX) --no-deps lms bash -c \ - "NODE_ENV=production ./node_modules/.bin/webpack --config=webpack.prod.config.js \ - && ./manage.py lms --settings=$(EDX_PLATFORM_SETTINGS) compile_sass lms \ - && python -c \"import pavelib.assets; pavelib.assets.collect_assets(['lms'], '$(EDX_PLATFORM_SETTINGS)')\"" -assets-cms: - $(DOCKER_COMPOSE_RUN_OPENEDX) --no-deps cms bash -c \ - "NODE_ENV=production ./node_modules/.bin/webpack --config=webpack.prod.config.js \ - && ./manage.py cms --settings=$(EDX_PLATFORM_SETTINGS) compile_sass studio \ - && python -c \"import pavelib.assets; pavelib.assets.collect_assets(['studio'], '$(EDX_PLATFORM_SETTINGS)')\"" +assets: ## Generate production-ready static assets + docker-compose -f docker-compose-scripts.yml run --rm \ + --volume=$(PWD)/data/lms/:/data/lms/ --volume=$(PWD)/data/cms/:/data/cms/ openedx bash -c \ + "rm -rf /data/lms/staticfiles /data/cms/staticfiles \ + && cp -r /openedx/data/staticfiles /data/lms/ \ + && cp -r /openedx/data/staticfiles /data/cms/" assets-development-lms: $(DOCKER_COMPOSE_RUN_OPENEDX) --no-deps lms bash -c \ "xmodule_assets common/static/xmodule \ @@ -178,6 +173,9 @@ endif ifdef EDX_PLATFORM_VERSION openedx_build_args += --build-arg="EDX_PLATFORM_VERSION=$(EDX_PLATFORM_VERSION)" endif +ifdef THEMES + openedx_build_args += --build-arg="THEMES=$(THEMES)" +endif build-openedx: ## Build the Open edX docker image docker build -t regis/openedx:latest -t regis/openedx:hawthorn $(openedx_build_args) openedx/ diff --git a/README.md b/README.md index 98c80bd..3f1b15c 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,7 @@ You will need to download the docker images from [Docker Hub](https://hub.docker make databases make assets -These commands should be run just once. They will create the required databases tables, apply database migrations and generate static assets, such as images, stylesheets and Javascript dependencies. +These commands should be run just once. They will create the required databases tables, apply database migrations and make sure that static assets, such as images, stylesheets and Javascript dependencies, can be served by the nginx container. If migrations are stopped with a `Killed` message, this certainly means the docker containers don't have enough RAM. See the [troubleshooting](#troubleshooting) section. @@ -254,26 +254,59 @@ This will open a shell in the LMS (or CMS) container. You can then run just any The images are built, tagged and uploaded to Docker Hub in one command: make dockerhub - -## Help/Troubleshooting -### How to add custom themes? +## Customising the `openedx` docker image -Comprehensive theming is enabled by default. Just drop your themes in `data/themes` and compile assets: +The LMS and the CMS all run from the `openedx` docker image. The base image is downloaded from [Docker Hub](https://hub.docker.com/r/regis/openedx/) when we run `make update` (or `make all`). But you can also customise and build the image yourself. The base image is built with: + + make build-openedx + +The following sections describe how to modify various aspects of the docker image. After you have built your own image, you can run it as usual: + + make run + +### Custom themes + +Comprehensive theming is enabled by default. Put your themes in `openedx/themes`: + + openedx/themes/ + mycustomtheme1/ + cms/ + ... + lms/ + ... + mycustomtheme2/ + ... + +Then you must rebuild the openedx Docker image and indicate which themes you want to build: + + make build-openedx THEMES=mycustomtheme1,mycustomtheme2,dark-theme + +Make sure the assets can be served by the web server: make assets -Then, follow the [Open edX documentation to enable your themes](https://edx.readthedocs.io/projects/edx-installing-configuring-and-running/en/latest/configuration/changing_appearance/theming/enable_themes.html#apply-a-theme-to-a-site). +Finally, follow the [Open edX documentation to enable your themes](https://edx.readthedocs.io/projects/edx-installing-configuring-and-running/en/latest/configuration/changing_appearance/theming/enable_themes.html#apply-a-theme-to-a-site). -### How to add extra XBlocks to the LMS/CMS? +### Extra xblocks and requirements -Additional requirements can be added to the `openedx/requirements/private.txt` file. Then, the `openedx` docker image must be rebuilt to include the new requirements. For instance: +Additional requirements can be added to the `openedx/requirements/private.txt` file. For instance: echo "git+https://github.com/open-craft/xblock-poll.git" >> openedx/requirements/private.txt - make build-openedx - make run -### How to run Open edX from a forked version of edx-platform? +Then, the `openedx` docker image must be rebuilt: + + make build-openedx + +To install xblocks from a private repository that requires authentication, you must first clone the repository inside the `openedx/requirements` folder on the host: + + git clone git@github.com:me/myprivaterepo.git ./openedx/requirements/myprivaterepo + +Then, declare your extra requirements with the `-e` flag in `openedx/requirements/private.txt` : + + echo "-e ./myprivaterepo" >> openedx/requirements/private.txt + +### Forked version of edx-platform You may want to run your own flavor of edx-platform instead of the [official version](https://github.com/edx/edx-platform/). To do so, you will have to re-build the openedx image with the proper environment variables pointing to your repository and version: @@ -285,13 +318,17 @@ You can then restart the services which will now be running your forked version Note that your release must be a fork of Hawthorn in order to work. Otherwise, you may have important compatibility issues with other services. -### How to run a customized Docker image instead of [regis/openedx](https://hub.docker.com/r/regis/openedx/)? +### Running a different Docker image instead of [regis/openedx](https://hub.docker.com/r/regis/openedx/) -Add the following content to the `.env` file: +This is for people who have an account on [hub.docker.com](https://hub.docker.com) or a private image registry. You can build your image and push it to your repo. Then add the following content to the `.env` file: OPENEDX_DOCKER_IMAGE=myusername/myimage:mytag -Note that the `make build` and `make push` command will no longer work. +Your own image will be used next time you run `make run`. + +Note that the `make build` and `make push` command will no longer work as you expect and that you are responsible for building and pushing the image yourself. + +## Help/Troubleshooting ### "Cannot start service nginx: driver failed programming external connectivity" diff --git a/configurator/templates/openedx/universal/cms/development.py b/configurator/templates/openedx/universal/cms/development.py index d068ff0..a449c91 100644 --- a/configurator/templates/openedx/universal/cms/development.py +++ b/configurator/templates/openedx/universal/cms/development.py @@ -16,6 +16,8 @@ LOGGING['handlers']['tracking'] = { 'formatter': 'standard', } +LOCALE_PATHS.append('/openedx/locale') + # Create folders if necessary import os for folder in [LOG_DIR, MEDIA_ROOT, STATIC_ROOT_BASE]: diff --git a/configurator/templates/openedx/universal/cms/production.py b/configurator/templates/openedx/universal/cms/production.py index 8ab1f29..92f0121 100644 --- a/configurator/templates/openedx/universal/cms/production.py +++ b/configurator/templates/openedx/universal/cms/production.py @@ -25,6 +25,8 @@ DEFAULT_FROM_EMAIL = ENV_TOKENS['CONTACT_EMAIL'] DEFAULT_FEEDBACK_EMAIL = ENV_TOKENS['CONTACT_EMAIL'] SERVER_EMAIL = ENV_TOKENS['CONTACT_EMAIL'] +LOCALE_PATHS.append('/openedx/locale') + # Create folders if necessary import os for folder in [LOG_DIR, MEDIA_ROOT, STATIC_ROOT_BASE]: diff --git a/configurator/templates/openedx/universal/lms/development.py b/configurator/templates/openedx/universal/lms/development.py index e9c0ff2..2ea4429 100644 --- a/configurator/templates/openedx/universal/lms/development.py +++ b/configurator/templates/openedx/universal/lms/development.py @@ -25,6 +25,8 @@ ORA2_FILEUPLOAD_BACKEND = 'filesystem' ORA2_FILEUPLOAD_ROOT = '/openedx/data/ora2' ORA2_FILEUPLOAD_CACHE_NAME = 'ora2-storage' +LOCALE_PATHS.append('/openedx/locale') + # Create folders if necessary import os for folder in [LOG_DIR, MEDIA_ROOT, STATIC_ROOT_BASE, ORA2_FILEUPLOAD_ROOT]: diff --git a/configurator/templates/openedx/universal/lms/production.py b/configurator/templates/openedx/universal/lms/production.py index d979497..4f3a8f8 100644 --- a/configurator/templates/openedx/universal/lms/production.py +++ b/configurator/templates/openedx/universal/lms/production.py @@ -51,6 +51,8 @@ ORA2_FILEUPLOAD_BACKEND = 'filesystem' ORA2_FILEUPLOAD_ROOT = '/openedx/data/ora2' ORA2_FILEUPLOAD_CACHE_NAME = 'ora2-storage' +LOCALE_PATHS.append('/openedx/locale') + # Create folders if necessary import os for folder in [LOG_DIR, MEDIA_ROOT, STATIC_ROOT_BASE, ORA2_FILEUPLOAD_ROOT]: diff --git a/docker-compose-scripts.yml b/docker-compose-scripts.yml new file mode 100644 index 0000000..41a5aaf --- /dev/null +++ b/docker-compose-scripts.yml @@ -0,0 +1,6 @@ +version: "3" +services: + + # Useful for running one-off scripts + openedx: + image: ${OPENEDX_DOCKER_IMAGE:-regis/openedx:hawthorn} diff --git a/docker-compose.yml b/docker-compose.yml index 54cd90b..b5ba132 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -87,7 +87,6 @@ services: volumes: - ./config/openedx:/openedx/config - ./data/lms:/openedx/data - - ./data/themes:/openedx/themes depends_on: - elasticsearch - forum @@ -107,7 +106,6 @@ services: volumes: - ./config/openedx:/openedx/config - ./data/cms:/openedx/data - - ./data/themes:/openedx/themes depends_on: - memcached - mongodb @@ -130,7 +128,6 @@ services: volumes: - ./config/openedx:/openedx/config - ./data/lms:/openedx/data - - ./data/themes:/openedx/themes depends_on: - lms @@ -146,6 +143,5 @@ services: volumes: - ./config/openedx:/openedx/config - ./data/cms:/openedx/data - - ./data/themes:/openedx/themes depends_on: - cms diff --git a/openedx/Dockerfile b/openedx/Dockerfile index 47ec610..b3cd792 100644 --- a/openedx/Dockerfile +++ b/openedx/Dockerfile @@ -16,6 +16,12 @@ RUN apt update && \ # This replaces the "nodeenv" install. RUN apt install -y nodejs-legacy +# Dockerize will be useful to wait for mysql DB availability +ENV DOCKERIZE_VERSION v0.6.1 +RUN curl -L -o /tmp/dockerize.tar.gz https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \ + && tar -C /usr/local/bin -xzvf /tmp/dockerize.tar.gz \ + && rm /tmp/dockerize.tar.gz + # Static assets will reside in /openedx/data, themes in /openedx/themes and # edx-platform will be checked-out in /openedx/ RUN mkdir /openedx /openedx/data /openedx/themes /openedx/edx-platform @@ -26,6 +32,16 @@ ARG EDX_PLATFORM_REPOSITORY=https://github.com/edx/edx-platform.git ARG EDX_PLATFORM_VERSION=open-release/hawthorn.2 RUN git clone $EDX_PLATFORM_REPOSITORY --branch $EDX_PLATFORM_VERSION --depth 1 . +# Download extra locales to /openedx/locale +RUN cd /tmp \ + && curl -L -o openedx-i18n.tar.gz https://github.com/regisb/openedx-i18n/archive/hawthorn.tar.gz \ + && tar xzf /tmp/openedx-i18n.tar.gz \ + && mv openedx-i18n-hawthorn/edx-platform/locale/ /openedx/ \ + && rm -rf openedx-i18n* + +# Copy convenient scripts +COPY ./bin/docker-entrypoint.sh /usr/local/bin/ + # Install python requirements (clone source repos in a separate dir, otherwise # will be overwritten when we mount edx-platform) ENV NO_PYTHON_UNINSTALL 1 @@ -40,49 +56,54 @@ RUN pip uninstall -y ora2 && \ RUN npm install ENV PATH ./node_modules/.bin:${PATH} -# Some parts of assets collection can be run without relying on specific -# settings, nor mounting the staticfiles/ folder -RUN xmodule_assets common/static/xmodule -RUN python -c "import pavelib.assets; pavelib.assets.process_npm_assets()" +# Install private requirements: this is useful for installing custom xblocks. +# In particular, to install xblocks from a private repository, clone the +# respositories to ./requirements on the host and add `-e ./myxblock/` to +# ./requirements/private.txt. +COPY ./requirements/ /openedx/requirements +RUN touch /openedx/requirements/private.txt \ + && pip install --src ../venv/src -r /openedx/requirements/private.txt # Link configuration files to common /openedx/config folder, which should later # be mounted as a volume. Note that this image will not be functional until # config files have been mounted inside the container -RUN mkdir /openedx/config -RUN ln -s /openedx/config/universal/lms/ /openedx/edx-platform/lms/envs/universal \ - && ln -s /openedx/config/universal/cms/ /openedx/edx-platform/cms/envs/universal -RUN ln -s /openedx/config/lms.env.json /openedx/ \ - && ln -s /openedx/config/cms.env.json /openedx/ \ - && ln -s /openedx/config/lms.auth.json /openedx/ \ - && ln -s /openedx/config/cms.auth.json /openedx/ +RUN mkdir -p /openedx/config/universal/lms /openedx/config/universal/cms \ + && ln -s /openedx/config/universal/lms/ /openedx/edx-platform/lms/envs/universal \ + && ln -s /openedx/config/universal/cms/ /openedx/edx-platform/cms/envs/universal \ + && ln -s /openedx/config/lms.env.json /openedx/ \ + && ln -s /openedx/config/cms.env.json /openedx/ \ + && ln -s /openedx/config/lms.auth.json /openedx/ \ + && ln -s /openedx/config/cms.auth.json /openedx/ +COPY settings/lms/*.py /openedx/config/universal/lms/ +COPY settings/cms/*.py /openedx/config/universal/cms/ -# Download extra locales -RUN cd /tmp \ - && curl -L -o openedx-i18n.tar.gz https://github.com/regisb/openedx-i18n/archive/hawthorn.tar.gz \ - && tar xzf /tmp/openedx-i18n.tar.gz \ - && cp -r openedx-i18n-hawthorn/edx-platform/locale/* /openedx/edx-platform/conf/locale/ \ - && rm -rf openedx-i18n* - -# Dockerize will be useful to wait for mysql DB availability -ENV DOCKERIZE_VERSION v0.6.1 -RUN curl -L -o dockerize.tar.gz https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \ - && tar -C /usr/local/bin -xzvf dockerize.tar.gz \ - && rm dockerize.tar.gz - -# Install private requirements as late as possible, so they can be modified frequently -COPY ./requirements/ /tmp/requirements -RUN touch /tmp/requirements/private.txt && pip install --src ../venv/src -r /tmp/requirements/private.txt - -# Copy convenient scripts -COPY ./bin/docker-entrypoint.sh /usr/local/bin/ +# Collect production assets. By default, only assets from the default theme +# will be processed. This makes the docker image lighter and faster to build. +# To compile assets from other themes, build the image with the THEMES +# argument: THEMES=dark-theme,edx.org +# Custom themes added to /openedx/themes can also be compiled. +# Here, we don't run "paver update_assets" which is slow, compiles all themes +# and requires a complex settings file. Instead, we decompose the commands +# and run each one individually to collect the production static assets to +# /openedx/data/staticfiles. +# 1. xmodule_assets: run xmodule.static_content.main, which lists all xblocks and generates webpack manifest in common/static/xmodule +# 2. pavelib.assets.process_npm_assets: copy libraries installed via npm to common/static/common/*/vendor +# 3. webpack: generate webpack-stats.json +# 4. compile_sass: compile each sass file individually with libsass. +# 5. pavelib.assets.collect_assets: run ./manage.py lms/cms collectstatic. This is the only command that requires a settings file. +RUN xmodule_assets common/static/xmodule \ + && python -c "import pavelib.assets; pavelib.assets.process_npm_assets()" \ + && STATIC_ROOT_LMS=/openedx/data/staticfiles STATIC_ROOT_CMS=/openedx/data/staticfiles/studio NODE_ENV=production ./node_modules/.bin/webpack --config=webpack.prod.config.js +ARG THEMES=open-edx +COPY ./themes/ /openedx/themes/ +RUN paver compile_sass --theme-dirs=/openedx/edx-platform/themes,/openedx/themes --themes=$THEMES \ + && python -c "import pavelib.assets; pavelib.assets.collect_assets(['lms', 'cms'], 'universal.assets')" # service variant is "lms" or "cms" ENV SERVICE_VARIANT lms ENV SETTINGS universal.production -ENV STATIC_ROOT_LMS /openedx/data/staticfiles -ENV STATIC_ROOT_CMS /openedx/data/staticfiles/studio -# Entrypoint will fix permissiosn of all files and run commands as openedx +# Entrypoint will fix permissions of all files and run commands as openedx ENTRYPOINT ["docker-entrypoint.sh"] # Run server diff --git a/openedx/settings/cms/__init__.py b/openedx/settings/cms/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/openedx/settings/cms/assets.py b/openedx/settings/cms/assets.py new file mode 100644 index 0000000..e62ffcd --- /dev/null +++ b/openedx/settings/cms/assets.py @@ -0,0 +1,21 @@ +""" +Bare minimum settings for collecting production assets. +""" +from ..common import * +from openedx.core.lib.derived import derive_settings + + +COMPREHENSIVE_THEME_DIRS.append('/openedx/themes') +STATIC_ROOT_BASE = '/openedx/data/staticfiles' +STATIC_ROOT = path(STATIC_ROOT_BASE) / 'studio' +WEBPACK_LOADER['DEFAULT']['STATS_FILE'] = STATIC_ROOT / "webpack-stats.json" + +SECRET_KEY = 'secret' +XQUEUE_INTERFACE = { + 'django_auth': None, + 'url': None, +} +DATABASES = { + "default": {}, +} +derive_settings(__name__) diff --git a/openedx/settings/lms/__init__.py b/openedx/settings/lms/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/openedx/settings/lms/assets.py b/openedx/settings/lms/assets.py new file mode 100644 index 0000000..a195c1a --- /dev/null +++ b/openedx/settings/lms/assets.py @@ -0,0 +1,20 @@ +""" +Bare minimum settings for collecting production assets. +""" +from ..common import * +from openedx.core.lib.derived import derive_settings + +COMPREHENSIVE_THEME_DIRS.append('/openedx/themes') +STATIC_ROOT_BASE = '/openedx/data/staticfiles' +STATIC_ROOT = path(STATIC_ROOT_BASE) +WEBPACK_LOADER['DEFAULT']['STATS_FILE'] = STATIC_ROOT / "webpack-stats.json" + +SECRET_KEY = 'secret' +XQUEUE_INTERFACE = { + 'django_auth': None, + 'url': None, +} +DATABASES = { + "default": {}, +} +derive_settings(__name__) diff --git a/openedx/themes/.gitignore b/openedx/themes/.gitignore new file mode 100644 index 0000000..72e8ffc --- /dev/null +++ b/openedx/themes/.gitignore @@ -0,0 +1 @@ +*