Package static assets inside the openedx image

This allows us to deploy much faster: all we have to do is to copy the
assets from the container to the shared volume.

We also changed the way themes are managed: similarly to static assets,
they are now packaged inside the docker image.

.gitignore

@@ -4,3 +4,4 @@ data-*/
 TODO
 openedx/requirements/private.txt
 .env
+!.gitignore

.travis.yml

@@ -5,7 +5,7 @@ script:
   - make configure SILENT=1 CONFIGURE_OPTS="-e SETTING_ACTIVATE_NOTES=1 -e SETTING_ACTIVATE_XQUEUE=1"
   - make build
   - make databases
-  #- make assets # too time-consuming
+  - make assets
 deploy:
   provider: script
   script: docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" && make push

CHANGELOG.md

@@ -1,5 +1,6 @@
 # Changelog
 
+- 2018-12-07 [Improvement] Bundle theme and production static assets in the openedx docker image
 - 2018-12-02 [Feature] Download extra locales from [openedx-i18n](https://github.com/regisb/openedx-i18n/) to the Open edX Docker image
 - 2018-11-28 [Feature] Easily change openedx docker image
 - 2018-11-28 [Feature] Enable comprehensive theming!

Makefile

@@ -104,19 +104,14 @@ reindex-courses: ## Refresh course index so they can be found in the LMS search
 #assets-cms: ## Collect static assets for the CMS
 #	$(DOCKER_COMPOSE_RUN_OPENEDX) cms -e NO_PREREQ_INSTALL=True cms paver update_assets cms --settings=$(EDX_PLATFORM_SETTINGS)
 
-assets: assets-lms assets-cms ## Generate production-ready static assets
 assets-development: assets-development-lms assets-development-cms ## Generate static assets for local development
 
-assets-lms:
-	$(DOCKER_COMPOSE_RUN_OPENEDX) --no-deps lms bash -c \
-		"NODE_ENV=production ./node_modules/.bin/webpack --config=webpack.prod.config.js \
-		&& ./manage.py lms --settings=$(EDX_PLATFORM_SETTINGS) compile_sass lms \
-		&& python -c \"import pavelib.assets; pavelib.assets.collect_assets(['lms'], '$(EDX_PLATFORM_SETTINGS)')\""
-assets-cms:
-	$(DOCKER_COMPOSE_RUN_OPENEDX) --no-deps cms bash -c \
-		"NODE_ENV=production ./node_modules/.bin/webpack --config=webpack.prod.config.js \
-		&& ./manage.py cms --settings=$(EDX_PLATFORM_SETTINGS) compile_sass studio \
-		&& python -c \"import pavelib.assets; pavelib.assets.collect_assets(['studio'], '$(EDX_PLATFORM_SETTINGS)')\""
+assets: ## Generate production-ready static assets
+	docker-compose -f docker-compose-scripts.yml run --rm \
+		--volume=$(PWD)/data/lms/:/data/lms/ --volume=$(PWD)/data/cms/:/data/cms/ openedx bash -c \
+		"rm -rf /data/lms/staticfiles /data/cms/staticfiles \
+		&& cp -r /openedx/data/staticfiles /data/lms/ \
+		&& cp -r /openedx/data/staticfiles /data/cms/"
 assets-development-lms:
 	$(DOCKER_COMPOSE_RUN_OPENEDX) --no-deps lms bash -c \
 		"xmodule_assets common/static/xmodule \
@@ -178,6 +173,9 @@ endif
 ifdef EDX_PLATFORM_VERSION
 	openedx_build_args += --build-arg="EDX_PLATFORM_VERSION=$(EDX_PLATFORM_VERSION)"
 endif
+ifdef THEMES
+	openedx_build_args += --build-arg="THEMES=$(THEMES)"
+endif
 
 build-openedx: ## Build the Open edX docker image
 	docker build -t regis/openedx:latest -t regis/openedx:hawthorn $(openedx_build_args) openedx/

README.md

@@ -123,7 +123,7 @@ You will need to download the docker images from [Docker Hub](https://hub.docker
     make databases
     make assets
 
-These commands should be run just once. They will create the required databases tables, apply database migrations and generate static assets, such as images, stylesheets and Javascript dependencies.
+These commands should be run just once. They will create the required databases tables, apply database migrations and make sure that static assets, such as images, stylesheets and Javascript dependencies, can be served by the nginx container.
 
 If migrations are stopped with a `Killed` message, this certainly means the docker containers don't have enough RAM. See the [troubleshooting](#troubleshooting) section.
 
@@ -254,26 +254,59 @@ This will open a shell in the LMS (or CMS) container. You can then run just any
 The images are built, tagged and uploaded to Docker Hub in one command:
 
     make dockerhub
-  
-## Help/Troubleshooting
 
-### How to add custom themes?
+## Customising the `openedx` docker image
 
-Comprehensive theming is enabled by default. Just drop your themes in `data/themes` and compile assets:
+The LMS and the CMS all run from the `openedx` docker image. The base image is downloaded from [Docker Hub](https://hub.docker.com/r/regis/openedx/) when we run `make update` (or `make all`). But you can also customise and build the image yourself. The base image is built with:
+
+    make build-openedx
+
+The following sections describe how to modify various aspects of the docker image. After you have built your own image, you can run it as usual:
+
+    make run
+
+### Custom themes
+
+Comprehensive theming is enabled by default. Put your themes in `openedx/themes`:
+
+    openedx/themes/
+        mycustomtheme1/
+            cms/
+                ...
+            lms/
+                ...
+        mycustomtheme2/
+            ...
+
+Then you must rebuild the openedx Docker image and indicate which themes you want to build:
+
+    make build-openedx THEMES=mycustomtheme1,mycustomtheme2,dark-theme
+
+Make sure the assets can be served by the web server:
 
     make assets
 
-Then, follow the [Open edX documentation to enable your themes](https://edx.readthedocs.io/projects/edx-installing-configuring-and-running/en/latest/configuration/changing_appearance/theming/enable_themes.html#apply-a-theme-to-a-site).
+Finally, follow the [Open edX documentation to enable your themes](https://edx.readthedocs.io/projects/edx-installing-configuring-and-running/en/latest/configuration/changing_appearance/theming/enable_themes.html#apply-a-theme-to-a-site).
 
-### How to add extra XBlocks to the LMS/CMS?
+### Extra xblocks and requirements
 
-Additional requirements can be added to the `openedx/requirements/private.txt` file. Then, the `openedx` docker image must be rebuilt to include the new requirements. For instance:
+Additional requirements can be added to the `openedx/requirements/private.txt` file. For instance:
 
     echo "git+https://github.com/open-craft/xblock-poll.git" >> openedx/requirements/private.txt
-    make build-openedx
-    make run
 
-### How to run Open edX from a forked version of edx-platform?
+Then, the `openedx` docker image must be rebuilt:
+
+    make build-openedx
+
+To install xblocks from a private repository that requires authentication, you must first clone the repository inside the `openedx/requirements` folder on the host:
+
+    git clone git@github.com:me/myprivaterepo.git ./openedx/requirements/myprivaterepo
+
+Then, declare your extra requirements with the `-e` flag in `openedx/requirements/private.txt` :
+
+    echo "-e ./myprivaterepo" >> openedx/requirements/private.txt
+
+### Forked version of edx-platform
 
 You may want to run your own flavor of edx-platform instead of the [official version](https://github.com/edx/edx-platform/). To do so, you will have to re-build the openedx image with the proper environment variables pointing to your repository and version:
 
@@ -285,13 +318,17 @@ You can then restart the services which will now be running your forked version
 
 Note that your release must be a fork of Hawthorn in order to work. Otherwise, you may have important compatibility issues with other services.
 
-### How to run a customized Docker image instead of [regis/openedx](https://hub.docker.com/r/regis/openedx/)?
+### Running a different Docker image instead of [regis/openedx](https://hub.docker.com/r/regis/openedx/)
 
-Add the following content to the `.env` file:
+This is for people who have an account on [hub.docker.com](https://hub.docker.com) or a private image registry. You can build your image and push it to your repo. Then add the following content to the `.env` file:
 
     OPENEDX_DOCKER_IMAGE=myusername/myimage:mytag
 
-Note that the `make build` and `make push` command will no longer work.
+Your own image will be used next time you run `make run`.
+
+Note that the `make build` and `make push` command will no longer work as you expect and that you are responsible for building and pushing the image yourself.
+
+## Help/Troubleshooting
 
 ### "Cannot start service nginx: driver failed programming external connectivity"
 

configurator/templates/openedx/universal/cms/development.py

@@ -16,6 +16,8 @@ LOGGING['handlers']['tracking'] = {
     'formatter': 'standard',
 }
 
+LOCALE_PATHS.append('/openedx/locale')
+
 # Create folders if necessary
 import os
 for folder in [LOG_DIR, MEDIA_ROOT, STATIC_ROOT_BASE]:

configurator/templates/openedx/universal/cms/production.py

@@ -25,6 +25,8 @@ DEFAULT_FROM_EMAIL = ENV_TOKENS['CONTACT_EMAIL']
 DEFAULT_FEEDBACK_EMAIL = ENV_TOKENS['CONTACT_EMAIL']
 SERVER_EMAIL = ENV_TOKENS['CONTACT_EMAIL']
 
+LOCALE_PATHS.append('/openedx/locale')
+
 # Create folders if necessary
 import os
 for folder in [LOG_DIR, MEDIA_ROOT, STATIC_ROOT_BASE]:

configurator/templates/openedx/universal/lms/development.py

@@ -25,6 +25,8 @@ ORA2_FILEUPLOAD_BACKEND = 'filesystem'
 ORA2_FILEUPLOAD_ROOT = '/openedx/data/ora2'
 ORA2_FILEUPLOAD_CACHE_NAME = 'ora2-storage'
 
+LOCALE_PATHS.append('/openedx/locale')
+
 # Create folders if necessary
 import os
 for folder in [LOG_DIR, MEDIA_ROOT, STATIC_ROOT_BASE, ORA2_FILEUPLOAD_ROOT]:

configurator/templates/openedx/universal/lms/production.py

@@ -51,6 +51,8 @@ ORA2_FILEUPLOAD_BACKEND = 'filesystem'
 ORA2_FILEUPLOAD_ROOT = '/openedx/data/ora2'
 ORA2_FILEUPLOAD_CACHE_NAME = 'ora2-storage'
 
+LOCALE_PATHS.append('/openedx/locale')
+
 # Create folders if necessary
 import os
 for folder in [LOG_DIR, MEDIA_ROOT, STATIC_ROOT_BASE, ORA2_FILEUPLOAD_ROOT]:

docker-compose-scripts.yml

@@ -0,0 +1,6 @@
+version: "3"
+services:
+
+  # Useful for running one-off scripts
+  openedx:
+    image: ${OPENEDX_DOCKER_IMAGE:-regis/openedx:hawthorn}

docker-compose.yml

@@ -87,7 +87,6 @@ services:
     volumes:
       - ./config/openedx:/openedx/config
       - ./data/lms:/openedx/data
-      - ./data/themes:/openedx/themes
     depends_on:
       - elasticsearch
       - forum
@@ -107,7 +106,6 @@ services:
     volumes:
       - ./config/openedx:/openedx/config
       - ./data/cms:/openedx/data
-      - ./data/themes:/openedx/themes
     depends_on:
       - memcached
       - mongodb
@@ -130,7 +128,6 @@ services:
     volumes:
       - ./config/openedx:/openedx/config
       - ./data/lms:/openedx/data
-      - ./data/themes:/openedx/themes
     depends_on:
       - lms
 
@@ -146,6 +143,5 @@ services:
     volumes:
       - ./config/openedx:/openedx/config
       - ./data/cms:/openedx/data
-      - ./data/themes:/openedx/themes
     depends_on:
       - cms

openedx/Dockerfile

@@ -16,6 +16,12 @@ RUN apt update && \
 # This replaces the "nodeenv" install.
 RUN apt install -y nodejs-legacy
 
+# Dockerize will be useful to wait for mysql DB availability
+ENV DOCKERIZE_VERSION v0.6.1
+RUN curl -L -o /tmp/dockerize.tar.gz https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \
+    && tar -C /usr/local/bin -xzvf /tmp/dockerize.tar.gz \
+    && rm /tmp/dockerize.tar.gz
+
 # Static assets will reside in /openedx/data, themes in /openedx/themes and
 # edx-platform will be checked-out in /openedx/
 RUN mkdir /openedx /openedx/data /openedx/themes /openedx/edx-platform
@@ -26,6 +32,16 @@ ARG EDX_PLATFORM_REPOSITORY=https://github.com/edx/edx-platform.git
 ARG EDX_PLATFORM_VERSION=open-release/hawthorn.2
 RUN git clone $EDX_PLATFORM_REPOSITORY --branch $EDX_PLATFORM_VERSION --depth 1 .
 
+# Download extra locales to /openedx/locale
+RUN cd /tmp \
+    && curl -L -o openedx-i18n.tar.gz https://github.com/regisb/openedx-i18n/archive/hawthorn.tar.gz \
+    && tar xzf /tmp/openedx-i18n.tar.gz \
+    && mv openedx-i18n-hawthorn/edx-platform/locale/ /openedx/ \
+    && rm -rf openedx-i18n*
+
+# Copy convenient scripts
+COPY ./bin/docker-entrypoint.sh /usr/local/bin/
+
 # Install python requirements (clone source repos in a separate dir, otherwise
 # will be overwritten when we mount edx-platform)
 ENV NO_PYTHON_UNINSTALL 1
@@ -40,49 +56,54 @@ RUN pip uninstall -y ora2 && \
 RUN npm install
 ENV PATH ./node_modules/.bin:${PATH}
 
-# Some parts of assets collection can be run without relying on specific
-# settings, nor mounting the staticfiles/ folder
-RUN xmodule_assets common/static/xmodule
-RUN python -c "import pavelib.assets; pavelib.assets.process_npm_assets()"
+# Install private requirements: this is useful for installing custom xblocks.
+# In particular, to install xblocks from a private repository, clone the
+# respositories to ./requirements on the host and add `-e ./myxblock/` to
+# ./requirements/private.txt.
+COPY ./requirements/ /openedx/requirements
+RUN touch /openedx/requirements/private.txt \
+    && pip install --src ../venv/src -r /openedx/requirements/private.txt
 
 # Link configuration files to common /openedx/config folder, which should later
 # be mounted as a volume. Note that this image will not be functional until
 # config files have been mounted inside the container
-RUN mkdir /openedx/config
-RUN ln -s /openedx/config/universal/lms/ /openedx/edx-platform/lms/envs/universal \
-  && ln -s /openedx/config/universal/cms/ /openedx/edx-platform/cms/envs/universal
-RUN ln -s /openedx/config/lms.env.json /openedx/ \
-  && ln -s /openedx/config/cms.env.json /openedx/ \
-  && ln -s /openedx/config/lms.auth.json /openedx/ \
-  && ln -s /openedx/config/cms.auth.json /openedx/
+RUN mkdir -p /openedx/config/universal/lms /openedx/config/universal/cms \
+    && ln -s /openedx/config/universal/lms/ /openedx/edx-platform/lms/envs/universal \
+    && ln -s /openedx/config/universal/cms/ /openedx/edx-platform/cms/envs/universal \
+    && ln -s /openedx/config/lms.env.json /openedx/ \
+    && ln -s /openedx/config/cms.env.json /openedx/ \
+    && ln -s /openedx/config/lms.auth.json /openedx/ \
+    && ln -s /openedx/config/cms.auth.json /openedx/
+COPY settings/lms/*.py /openedx/config/universal/lms/
+COPY settings/cms/*.py /openedx/config/universal/cms/
 
-# Download extra locales
-RUN cd /tmp \
-    && curl -L -o openedx-i18n.tar.gz https://github.com/regisb/openedx-i18n/archive/hawthorn.tar.gz \
-    && tar xzf /tmp/openedx-i18n.tar.gz \
-    && cp -r openedx-i18n-hawthorn/edx-platform/locale/* /openedx/edx-platform/conf/locale/ \
-    && rm -rf openedx-i18n*
-
-# Dockerize will be useful to wait for mysql DB availability
-ENV DOCKERIZE_VERSION v0.6.1
-RUN curl -L -o dockerize.tar.gz https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \
-    && tar -C /usr/local/bin -xzvf dockerize.tar.gz \
-    && rm dockerize.tar.gz
-
-# Install private requirements as late as possible, so they can be modified frequently
-COPY ./requirements/ /tmp/requirements
-RUN touch /tmp/requirements/private.txt && pip install --src ../venv/src -r /tmp/requirements/private.txt
-
-# Copy convenient scripts
-COPY ./bin/docker-entrypoint.sh /usr/local/bin/
+# Collect production assets. By default, only assets from the default theme
+# will be processed. This makes the docker image lighter and faster to build.
+# To compile assets from other themes, build the image with the THEMES
+# argument: THEMES=dark-theme,edx.org
+# Custom themes added to /openedx/themes can also be compiled.
+# Here, we don't run "paver update_assets" which is slow, compiles all themes
+# and requires a complex settings file. Instead, we decompose the commands
+# and run each one individually to collect the production static assets to
+# /openedx/data/staticfiles.
+# 1. xmodule_assets: run xmodule.static_content.main, which lists all xblocks and generates webpack manifest in common/static/xmodule
+# 2. pavelib.assets.process_npm_assets: copy libraries installed via npm to common/static/common/*/vendor
+# 3. webpack: generate webpack-stats.json
+# 4. compile_sass: compile each sass file individually with libsass.
+# 5. pavelib.assets.collect_assets: run ./manage.py lms/cms collectstatic. This is the only command that requires a settings file.
+RUN xmodule_assets common/static/xmodule \
+    && python -c "import pavelib.assets; pavelib.assets.process_npm_assets()" \ 
+    && STATIC_ROOT_LMS=/openedx/data/staticfiles STATIC_ROOT_CMS=/openedx/data/staticfiles/studio NODE_ENV=production ./node_modules/.bin/webpack --config=webpack.prod.config.js
+ARG THEMES=open-edx
+COPY ./themes/ /openedx/themes/
+RUN paver compile_sass --theme-dirs=/openedx/edx-platform/themes,/openedx/themes --themes=$THEMES \
+    && python -c "import pavelib.assets; pavelib.assets.collect_assets(['lms', 'cms'], 'universal.assets')"
 
 # service variant is "lms" or "cms"
 ENV SERVICE_VARIANT lms
 ENV SETTINGS universal.production
-ENV STATIC_ROOT_LMS /openedx/data/staticfiles
-ENV STATIC_ROOT_CMS /openedx/data/staticfiles/studio 
 
-# Entrypoint will fix permissiosn of all files and run commands as openedx
+# Entrypoint will fix permissions of all files and run commands as openedx
 ENTRYPOINT ["docker-entrypoint.sh"]
 
 # Run server

openedx/settings/cms/assets.py

@@ -0,0 +1,21 @@
+"""
+Bare minimum settings for collecting production assets.
+"""
+from ..common import *
+from openedx.core.lib.derived import derive_settings
+
+
+COMPREHENSIVE_THEME_DIRS.append('/openedx/themes')
+STATIC_ROOT_BASE = '/openedx/data/staticfiles'
+STATIC_ROOT = path(STATIC_ROOT_BASE) / 'studio'
+WEBPACK_LOADER['DEFAULT']['STATS_FILE'] = STATIC_ROOT / "webpack-stats.json"
+
+SECRET_KEY = 'secret'
+XQUEUE_INTERFACE = {
+    'django_auth': None,
+    'url': None,
+}
+DATABASES = {
+    "default": {},
+}
+derive_settings(__name__)

openedx/settings/lms/assets.py

@@ -0,0 +1,20 @@
+"""
+Bare minimum settings for collecting production assets.
+"""
+from ..common import *
+from openedx.core.lib.derived import derive_settings
+
+COMPREHENSIVE_THEME_DIRS.append('/openedx/themes')
+STATIC_ROOT_BASE = '/openedx/data/staticfiles'
+STATIC_ROOT = path(STATIC_ROOT_BASE)
+WEBPACK_LOADER['DEFAULT']['STATS_FILE'] = STATIC_ROOT / "webpack-stats.json"
+
+SECRET_KEY = 'secret'
+XQUEUE_INTERFACE = {
+    'django_auth': None,
+    'url': None,
+}
+DATABASES = {
+    "default": {},
+}
+derive_settings(__name__)

openedx/themes/.gitignore

@@ -0,0 +1 @@
+*