Add openedx-assets command

This is great for running a local webserver and watching theme changes
in just two commands.

Makefile

@@ -29,7 +29,8 @@ ifeq ($(ACTIVATE_PORTAINER), 1)
 endif
 
 DOCKER_COMPOSE_RUN = $(DOCKER_COMPOSE) run --rm
-DOCKER_COMPOSE_RUN_OPENEDX = $(DOCKER_COMPOSE_RUN) -e SETTINGS=$(EDX_PLATFORM_SETTINGS)
+DOCKER_COMPOSE_RUN_OPENEDX = $(DOCKER_COMPOSE_RUN) -e SETTINGS=$(EDX_PLATFORM_SETTINGS) \
+							 --volume="$(PWD)/openedx/themes:/openedx/themes"
 ifneq ($(EDX_PLATFORM_PATH),)
 	DOCKER_COMPOSE_RUN_OPENEDX += -e USERID=$(USERID) --volume="$(EDX_PLATFORM_PATH):/openedx/edx-platform"
 endif
@@ -99,34 +100,20 @@ reindex-courses: ## Refresh course index so they can be found in the LMS search
 # webpack collection incorrectly sets the NODE_ENV variable when using custom
 # settings. Thus, each step must be performed separately. This should be fixed
 # in the next edx-platform release thanks to https://github.com/edx/edx-platform/pull/18430/
-#assets-lms: ## Collect static assets for the LMS
-#	$(DOCKER_COMPOSE_RUN_OPENEDX) lms -e NO_PREREQ_INSTALL=True lms paver update_assets lms --settings=$(EDX_PLATFORM_SETTINGS)
-#assets-cms: ## Collect static assets for the CMS
-#	$(DOCKER_COMPOSE_RUN_OPENEDX) cms -e NO_PREREQ_INSTALL=True cms paver update_assets cms --settings=$(EDX_PLATFORM_SETTINGS)
-
-assets-development: assets-development-lms assets-development-cms ## Generate static assets for local development
 
 assets: ## Generate production-ready static assets
 	docker-compose -f docker-compose-scripts.yml run --rm \
-		--volume=$(PWD)/data/lms/:/data/lms/ --volume=$(PWD)/data/cms/:/data/cms/ openedx bash -c \
+		--volume=$(PWD)/data/openedx:/tmp/openedx/ openedx bash -c \
-		"rm -rf /data/lms/staticfiles /data/cms/staticfiles \
+		"rm -rf /tmp/openedx/staticfiles \
-		&& cp -r /openedx/data/staticfiles /data/lms/ \
+		&& cp -r /openedx/staticfiles /tmp/openedx"
-		&& cp -r /openedx/data/staticfiles /data/cms/"
+assets-development: ## Generate static assets for local development
+	$(DOCKER_COMPOSE_RUN_OPENEDX) --no-deps lms bash -c "openedx-assets build --env=dev"
 assets-development-lms:
-	$(DOCKER_COMPOSE_RUN_OPENEDX) --no-deps lms bash -c \
+	$(DOCKER_COMPOSE_RUN_OPENEDX) --no-deps lms bash -c "openedx-assets build --env=dev --system lms"
-		"xmodule_assets common/static/xmodule \
-		&& python -c \"import pavelib.assets; pavelib.assets.process_npm_assets()\" \
-		&& NODE_ENV=development ./node_modules/.bin/webpack --config=webpack.dev.config.js \
-		&& ./manage.py lms --settings=$(EDX_PLATFORM_SETTINGS) compile_sass lms \
-		&& python -c \"import pavelib.assets; pavelib.assets.collect_assets(['lms'], '$(EDX_PLATFORM_SETTINGS)')\""
 assets-development-cms:
-	$(DOCKER_COMPOSE_RUN_OPENEDX) --no-deps cms bash -c \
+	$(DOCKER_COMPOSE_RUN_OPENEDX) --no-deps lms bash -c "openedx-assets build --env=dev --system cms"
-		"xmodule_assets common/static/xmodule \
+watch-themes: ## Watch for changes in your themes and build development assets
-		&& python -c \"import pavelib.assets; pavelib.assets.process_npm_assets()\" \
+	$(DOCKER_COMPOSE_RUN_OPENEDX) --no-deps lms openedx-assets watch-themes --env dev
-		&& NODE_ENV=development ./node_modules/.bin/webpack --config=webpack.dev.config.js \
-		&& ./manage.py cms --settings=$(EDX_PLATFORM_SETTINGS) compile_sass studio \
-		&& python -c \"import pavelib.assets; pavelib.assets.collect_assets(['studio'], '$(EDX_PLATFORM_SETTINGS)')\""
-	
 
 ##################### Information
 
@@ -220,9 +207,13 @@ cms: ## Open a bash shell in the CMS
 lms-python: ## Open a python shell in the LMS
 	$(DOCKER_COMPOSE_RUN_OPENEDX) lms ./manage.py lms shell
 lms-shell: lms-python
+lms-runserver: ## Run a local webserver, useful for debugging
+	$(DOCKER_COMPOSE_RUN_LMS) ./manage.py lms runserver 0.0.0.0:8000
 cms-python: ## Open a python shell in the CMS
 	$(DOCKER_COMPOSE_RUN_OPENEDX) cms ./manage.py cms shell
 cms-shell: cms-python
+cms-runserver: ## Run a local webserver, useful for debugging
+	$(DOCKER_COMPOSE_RUN_CMS) ./manage.py cms runserver 0.0.0.0:8001
 
 restart-openedx: ## Restart lms, cms, and workers
 	docker-compose restart lms lms_worker cms cms_worker

README.md

@@ -201,27 +201,21 @@ Open a python shell in the lms or the cms:
 
 In addition to running Open edX in production, you can use the docker containers for local development. This means you can hack on Open edX without setting up a Virtual Machine. Essentially, this replaces the devstack provided by edX.
 
-(Note: containers are built on the Hawthorn release. If you are working on a different version of Open edX, you will have to rebuild the images with a different `EDX_PLATFORM_VERSION` argument. You may also want to change the `EDX_PLATFORM_REPOSITORY` argument to point to your own fork of edx-platform.)
+To begin with, define development settings:
-
-### Standard devstack
-
-Define development settings (on the host):
 
     export EDX_PLATFORM_SETTINGS=universal.development
 
-Then open an LMS shell:
+### Run a local webserver
+
+    make lms-runserver
+    make cms-runserver
+
+### Open a bash shell
 
     make lms
+    make cms
 
-You can then run a local web server, as usual:
+### Debug edx-platform
-
-    paver update_assets lms --settings=universal.development
-
-Note that assets collection is made more difficult by the fact that development settings are [incorrectly loaded in hawthorn](https://github.com/edx/edx-platform/pull/18430/files). This should be fixed in the next release. Meanwhile, do not run `paver update_assets` while in development mode. Instead, run on the host:
-
-    make assets-development
-
-### Custom devstack
 
 If you have one, you can point to a local version of [edx-platform](https://github.com/edx/edx-platform/) on your host machine:
 
@@ -229,31 +223,38 @@ If you have one, you can point to a local version of [edx-platform](https://gith
 
 Note that you should use an absolute path here, not a relative path (e.g: `/path/to/edx-platform` and not `../edx-platform`).
 
-Point to your settings file:
+All development commands will then automatically mount your local repo. For instance, you can add a `import pdb; pdb.set_trace()` breakpoint anywhere in your code and run:
 
-    export EDX_PLATFORM_SETTINGS=mysettings.py
+    make lms-runserver
 
-In this example, you should have a `mysettings.py` file in `edx-platform/lms/envs` and `edx-platform/cms/envs`. Development settings file for docker are a bit different from stock devstack settings. For valid development settings files, check [`config/openedx/universal/lms/development.py`](https://github.com/regisb/openedx-docker/blob/master/config/openedx/universal/lms/development.py) and [`config/openedx/universal/cms/development.py`](https://github.com/regisb/openedx-docker/blob/master/config/openedx/universal/cms/development.py)
+Note: containers are built on the Hawthorn release. If you are working on a different version of Open edX, you will have to rebuild the images with the right `EDX_PLATFORM_VERSION` argument. You may also want to change the `EDX_PLATFORM_REPOSITORY` argument to point to your own fork of edx-platform.
 
-You are ready to go! Run:
+With a customised edx-platform repo, you must be careful to have settings that are compatible with the docker environment. You are encouraged to copy the `universal.development` settings files to our own repo:
 
-    make lms
+    cp -r config/openedx/universal/lms/ /path/to/edx-platform/lms/envs/universal
+    cp -r config/openedx/universal/cms/ /path/to/edx-platform/cms/envs/universal
 
-Or:
+You can then run your platform with the `universal.development` settings.
 
-    make cms
+### Develop customised themes
 
-This will open a shell in the LMS (or CMS) container. You can then run just any command you are used to. For example, install node requirements, collect assets and run a local server:
+Run a local webserver:
 
-    npm install
+    make lms-runserver
-    paver update_assets lms --settings=mysettings
-    ./manage.py lms runserver 0.0.0.0:8000
 
-## Maintainers
+Watch the themes folders for changes:
 
-The images are built, tagged and uploaded to Docker Hub in one command:
+    make watch-themes
 
-    make dockerhub
+Make changes to `openedx/themes/yourtheme`: the theme assets should be automatically recompiled and visible at http://localhost:8000.
+
+### Assets management
+
+Assets building and collecting is made more difficult by the fact that development settings are [incorrectly loaded in Hawthorn](https://github.com/edx/edx-platform/pull/18430/files). This should be fixed in the next Open edX release. Meanwhile, do not run `paver update_assets` while in development mode. When working locally on a theme, build assets by running in the container:
+
+    openedx-assets build
+
+This command will take quite some time to run. You can speed up this process by running only part of the full build. Run `openedx-assets -h` for more information.
 
 ## Customising the `openedx` docker image
 
@@ -328,6 +329,12 @@ Your own image will be used next time you run `make run`.
 
 Note that the `make build` and `make push` command will no longer work as you expect and that you are responsible for building and pushing the image yourself.
 
+## Maintainers
+
+The images are built, tagged and uploaded to Docker Hub in one command:
+
+    make dockerhub
+
 ## Help/Troubleshooting
 
 ### "Cannot start service nginx: driver failed programming external connectivity"

configurator/templates/nginx/cms.conf

@@ -46,7 +46,7 @@ server {
   }
 
   location ~ ^/static/(?P<file>.*) {
-    root /openedx/data/cms;
+    root /var/www/openedx;
     try_files /staticfiles/$file /course_static/$file =404;
 
     # return a 403 for static files that shouldn't be

configurator/templates/nginx/lms.conf

@@ -68,7 +68,7 @@ server {
   }
 
   location ~ ^/static/(?P<file>.*) {
-    root /openedx/data/lms;
+    root /var/www/openedx;
     try_files /staticfiles/$file /course_static/$file =404;
 
     # return a 403 for static files that shouldn't be

configurator/templates/openedx/cms.env.json

@@ -20,7 +20,7 @@
   "CELERY_BROKER_TRANSPORT": "amqp",
   "COMPREHENSIVE_THEME_DIRS": ["/openedx/themes"],
   "MEDIA_ROOT": "/openedx/data/uploads/",
-  "STATIC_ROOT_BASE": "/openedx/data/staticfiles",
+  "STATIC_ROOT_BASE": "/openedx/staticfiles",
   "ELASTIC_SEARCH_CONFIG": [{
     "host": "elasticsearch",
     "port": 9200

configurator/templates/openedx/lms.env.json

@@ -24,7 +24,7 @@
   "COMMENTS_SERVICE_KEY": "forumapikey",
   "COMPREHENSIVE_THEME_DIRS": ["/openedx/themes"],
   "MEDIA_ROOT": "/openedx/data/uploads/",
-  "STATIC_ROOT_BASE": "/openedx/data/staticfiles",
+  "STATIC_ROOT_BASE": "/openedx/staticfiles",
   "ELASTIC_SEARCH_CONFIG": [{
     "host": "elasticsearch",
     "port": 9200

data/.gitignore

@@ -7,6 +7,7 @@ lms_worker/
 elasticsearch/
 mysql/
 mongodb/
+openedx/
 portainer/
 rabbitmq/
 xqueue/

docker-compose.yml

@@ -45,8 +45,7 @@ services:
       - "${NGINX_HTTPS_PORT:-443}:443"
     volumes:
       - ./config/nginx:/etc/nginx/conf.d/
-      - ./data/lms:/openedx/data/lms:ro
+      - ./data/openedx:/var/www/openedx:ro
-      - ./data/cms:/openedx/data/cms:ro
       - ./data/letsencrypt:/etc/letsencrypt/:ro
 
   rabbitmq:

openedx/Dockerfile

@@ -63,7 +63,7 @@ ENV PATH ./node_modules/.bin:${PATH}
 # ./requirements/private.txt.
 COPY ./requirements/ /openedx/requirements
 RUN touch /openedx/requirements/private.txt \
-    && pip install --src ../venv/src -r /openedx/requirements/private.txt
+    && pip install -r /openedx/requirements/private.txt
 
 # Link configuration files to common /openedx/config folder, which should later
 # be mounted as a volume. Note that this image will not be functional until
@@ -84,18 +84,15 @@ COPY settings/cms/*.py /openedx/config/universal/cms/
 # Here, we don't run "paver update_assets" which is slow, compiles all themes
 # and requires a complex settings file. Instead, we decompose the commands
 # and run each one individually to collect the production static assets to
-# /openedx/data/staticfiles.
+# /openedx/staticfiles.
-# 1. xmodule_assets: run xmodule.static_content.main, which lists all xblocks and generates webpack manifest in common/static/xmodule
+COPY ./bin/openedx-assets /usr/local/bin/
-# 2. pavelib.assets.process_npm_assets: copy libraries installed via npm to common/static/common/*/vendor
+RUN openedx-assets xmodule \
-# 3. webpack: generate webpack-stats.json
+    && openedx-assets npm \
-# 4. compile_sass: compile each sass file individually with libsass.
+    && openedx-assets webpack --env=prod \
-# 5. pavelib.assets.collect_assets: run ./manage.py lms/cms collectstatic. This is the only command that requires a settings file.
+    && openedx-assets common
-RUN xmodule_assets common/static/xmodule \
-    && python -c "import pavelib.assets; pavelib.assets.process_npm_assets()" \ 
-    && STATIC_ROOT_LMS=/openedx/data/staticfiles STATIC_ROOT_CMS=/openedx/data/staticfiles/studio NODE_ENV=production ./node_modules/.bin/webpack --config=webpack.prod.config.js
 COPY ./themes/ /openedx/themes/
-RUN paver compile_sass --theme-dirs=/openedx/edx-platform/themes,/openedx/themes --themes=open-edx,$(ls /openedx/themes | paste -s -d, -) \
+RUN openedx-assets themes \
-    && python -c "import pavelib.assets; pavelib.assets.collect_assets(['lms', 'cms'], 'universal.assets')"
+    && openedx-assets collect --settings=universal.assets
 
 # service variant is "lms" or "cms"
 ENV SERVICE_VARIANT lms

openedx/bin/openedx-assets

@@ -0,0 +1,164 @@
+#! /usr/bin/env python
+from __future__ import print_function
+import argparse
+import os
+import subprocess
+import sys
+import traceback
+
+from path import Path
+
+from pavelib import assets
+from xmodule import static_content as xmodule_static_content
+
+
+DEFAULT_STATIC_ROOT = '/openedx/staticfiles'
+DEFAULT_THEMES_DIR = '/openedx/themes'
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Various assets processing/building/collection utility for Open edX"
+    )
+    subparsers = parser.add_subparsers()
+
+    npm = subparsers.add_parser('npm', help="Copy static assets from node_modules")
+    npm.set_defaults(func=run_npm)
+
+    build = subparsers.add_parser('build', help="Build all assets")
+    build.add_argument('-e', '--env', choices=['prod', 'dev'], default='prod')
+    build.add_argument('--theme-dirs', nargs='+', default=[DEFAULT_THEMES_DIR])
+    build.add_argument('--themes', nargs='+', default=['all'])
+    build.add_argument('-r', '--static-root', default=DEFAULT_STATIC_ROOT)
+    build.add_argument('--systems', nargs='+', default=['lms', 'cms'])
+    build.set_defaults(func=run_build)
+
+    xmodule = subparsers.add_parser('xmodule', help="Process assets from xmodule")
+    xmodule.set_defaults(func=run_xmodule)
+
+    webpack = subparsers.add_parser('webpack', help="Run webpack")
+    webpack.add_argument('-r', '--static-root', default=DEFAULT_STATIC_ROOT)
+    webpack.add_argument('-e', '--env', choices=['prod', 'dev'], default='prod')
+    webpack.set_defaults(func=run_webpack)
+
+    common = subparsers.add_parser('common', help="Compile static assets for common theme")
+    common.add_argument('--systems', nargs='+', default=['lms', 'cms'])
+    common.set_defaults(func=run_common)
+
+    themes = subparsers.add_parser('themes', help="Compile static assets for custom themes")
+    themes.add_argument('--theme-dirs', nargs='+', default=[DEFAULT_THEMES_DIR])
+    themes.add_argument('--themes', nargs='+', default=['all'])
+    themes.add_argument('--systems', nargs='+', default=['lms', 'cms'])
+    themes.set_defaults(func=run_themes)
+
+    collect = subparsers.add_parser('collect', help="Collect static assets to be served by webserver")
+    collect.add_argument('-s', '--settings', default=os.environ.get('SETTINGS'), help="Django settings module")
+    collect.add_argument('--systems', nargs='+', choices=['lms', 'cms'], default=['lms', 'cms'], help="Limit collection to lms or cms")
+    collect.set_defaults(func=run_collect)
+
+    watch_themes = subparsers.add_parser('watch-themes', help="Watch theme assets for changes and recompile on-the-fly")
+    watch_themes.add_argument('-e', '--env', choices=['prod', 'dev'], default='prod', help="Webpack target to run")
+    watch_themes.add_argument('--theme-dirs', default=[DEFAULT_THEMES_DIR])
+    watch_themes.set_defaults(func=run_watch_themes)
+
+    args = parser.parse_args()
+    args.func(args)
+
+def run_build(args):
+    run_xmodule(args)
+    run_npm(args)
+    run_webpack(args)
+    run_common(args)
+    run_themes(args)
+
+def run_xmodule(args):
+    sys.argv[1:] = ['common/static/xmodule']
+    xmodule_static_content.main()
+
+def run_npm(args):
+    assets.process_npm_assets()
+
+def run_webpack(args):
+    os.environ['STATIC_ROOT_LMS'] = args.static_root
+    os.environ['STATIC_ROOT_CMS'] = args.static_root
+    os.environ['NODE_ENV'] = {
+        'prod': 'production',
+        'dev': 'development',
+    }[args.env]
+    subprocess.call([
+        'webpack', '--config=webpack.{env}.config.js'.format(env=args.env)
+    ])
+
+def run_common(args):
+    for system in args.systems:
+        print("Compiling {} sass assets from common theme...".format(system))
+        assets._compile_sass(system, None, False, False, [])
+
+def run_themes(args):
+    for theme_dir in args.theme_dirs:
+        local_themes = list_subdirectories(theme_dir) if 'all' in args.themes else args.themes
+        for theme in local_themes:
+            theme_path = os.path.join(theme_dir, theme)
+            if os.path.exists(theme_path):
+                for system in args.systems:
+                    print("Compiling {} sass assets from theme {}...".format(system, theme_path))
+                    assets._compile_sass(system, Path(theme_path), False, False, [])
+
+def run_collect(args):
+    assets.collect_assets(args.systems, args.settings)
+
+def run_watch_themes(args):
+    """
+    Watch static assets for changes and re-compile those changes when
+    necessary. This piece of code is heavily inspired from the
+    edx-platform/pavelib/assets.py:watch_assets function, which could not be
+    used directly because it does not properly read the platform settings
+    environment variable.
+
+    Note that this function will only work for watching assets in development
+    mode. In production, watching changes does not make much sense anyway.
+    """
+    observer = assets.Observer()
+    for theme_dir in args.theme_dirs:
+        print("Watching changes in {}...".format(theme_dir))
+        ThemeWatcher(theme_dir).register(observer)
+    observer.start()
+    try:
+        while True:
+            observer.join(2)
+    except KeyboardInterrupt:
+        observer.stop()
+
+def list_subdirectories(path):
+    return [subpath for subpath in os.listdir(path) if os.path.isdir(os.path.join(path, subpath))]
+
+
+class ThemeWatcher(assets.SassWatcher):
+
+    def __init__(self, theme_dir):
+        super(ThemeWatcher, self).__init__()
+        self.theme_dir = theme_dir
+
+    #pylint: disable=arguments-differ
+    def register(self, observer):
+        return super(ThemeWatcher, self).register(observer, [self.theme_dir])
+
+    @assets.debounce()
+    def on_any_event(self, event):
+        components = os.path.relpath(event.src_path, self.theme_dir).split('/')
+        try:
+            theme = components[0]
+            system = components[1]
+        except IndexError:
+            return
+        try:
+            print("Detected change:", event.src_path)
+            print("\tRecompiling {} theme for {}".format(theme, system))
+            assets._compile_sass(system, Path(self.theme_dir) / theme, False, False, [])
+            print("\tDone recompiling {} theme for {}".format(theme, system))
+        except Exception: # pylint: disable=broad-except
+            traceback.print_exc()
+
+
+if __name__ == '__main__':
+    main()

openedx/settings/cms/assets.py

@@ -6,7 +6,7 @@ from openedx.core.lib.derived import derive_settings
 
 
 COMPREHENSIVE_THEME_DIRS.append('/openedx/themes')
-STATIC_ROOT_BASE = '/openedx/data/staticfiles'
+STATIC_ROOT_BASE = '/openedx/staticfiles'
 STATIC_ROOT = path(STATIC_ROOT_BASE) / 'studio'
 WEBPACK_LOADER['DEFAULT']['STATS_FILE'] = STATIC_ROOT / "webpack-stats.json"
 

openedx/settings/lms/assets.py

@@ -5,7 +5,7 @@ from ..common import *
 from openedx.core.lib.derived import derive_settings
 
 COMPREHENSIVE_THEME_DIRS.append('/openedx/themes')
-STATIC_ROOT_BASE = '/openedx/data/staticfiles'
+STATIC_ROOT_BASE = '/openedx/staticfiles'
 STATIC_ROOT = path(STATIC_ROOT_BASE)
 WEBPACK_LOADER['DEFAULT']['STATS_FILE'] = STATIC_ROOT / "webpack-stats.json"