diff --git a/Makefile b/Makefile index a632c59..78fe08a 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ endif ifeq ($(ACTIVATE_HTTPS), 1) post_configure_targets += https-certificate endif -extra_migrate_targets = +extra_migrate_targets = ifeq ($(ACTIVATE_XQUEUE), 1) extra_migrate_targets += migrate-xqueue DOCKER_COMPOSE += -f docker-compose-xqueue.yml @@ -29,7 +29,8 @@ ifeq ($(ACTIVATE_PORTAINER), 1) endif DOCKER_COMPOSE_RUN = $(DOCKER_COMPOSE) run --rm -DOCKER_COMPOSE_RUN_OPENEDX = $(DOCKER_COMPOSE_RUN) -e SETTINGS=$(EDX_PLATFORM_SETTINGS) +DOCKER_COMPOSE_RUN_OPENEDX = $(DOCKER_COMPOSE_RUN) -e SETTINGS=$(EDX_PLATFORM_SETTINGS) \ + --volume="$(PWD)/openedx/themes:/openedx/themes" ifneq ($(EDX_PLATFORM_PATH),) DOCKER_COMPOSE_RUN_OPENEDX += -e USERID=$(USERID) --volume="$(EDX_PLATFORM_PATH):/openedx/edx-platform" endif @@ -99,34 +100,20 @@ reindex-courses: ## Refresh course index so they can be found in the LMS search # webpack collection incorrectly sets the NODE_ENV variable when using custom # settings. Thus, each step must be performed separately. This should be fixed # in the next edx-platform release thanks to https://github.com/edx/edx-platform/pull/18430/ -#assets-lms: ## Collect static assets for the LMS -# $(DOCKER_COMPOSE_RUN_OPENEDX) lms -e NO_PREREQ_INSTALL=True lms paver update_assets lms --settings=$(EDX_PLATFORM_SETTINGS) -#assets-cms: ## Collect static assets for the CMS -# $(DOCKER_COMPOSE_RUN_OPENEDX) cms -e NO_PREREQ_INSTALL=True cms paver update_assets cms --settings=$(EDX_PLATFORM_SETTINGS) - -assets-development: assets-development-lms assets-development-cms ## Generate static assets for local development assets: ## Generate production-ready static assets docker-compose -f docker-compose-scripts.yml run --rm \ - --volume=$(PWD)/data/lms/:/data/lms/ --volume=$(PWD)/data/cms/:/data/cms/ openedx bash -c \ - "rm -rf /data/lms/staticfiles /data/cms/staticfiles \ - && cp -r /openedx/data/staticfiles /data/lms/ \ - && cp -r /openedx/data/staticfiles /data/cms/" + --volume=$(PWD)/data/openedx:/tmp/openedx/ openedx bash -c \ + "rm -rf /tmp/openedx/staticfiles \ + && cp -r /openedx/staticfiles /tmp/openedx" +assets-development: ## Generate static assets for local development + $(DOCKER_COMPOSE_RUN_OPENEDX) --no-deps lms bash -c "openedx-assets build --env=dev" assets-development-lms: - $(DOCKER_COMPOSE_RUN_OPENEDX) --no-deps lms bash -c \ - "xmodule_assets common/static/xmodule \ - && python -c \"import pavelib.assets; pavelib.assets.process_npm_assets()\" \ - && NODE_ENV=development ./node_modules/.bin/webpack --config=webpack.dev.config.js \ - && ./manage.py lms --settings=$(EDX_PLATFORM_SETTINGS) compile_sass lms \ - && python -c \"import pavelib.assets; pavelib.assets.collect_assets(['lms'], '$(EDX_PLATFORM_SETTINGS)')\"" + $(DOCKER_COMPOSE_RUN_OPENEDX) --no-deps lms bash -c "openedx-assets build --env=dev --system lms" assets-development-cms: - $(DOCKER_COMPOSE_RUN_OPENEDX) --no-deps cms bash -c \ - "xmodule_assets common/static/xmodule \ - && python -c \"import pavelib.assets; pavelib.assets.process_npm_assets()\" \ - && NODE_ENV=development ./node_modules/.bin/webpack --config=webpack.dev.config.js \ - && ./manage.py cms --settings=$(EDX_PLATFORM_SETTINGS) compile_sass studio \ - && python -c \"import pavelib.assets; pavelib.assets.collect_assets(['studio'], '$(EDX_PLATFORM_SETTINGS)')\"" - + $(DOCKER_COMPOSE_RUN_OPENEDX) --no-deps lms bash -c "openedx-assets build --env=dev --system cms" +watch-themes: ## Watch for changes in your themes and build development assets + $(DOCKER_COMPOSE_RUN_OPENEDX) --no-deps lms openedx-assets watch-themes --env dev ##################### Information @@ -184,7 +171,7 @@ build-notes: ## Build the Notes docker image docker build -t regis/openedx-notes:latest -t regis/openedx-notes:hawthorn notes/ build-xqueue: ## Build the Xqueue docker image docker build -t regis/openedx-xqueue:latest -t regis/openedx-xqueue:hawthorn xqueue/ -build-android: ## Build the docker image for Android +build-android: ## Build the docker image for Android docker build -t regis/openedx-android:latest android/ ################### Pushing images to docker hub @@ -220,9 +207,13 @@ cms: ## Open a bash shell in the CMS lms-python: ## Open a python shell in the LMS $(DOCKER_COMPOSE_RUN_OPENEDX) lms ./manage.py lms shell lms-shell: lms-python +lms-runserver: ## Run a local webserver, useful for debugging + $(DOCKER_COMPOSE_RUN_LMS) ./manage.py lms runserver 0.0.0.0:8000 cms-python: ## Open a python shell in the CMS $(DOCKER_COMPOSE_RUN_OPENEDX) cms ./manage.py cms shell cms-shell: cms-python +cms-runserver: ## Run a local webserver, useful for debugging + $(DOCKER_COMPOSE_RUN_CMS) ./manage.py cms runserver 0.0.0.0:8001 restart-openedx: ## Restart lms, cms, and workers docker-compose restart lms lms_worker cms cms_worker diff --git a/README.md b/README.md index f417224..44f8da8 100644 --- a/README.md +++ b/README.md @@ -201,27 +201,21 @@ Open a python shell in the lms or the cms: In addition to running Open edX in production, you can use the docker containers for local development. This means you can hack on Open edX without setting up a Virtual Machine. Essentially, this replaces the devstack provided by edX. -(Note: containers are built on the Hawthorn release. If you are working on a different version of Open edX, you will have to rebuild the images with a different `EDX_PLATFORM_VERSION` argument. You may also want to change the `EDX_PLATFORM_REPOSITORY` argument to point to your own fork of edx-platform.) - -### Standard devstack - -Define development settings (on the host): +To begin with, define development settings: export EDX_PLATFORM_SETTINGS=universal.development -Then open an LMS shell: +### Run a local webserver + + make lms-runserver + make cms-runserver + +### Open a bash shell make lms + make cms -You can then run a local web server, as usual: - - paver update_assets lms --settings=universal.development - -Note that assets collection is made more difficult by the fact that development settings are [incorrectly loaded in hawthorn](https://github.com/edx/edx-platform/pull/18430/files). This should be fixed in the next release. Meanwhile, do not run `paver update_assets` while in development mode. Instead, run on the host: - - make assets-development - -### Custom devstack +### Debug edx-platform If you have one, you can point to a local version of [edx-platform](https://github.com/edx/edx-platform/) on your host machine: @@ -229,31 +223,38 @@ If you have one, you can point to a local version of [edx-platform](https://gith Note that you should use an absolute path here, not a relative path (e.g: `/path/to/edx-platform` and not `../edx-platform`). -Point to your settings file: +All development commands will then automatically mount your local repo. For instance, you can add a `import pdb; pdb.set_trace()` breakpoint anywhere in your code and run: - export EDX_PLATFORM_SETTINGS=mysettings.py + make lms-runserver -In this example, you should have a `mysettings.py` file in `edx-platform/lms/envs` and `edx-platform/cms/envs`. Development settings file for docker are a bit different from stock devstack settings. For valid development settings files, check [`config/openedx/universal/lms/development.py`](https://github.com/regisb/openedx-docker/blob/master/config/openedx/universal/lms/development.py) and [`config/openedx/universal/cms/development.py`](https://github.com/regisb/openedx-docker/blob/master/config/openedx/universal/cms/development.py) +Note: containers are built on the Hawthorn release. If you are working on a different version of Open edX, you will have to rebuild the images with the right `EDX_PLATFORM_VERSION` argument. You may also want to change the `EDX_PLATFORM_REPOSITORY` argument to point to your own fork of edx-platform. -You are ready to go! Run: +With a customised edx-platform repo, you must be careful to have settings that are compatible with the docker environment. You are encouraged to copy the `universal.development` settings files to our own repo: - make lms + cp -r config/openedx/universal/lms/ /path/to/edx-platform/lms/envs/universal + cp -r config/openedx/universal/cms/ /path/to/edx-platform/cms/envs/universal -Or: +You can then run your platform with the `universal.development` settings. - make cms +### Develop customised themes -This will open a shell in the LMS (or CMS) container. You can then run just any command you are used to. For example, install node requirements, collect assets and run a local server: +Run a local webserver: - npm install - paver update_assets lms --settings=mysettings - ./manage.py lms runserver 0.0.0.0:8000 + make lms-runserver -## Maintainers +Watch the themes folders for changes: -The images are built, tagged and uploaded to Docker Hub in one command: + make watch-themes - make dockerhub +Make changes to `openedx/themes/yourtheme`: the theme assets should be automatically recompiled and visible at http://localhost:8000. + +### Assets management + +Assets building and collecting is made more difficult by the fact that development settings are [incorrectly loaded in Hawthorn](https://github.com/edx/edx-platform/pull/18430/files). This should be fixed in the next Open edX release. Meanwhile, do not run `paver update_assets` while in development mode. When working locally on a theme, build assets by running in the container: + + openedx-assets build + +This command will take quite some time to run. You can speed up this process by running only part of the full build. Run `openedx-assets -h` for more information. ## Customising the `openedx` docker image @@ -328,6 +329,12 @@ Your own image will be used next time you run `make run`. Note that the `make build` and `make push` command will no longer work as you expect and that you are responsible for building and pushing the image yourself. +## Maintainers + +The images are built, tagged and uploaded to Docker Hub in one command: + + make dockerhub + ## Help/Troubleshooting ### "Cannot start service nginx: driver failed programming external connectivity" diff --git a/configurator/templates/nginx/cms.conf b/configurator/templates/nginx/cms.conf index ce94b9d..99d6b61 100644 --- a/configurator/templates/nginx/cms.conf +++ b/configurator/templates/nginx/cms.conf @@ -46,7 +46,7 @@ server { } location ~ ^/static/(?P.*) { - root /openedx/data/cms; + root /var/www/openedx; try_files /staticfiles/$file /course_static/$file =404; # return a 403 for static files that shouldn't be diff --git a/configurator/templates/nginx/lms.conf b/configurator/templates/nginx/lms.conf index c202a8c..65bb94b 100644 --- a/configurator/templates/nginx/lms.conf +++ b/configurator/templates/nginx/lms.conf @@ -68,7 +68,7 @@ server { } location ~ ^/static/(?P.*) { - root /openedx/data/lms; + root /var/www/openedx; try_files /staticfiles/$file /course_static/$file =404; # return a 403 for static files that shouldn't be diff --git a/configurator/templates/openedx/cms.env.json b/configurator/templates/openedx/cms.env.json index 1084eec..1b558d2 100644 --- a/configurator/templates/openedx/cms.env.json +++ b/configurator/templates/openedx/cms.env.json @@ -20,7 +20,7 @@ "CELERY_BROKER_TRANSPORT": "amqp", "COMPREHENSIVE_THEME_DIRS": ["/openedx/themes"], "MEDIA_ROOT": "/openedx/data/uploads/", - "STATIC_ROOT_BASE": "/openedx/data/staticfiles", + "STATIC_ROOT_BASE": "/openedx/staticfiles", "ELASTIC_SEARCH_CONFIG": [{ "host": "elasticsearch", "port": 9200 diff --git a/configurator/templates/openedx/lms.env.json b/configurator/templates/openedx/lms.env.json index 77c0be2..6efd3b5 100644 --- a/configurator/templates/openedx/lms.env.json +++ b/configurator/templates/openedx/lms.env.json @@ -24,7 +24,7 @@ "COMMENTS_SERVICE_KEY": "forumapikey", "COMPREHENSIVE_THEME_DIRS": ["/openedx/themes"], "MEDIA_ROOT": "/openedx/data/uploads/", - "STATIC_ROOT_BASE": "/openedx/data/staticfiles", + "STATIC_ROOT_BASE": "/openedx/staticfiles", "ELASTIC_SEARCH_CONFIG": [{ "host": "elasticsearch", "port": 9200 diff --git a/data/.gitignore b/data/.gitignore index f7ff79c..2cb709a 100644 --- a/data/.gitignore +++ b/data/.gitignore @@ -7,6 +7,7 @@ lms_worker/ elasticsearch/ mysql/ mongodb/ +openedx/ portainer/ rabbitmq/ xqueue/ diff --git a/docker-compose.yml b/docker-compose.yml index b5ba132..df61af0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -45,8 +45,7 @@ services: - "${NGINX_HTTPS_PORT:-443}:443" volumes: - ./config/nginx:/etc/nginx/conf.d/ - - ./data/lms:/openedx/data/lms:ro - - ./data/cms:/openedx/data/cms:ro + - ./data/openedx:/var/www/openedx:ro - ./data/letsencrypt:/etc/letsencrypt/:ro rabbitmq: diff --git a/openedx/Dockerfile b/openedx/Dockerfile index 84021bf..f0247f7 100644 --- a/openedx/Dockerfile +++ b/openedx/Dockerfile @@ -63,7 +63,7 @@ ENV PATH ./node_modules/.bin:${PATH} # ./requirements/private.txt. COPY ./requirements/ /openedx/requirements RUN touch /openedx/requirements/private.txt \ - && pip install --src ../venv/src -r /openedx/requirements/private.txt + && pip install -r /openedx/requirements/private.txt # Link configuration files to common /openedx/config folder, which should later # be mounted as a volume. Note that this image will not be functional until @@ -84,18 +84,15 @@ COPY settings/cms/*.py /openedx/config/universal/cms/ # Here, we don't run "paver update_assets" which is slow, compiles all themes # and requires a complex settings file. Instead, we decompose the commands # and run each one individually to collect the production static assets to -# /openedx/data/staticfiles. -# 1. xmodule_assets: run xmodule.static_content.main, which lists all xblocks and generates webpack manifest in common/static/xmodule -# 2. pavelib.assets.process_npm_assets: copy libraries installed via npm to common/static/common/*/vendor -# 3. webpack: generate webpack-stats.json -# 4. compile_sass: compile each sass file individually with libsass. -# 5. pavelib.assets.collect_assets: run ./manage.py lms/cms collectstatic. This is the only command that requires a settings file. -RUN xmodule_assets common/static/xmodule \ - && python -c "import pavelib.assets; pavelib.assets.process_npm_assets()" \ - && STATIC_ROOT_LMS=/openedx/data/staticfiles STATIC_ROOT_CMS=/openedx/data/staticfiles/studio NODE_ENV=production ./node_modules/.bin/webpack --config=webpack.prod.config.js +# /openedx/staticfiles. +COPY ./bin/openedx-assets /usr/local/bin/ +RUN openedx-assets xmodule \ + && openedx-assets npm \ + && openedx-assets webpack --env=prod \ + && openedx-assets common COPY ./themes/ /openedx/themes/ -RUN paver compile_sass --theme-dirs=/openedx/edx-platform/themes,/openedx/themes --themes=open-edx,$(ls /openedx/themes | paste -s -d, -) \ - && python -c "import pavelib.assets; pavelib.assets.collect_assets(['lms', 'cms'], 'universal.assets')" +RUN openedx-assets themes \ + && openedx-assets collect --settings=universal.assets # service variant is "lms" or "cms" ENV SERVICE_VARIANT lms diff --git a/openedx/bin/openedx-assets b/openedx/bin/openedx-assets new file mode 100755 index 0000000..4751add --- /dev/null +++ b/openedx/bin/openedx-assets @@ -0,0 +1,164 @@ +#! /usr/bin/env python +from __future__ import print_function +import argparse +import os +import subprocess +import sys +import traceback + +from path import Path + +from pavelib import assets +from xmodule import static_content as xmodule_static_content + + +DEFAULT_STATIC_ROOT = '/openedx/staticfiles' +DEFAULT_THEMES_DIR = '/openedx/themes' + + +def main(): + parser = argparse.ArgumentParser( + description="Various assets processing/building/collection utility for Open edX" + ) + subparsers = parser.add_subparsers() + + npm = subparsers.add_parser('npm', help="Copy static assets from node_modules") + npm.set_defaults(func=run_npm) + + build = subparsers.add_parser('build', help="Build all assets") + build.add_argument('-e', '--env', choices=['prod', 'dev'], default='prod') + build.add_argument('--theme-dirs', nargs='+', default=[DEFAULT_THEMES_DIR]) + build.add_argument('--themes', nargs='+', default=['all']) + build.add_argument('-r', '--static-root', default=DEFAULT_STATIC_ROOT) + build.add_argument('--systems', nargs='+', default=['lms', 'cms']) + build.set_defaults(func=run_build) + + xmodule = subparsers.add_parser('xmodule', help="Process assets from xmodule") + xmodule.set_defaults(func=run_xmodule) + + webpack = subparsers.add_parser('webpack', help="Run webpack") + webpack.add_argument('-r', '--static-root', default=DEFAULT_STATIC_ROOT) + webpack.add_argument('-e', '--env', choices=['prod', 'dev'], default='prod') + webpack.set_defaults(func=run_webpack) + + common = subparsers.add_parser('common', help="Compile static assets for common theme") + common.add_argument('--systems', nargs='+', default=['lms', 'cms']) + common.set_defaults(func=run_common) + + themes = subparsers.add_parser('themes', help="Compile static assets for custom themes") + themes.add_argument('--theme-dirs', nargs='+', default=[DEFAULT_THEMES_DIR]) + themes.add_argument('--themes', nargs='+', default=['all']) + themes.add_argument('--systems', nargs='+', default=['lms', 'cms']) + themes.set_defaults(func=run_themes) + + collect = subparsers.add_parser('collect', help="Collect static assets to be served by webserver") + collect.add_argument('-s', '--settings', default=os.environ.get('SETTINGS'), help="Django settings module") + collect.add_argument('--systems', nargs='+', choices=['lms', 'cms'], default=['lms', 'cms'], help="Limit collection to lms or cms") + collect.set_defaults(func=run_collect) + + watch_themes = subparsers.add_parser('watch-themes', help="Watch theme assets for changes and recompile on-the-fly") + watch_themes.add_argument('-e', '--env', choices=['prod', 'dev'], default='prod', help="Webpack target to run") + watch_themes.add_argument('--theme-dirs', default=[DEFAULT_THEMES_DIR]) + watch_themes.set_defaults(func=run_watch_themes) + + args = parser.parse_args() + args.func(args) + +def run_build(args): + run_xmodule(args) + run_npm(args) + run_webpack(args) + run_common(args) + run_themes(args) + +def run_xmodule(args): + sys.argv[1:] = ['common/static/xmodule'] + xmodule_static_content.main() + +def run_npm(args): + assets.process_npm_assets() + +def run_webpack(args): + os.environ['STATIC_ROOT_LMS'] = args.static_root + os.environ['STATIC_ROOT_CMS'] = args.static_root + os.environ['NODE_ENV'] = { + 'prod': 'production', + 'dev': 'development', + }[args.env] + subprocess.call([ + 'webpack', '--config=webpack.{env}.config.js'.format(env=args.env) + ]) + +def run_common(args): + for system in args.systems: + print("Compiling {} sass assets from common theme...".format(system)) + assets._compile_sass(system, None, False, False, []) + +def run_themes(args): + for theme_dir in args.theme_dirs: + local_themes = list_subdirectories(theme_dir) if 'all' in args.themes else args.themes + for theme in local_themes: + theme_path = os.path.join(theme_dir, theme) + if os.path.exists(theme_path): + for system in args.systems: + print("Compiling {} sass assets from theme {}...".format(system, theme_path)) + assets._compile_sass(system, Path(theme_path), False, False, []) + +def run_collect(args): + assets.collect_assets(args.systems, args.settings) + +def run_watch_themes(args): + """ + Watch static assets for changes and re-compile those changes when + necessary. This piece of code is heavily inspired from the + edx-platform/pavelib/assets.py:watch_assets function, which could not be + used directly because it does not properly read the platform settings + environment variable. + + Note that this function will only work for watching assets in development + mode. In production, watching changes does not make much sense anyway. + """ + observer = assets.Observer() + for theme_dir in args.theme_dirs: + print("Watching changes in {}...".format(theme_dir)) + ThemeWatcher(theme_dir).register(observer) + observer.start() + try: + while True: + observer.join(2) + except KeyboardInterrupt: + observer.stop() + +def list_subdirectories(path): + return [subpath for subpath in os.listdir(path) if os.path.isdir(os.path.join(path, subpath))] + + +class ThemeWatcher(assets.SassWatcher): + + def __init__(self, theme_dir): + super(ThemeWatcher, self).__init__() + self.theme_dir = theme_dir + + #pylint: disable=arguments-differ + def register(self, observer): + return super(ThemeWatcher, self).register(observer, [self.theme_dir]) + + @assets.debounce() + def on_any_event(self, event): + components = os.path.relpath(event.src_path, self.theme_dir).split('/') + try: + theme = components[0] + system = components[1] + except IndexError: + return + try: + print("Detected change:", event.src_path) + print("\tRecompiling {} theme for {}".format(theme, system)) + assets._compile_sass(system, Path(self.theme_dir) / theme, False, False, []) + print("\tDone recompiling {} theme for {}".format(theme, system)) + except Exception: # pylint: disable=broad-except + traceback.print_exc() + + +if __name__ == '__main__': + main() diff --git a/openedx/settings/cms/assets.py b/openedx/settings/cms/assets.py index e62ffcd..03b4433 100644 --- a/openedx/settings/cms/assets.py +++ b/openedx/settings/cms/assets.py @@ -6,7 +6,7 @@ from openedx.core.lib.derived import derive_settings COMPREHENSIVE_THEME_DIRS.append('/openedx/themes') -STATIC_ROOT_BASE = '/openedx/data/staticfiles' +STATIC_ROOT_BASE = '/openedx/staticfiles' STATIC_ROOT = path(STATIC_ROOT_BASE) / 'studio' WEBPACK_LOADER['DEFAULT']['STATS_FILE'] = STATIC_ROOT / "webpack-stats.json" diff --git a/openedx/settings/lms/assets.py b/openedx/settings/lms/assets.py index a195c1a..eeb7c71 100644 --- a/openedx/settings/lms/assets.py +++ b/openedx/settings/lms/assets.py @@ -5,7 +5,7 @@ from ..common import * from openedx.core.lib.derived import derive_settings COMPREHENSIVE_THEME_DIRS.append('/openedx/themes') -STATIC_ROOT_BASE = '/openedx/data/staticfiles' +STATIC_ROOT_BASE = '/openedx/staticfiles' STATIC_ROOT = path(STATIC_ROOT_BASE) WEBPACK_LOADER['DEFAULT']['STATS_FILE'] = STATIC_ROOT / "webpack-stats.json"