christianlight
/
tutor
mirror of https://github.com/ChristianLight/tutor.git
7
0
Fork 0
Code Issues Releases Activity

Add openedx-assets command 

This is great for running a local webserver and watching theme changes
in just two commands.
This commit is contained in:
Régis Behmo 2018-12-24 08:54:32 +01:00 committed by Régis Behmo
parent 22aa54d88f
commit d454cfd72d
12 changed files with 234 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
Makefile
View File

@ -15,7 +15,7 @@ endif
ifeq ($(ACTIVATE_HTTPS), 1)
post_configure_targets += https-certificate
endif
extra_migrate_targets =
extra_migrate_targets =
ifeq ($(ACTIVATE_XQUEUE), 1)
extra_migrate_targets += migrate-xqueue
DOCKER_COMPOSE += -f docker-compose-xqueue.yml
@ -29,7 +29,8 @@ ifeq ($(ACTIVATE_PORTAINER), 1)
endif
DOCKER_COMPOSE_RUN = $(DOCKER_COMPOSE) run --rm
DOCKER_COMPOSE_RUN_OPENEDX = $(DOCKER_COMPOSE_RUN) -e SETTINGS=$(EDX_PLATFORM_SETTINGS)
DOCKER_COMPOSE_RUN_OPENEDX = $(DOCKER_COMPOSE_RUN) -e SETTINGS=$(EDX_PLATFORM_SETTINGS) \
--volume="$(PWD)/openedx/themes:/openedx/themes"
ifneq ($(EDX_PLATFORM_PATH),)
DOCKER_COMPOSE_RUN_OPENEDX += -e USERID=$(USERID) --volume="$(EDX_PLATFORM_PATH):/openedx/edx-platform"
endif
@ -99,34 +100,20 @@ reindex-courses: ## Refresh course index so they can be found in the LMS search
# webpack collection incorrectly sets the NODE_ENV variable when using custom
# settings. Thus, each step must be performed separately. This should be fixed
# in the next edx-platform release thanks to https://github.com/edx/edx-platform/pull/18430/
#assets-lms: ## Collect static assets for the LMS
# $(DOCKER_COMPOSE_RUN_OPENEDX) lms -e NO_PREREQ_INSTALL=True lms paver update_assets lms --settings=$(EDX_PLATFORM_SETTINGS)
#assets-cms: ## Collect static assets for the CMS
# $(DOCKER_COMPOSE_RUN_OPENEDX) cms -e NO_PREREQ_INSTALL=True cms paver update_assets cms --settings=$(EDX_PLATFORM_SETTINGS)
assets-development: assets-development-lms assets-development-cms ## Generate static assets for local development
assets: ## Generate production-ready static assets
docker-compose -f docker-compose-scripts.yml run --rm \
--volume=$(PWD)/data/lms/:/data/lms/ --volume=$(PWD)/data/cms/:/data/cms/ openedx bash -c \
"rm -rf /data/lms/staticfiles /data/cms/staticfiles \
&& cp -r /openedx/data/staticfiles /data/lms/ \
&& cp -r /openedx/data/staticfiles /data/cms/"
--volume=$(PWD)/data/openedx:/tmp/openedx/ openedx bash -c \
"rm -rf /tmp/openedx/staticfiles \
&& cp -r /openedx/staticfiles /tmp/openedx"
assets-development: ## Generate static assets for local development
$(DOCKER_COMPOSE_RUN_OPENEDX) --no-deps lms bash -c "openedx-assets build --env=dev"
assets-development-lms:
$(DOCKER_COMPOSE_RUN_OPENEDX) --no-deps lms bash -c \
"xmodule_assets common/static/xmodule \
&& python -c \"import pavelib.assets; pavelib.assets.process_npm_assets()\" \
&& NODE_ENV=development ./node_modules/.bin/webpack --config=webpack.dev.config.js \
&& ./manage.py lms --settings=$(EDX_PLATFORM_SETTINGS) compile_sass lms \
&& python -c \"import pavelib.assets; pavelib.assets.collect_assets(['lms'], '$(EDX_PLATFORM_SETTINGS)')\""
$(DOCKER_COMPOSE_RUN_OPENEDX) --no-deps lms bash -c "openedx-assets build --env=dev --system lms"
assets-development-cms:
$(DOCKER_COMPOSE_RUN_OPENEDX) --no-deps cms bash -c \
"xmodule_assets common/static/xmodule \
&& python -c \"import pavelib.assets; pavelib.assets.process_npm_assets()\" \
&& NODE_ENV=development ./node_modules/.bin/webpack --config=webpack.dev.config.js \
&& ./manage.py cms --settings=$(EDX_PLATFORM_SETTINGS) compile_sass studio \
&& python -c \"import pavelib.assets; pavelib.assets.collect_assets(['studio'], '$(EDX_PLATFORM_SETTINGS)')\""
$(DOCKER_COMPOSE_RUN_OPENEDX) --no-deps lms bash -c "openedx-assets build --env=dev --system cms"
watch-themes: ## Watch for changes in your themes and build development assets
$(DOCKER_COMPOSE_RUN_OPENEDX) --no-deps lms openedx-assets watch-themes --env dev
##################### Information
@ -184,7 +171,7 @@ build-notes: ## Build the Notes docker image
docker build -t regis/openedx-notes:latest -t regis/openedx-notes:hawthorn notes/
build-xqueue: ## Build the Xqueue docker image
docker build -t regis/openedx-xqueue:latest -t regis/openedx-xqueue:hawthorn xqueue/
build-android: ## Build the docker image for Android
build-android: ## Build the docker image for Android
docker build -t regis/openedx-android:latest android/
################### Pushing images to docker hub
@ -220,9 +207,13 @@ cms: ## Open a bash shell in the CMS
lms-python: ## Open a python shell in the LMS
$(DOCKER_COMPOSE_RUN_OPENEDX) lms ./manage.py lms shell
lms-shell: lms-python
lms-runserver: ## Run a local webserver, useful for debugging
$(DOCKER_COMPOSE_RUN_LMS) ./manage.py lms runserver 0.0.0.0:8000
cms-python: ## Open a python shell in the CMS
$(DOCKER_COMPOSE_RUN_OPENEDX) cms ./manage.py cms shell
cms-shell: cms-python
cms-runserver: ## Run a local webserver, useful for debugging
$(DOCKER_COMPOSE_RUN_CMS) ./manage.py cms runserver 0.0.0.0:8001
restart-openedx: ## Restart lms, cms, and workers
docker-compose restart lms lms_worker cms cms_worker

65
README.md
View File

@ -201,27 +201,21 @@ Open a python shell in the lms or the cms:
In addition to running Open edX in production, you can use the docker containers for local development. This means you can hack on Open edX without setting up a Virtual Machine. Essentially, this replaces the devstack provided by edX.
(Note: containers are built on the Hawthorn release. If you are working on a different version of Open edX, you will have to rebuild the images with a different `EDX_PLATFORM_VERSION` argument. You may also want to change the `EDX_PLATFORM_REPOSITORY` argument to point to your own fork of edx-platform.)
### Standard devstack
Define development settings (on the host):
To begin with, define development settings:
export EDX_PLATFORM_SETTINGS=universal.development
Then open an LMS shell:
### Run a local webserver
make lms-runserver
make cms-runserver
### Open a bash shell
make lms
make cms
You can then run a local web server, as usual:
paver update_assets lms --settings=universal.development
Note that assets collection is made more difficult by the fact that development settings are [incorrectly loaded in hawthorn](https://github.com/edx/edx-platform/pull/18430/files). This should be fixed in the next release. Meanwhile, do not run `paver update_assets` while in development mode. Instead, run on the host:
make assets-development
### Custom devstack
### Debug edx-platform
If you have one, you can point to a local version of [edx-platform](https://github.com/edx/edx-platform/) on your host machine:
@ -229,31 +223,38 @@ If you have one, you can point to a local version of [edx-platform](https://gith
Note that you should use an absolute path here, not a relative path (e.g: `/path/to/edx-platform` and not `../edx-platform`).
Point to your settings file:
All development commands will then automatically mount your local repo. For instance, you can add a `import pdb; pdb.set_trace()` breakpoint anywhere in your code and run:
export EDX_PLATFORM_SETTINGS=mysettings.py
make lms-runserver
In this example, you should have a `mysettings.py` file in `edx-platform/lms/envs` and `edx-platform/cms/envs`. Development settings file for docker are a bit different from stock devstack settings. For valid development settings files, check [`config/openedx/universal/lms/development.py`](https://github.com/regisb/openedx-docker/blob/master/config/openedx/universal/lms/development.py) and [`config/openedx/universal/cms/development.py`](https://github.com/regisb/openedx-docker/blob/master/config/openedx/universal/cms/development.py)
Note: containers are built on the Hawthorn release. If you are working on a different version of Open edX, you will have to rebuild the images with the right `EDX_PLATFORM_VERSION` argument. You may also want to change the `EDX_PLATFORM_REPOSITORY` argument to point to your own fork of edx-platform.
You are ready to go! Run:
With a customised edx-platform repo, you must be careful to have settings that are compatible with the docker environment. You are encouraged to copy the `universal.development` settings files to our own repo:
make lms
cp -r config/openedx/universal/lms/ /path/to/edx-platform/lms/envs/universal
cp -r config/openedx/universal/cms/ /path/to/edx-platform/cms/envs/universal
Or:
You can then run your platform with the `universal.development` settings.
make cms
### Develop customised themes
This will open a shell in the LMS (or CMS) container. You can then run just any command you are used to. For example, install node requirements, collect assets and run a local server:
Run a local webserver:
npm install
paver update_assets lms --settings=mysettings
./manage.py lms runserver 0.0.0.0:8000
make lms-runserver
## Maintainers
Watch the themes folders for changes:
The images are built, tagged and uploaded to Docker Hub in one command:
make watch-themes
make dockerhub
Make changes to `openedx/themes/yourtheme`: the theme assets should be automatically recompiled and visible at http://localhost:8000.
### Assets management
Assets building and collecting is made more difficult by the fact that development settings are [incorrectly loaded in Hawthorn](https://github.com/edx/edx-platform/pull/18430/files). This should be fixed in the next Open edX release. Meanwhile, do not run `paver update_assets` while in development mode. When working locally on a theme, build assets by running in the container:
openedx-assets build
This command will take quite some time to run. You can speed up this process by running only part of the full build. Run `openedx-assets -h` for more information.
## Customising the `openedx` docker image
@ -328,6 +329,12 @@ Your own image will be used next time you run `make run`.
Note that the `make build` and `make push` command will no longer work as you expect and that you are responsible for building and pushing the image yourself.
## Maintainers
The images are built, tagged and uploaded to Docker Hub in one command:
make dockerhub
## Help/Troubleshooting
### "Cannot start service nginx: driver failed programming external connectivity"

2
configurator/templates/nginx/cms.conf
View File

@ -46,7 +46,7 @@ server {
}
location ~ ^/static/(?P<file>.*) {
root /openedx/data/cms;
root /var/www/openedx;
try_files /staticfiles/$file /course_static/$file =404;
# return a 403 for static files that shouldn't be

2
configurator/templates/nginx/lms.conf
View File

@ -68,7 +68,7 @@ server {
}
location ~ ^/static/(?P<file>.*) {
root /openedx/data/lms;
root /var/www/openedx;
try_files /staticfiles/$file /course_static/$file =404;
# return a 403 for static files that shouldn't be

2
configurator/templates/openedx/cms.env.json
View File

@ -20,7 +20,7 @@
"CELERY_BROKER_TRANSPORT": "amqp",
"COMPREHENSIVE_THEME_DIRS": ["/openedx/themes"],
"MEDIA_ROOT": "/openedx/data/uploads/",
"STATIC_ROOT_BASE": "/openedx/data/staticfiles",
"STATIC_ROOT_BASE": "/openedx/staticfiles",
"ELASTIC_SEARCH_CONFIG": [{
"host": "elasticsearch",
"port": 9200

2
configurator/templates/openedx/lms.env.json
View File

@ -24,7 +24,7 @@
"COMMENTS_SERVICE_KEY": "forumapikey",
"COMPREHENSIVE_THEME_DIRS": ["/openedx/themes"],
"MEDIA_ROOT": "/openedx/data/uploads/",
"STATIC_ROOT_BASE": "/openedx/data/staticfiles",
"STATIC_ROOT_BASE": "/openedx/staticfiles",
"ELASTIC_SEARCH_CONFIG": [{
"host": "elasticsearch",
"port": 9200

1
data/.gitignore vendored
View File

@ -7,6 +7,7 @@ lms_worker/
elasticsearch/
mysql/
mongodb/
openedx/
portainer/
rabbitmq/
xqueue/

3
docker-compose.yml
View File

@ -45,8 +45,7 @@ services:
- "${NGINX_HTTPS_PORT:-443}:443"
volumes:
- ./config/nginx:/etc/nginx/conf.d/
- ./data/lms:/openedx/data/lms:ro
- ./data/cms:/openedx/data/cms:ro
- ./data/openedx:/var/www/openedx:ro
- ./data/letsencrypt:/etc/letsencrypt/:ro
rabbitmq:

21
openedx/Dockerfile
View File

@ -63,7 +63,7 @@ ENV PATH ./node_modules/.bin:${PATH}
# ./requirements/private.txt.
COPY ./requirements/ /openedx/requirements
RUN touch /openedx/requirements/private.txt \
&& pip install --src ../venv/src -r /openedx/requirements/private.txt
&& pip install -r /openedx/requirements/private.txt
# Link configuration files to common /openedx/config folder, which should later
# be mounted as a volume. Note that this image will not be functional until
@ -84,18 +84,15 @@ COPY settings/cms/*.py /openedx/config/universal/cms/
# Here, we don't run "paver update_assets" which is slow, compiles all themes
# and requires a complex settings file. Instead, we decompose the commands
# and run each one individually to collect the production static assets to
# /openedx/data/staticfiles.
# 1. xmodule_assets: run xmodule.static_content.main, which lists all xblocks and generates webpack manifest in common/static/xmodule
# 2. pavelib.assets.process_npm_assets: copy libraries installed via npm to common/static/common/*/vendor
# 3. webpack: generate webpack-stats.json
# 4. compile_sass: compile each sass file individually with libsass.
# 5. pavelib.assets.collect_assets: run ./manage.py lms/cms collectstatic. This is the only command that requires a settings file.
RUN xmodule_assets common/static/xmodule \
&& python -c "import pavelib.assets; pavelib.assets.process_npm_assets()" \
&& STATIC_ROOT_LMS=/openedx/data/staticfiles STATIC_ROOT_CMS=/openedx/data/staticfiles/studio NODE_ENV=production ./node_modules/.bin/webpack --config=webpack.prod.config.js
# /openedx/staticfiles.
COPY ./bin/openedx-assets /usr/local/bin/
RUN openedx-assets xmodule \
&& openedx-assets npm \
&& openedx-assets webpack --env=prod \
&& openedx-assets common
COPY ./themes/ /openedx/themes/
RUN paver compile_sass --theme-dirs=/openedx/edx-platform/themes,/openedx/themes --themes=open-edx,$(ls /openedx/themes | paste -s -d, -) \
&& python -c "import pavelib.assets; pavelib.assets.collect_assets(['lms', 'cms'], 'universal.assets')"
RUN openedx-assets themes \
&& openedx-assets collect --settings=universal.assets
# service variant is "lms" or "cms"
ENV SERVICE_VARIANT lms

164
openedx/bin/openedx-assets Executable file
View File

@ -0,0 +1,164 @@
#! /usr/bin/env python
from __future__ import print_function
import argparse
import os
import subprocess
import sys
import traceback
from path import Path
from pavelib import assets
from xmodule import static_content as xmodule_static_content
DEFAULT_STATIC_ROOT = '/openedx/staticfiles'
DEFAULT_THEMES_DIR = '/openedx/themes'
def main():
parser = argparse.ArgumentParser(
description="Various assets processing/building/collection utility for Open edX"
)
subparsers = parser.add_subparsers()
npm = subparsers.add_parser('npm', help="Copy static assets from node_modules")
npm.set_defaults(func=run_npm)
build = subparsers.add_parser('build', help="Build all assets")
build.add_argument('-e', '--env', choices=['prod', 'dev'], default='prod')
build.add_argument('--theme-dirs', nargs='+', default=[DEFAULT_THEMES_DIR])
build.add_argument('--themes', nargs='+', default=['all'])
build.add_argument('-r', '--static-root', default=DEFAULT_STATIC_ROOT)
build.add_argument('--systems', nargs='+', default=['lms', 'cms'])
build.set_defaults(func=run_build)
xmodule = subparsers.add_parser('xmodule', help="Process assets from xmodule")
xmodule.set_defaults(func=run_xmodule)
webpack = subparsers.add_parser('webpack', help="Run webpack")
webpack.add_argument('-r', '--static-root', default=DEFAULT_STATIC_ROOT)
webpack.add_argument('-e', '--env', choices=['prod', 'dev'], default='prod')
webpack.set_defaults(func=run_webpack)
common = subparsers.add_parser('common', help="Compile static assets for common theme")
common.add_argument('--systems', nargs='+', default=['lms', 'cms'])
common.set_defaults(func=run_common)
themes = subparsers.add_parser('themes', help="Compile static assets for custom themes")
themes.add_argument('--theme-dirs', nargs='+', default=[DEFAULT_THEMES_DIR])
themes.add_argument('--themes', nargs='+', default=['all'])
themes.add_argument('--systems', nargs='+', default=['lms', 'cms'])
themes.set_defaults(func=run_themes)
collect = subparsers.add_parser('collect', help="Collect static assets to be served by webserver")
collect.add_argument('-s', '--settings', default=os.environ.get('SETTINGS'), help="Django settings module")
collect.add_argument('--systems', nargs='+', choices=['lms', 'cms'], default=['lms', 'cms'], help="Limit collection to lms or cms")
collect.set_defaults(func=run_collect)
watch_themes = subparsers.add_parser('watch-themes', help="Watch theme assets for changes and recompile on-the-fly")
watch_themes.add_argument('-e', '--env', choices=['prod', 'dev'], default='prod', help="Webpack target to run")
watch_themes.add_argument('--theme-dirs', default=[DEFAULT_THEMES_DIR])
watch_themes.set_defaults(func=run_watch_themes)
args = parser.parse_args()
args.func(args)
def run_build(args):
run_xmodule(args)
run_npm(args)
run_webpack(args)
run_common(args)
run_themes(args)
def run_xmodule(args):
sys.argv[1:] = ['common/static/xmodule']
xmodule_static_content.main()
def run_npm(args):
assets.process_npm_assets()
def run_webpack(args):
os.environ['STATIC_ROOT_LMS'] = args.static_root
os.environ['STATIC_ROOT_CMS'] = args.static_root
os.environ['NODE_ENV'] = {
'prod': 'production',
'dev': 'development',
}[args.env]
subprocess.call([
'webpack', '--config=webpack.{env}.config.js'.format(env=args.env)
])
def run_common(args):
for system in args.systems:
print("Compiling {} sass assets from common theme...".format(system))
assets._compile_sass(system, None, False, False, [])
def run_themes(args):
for theme_dir in args.theme_dirs:
local_themes = list_subdirectories(theme_dir) if 'all' in args.themes else args.themes
for theme in local_themes:
theme_path = os.path.join(theme_dir, theme)
if os.path.exists(theme_path):
for system in args.systems:
print("Compiling {} sass assets from theme {}...".format(system, theme_path))
assets._compile_sass(system, Path(theme_path), False, False, [])
def run_collect(args):
assets.collect_assets(args.systems, args.settings)
def run_watch_themes(args):
"""
Watch static assets for changes and re-compile those changes when
necessary. This piece of code is heavily inspired from the
edx-platform/pavelib/assets.py:watch_assets function, which could not be
used directly because it does not properly read the platform settings
environment variable.
Note that this function will only work for watching assets in development
mode. In production, watching changes does not make much sense anyway.
"""
observer = assets.Observer()
for theme_dir in args.theme_dirs:
print("Watching changes in {}...".format(theme_dir))
ThemeWatcher(theme_dir).register(observer)
observer.start()
try:
while True:
observer.join(2)
except KeyboardInterrupt:
observer.stop()
def list_subdirectories(path):
return [subpath for subpath in os.listdir(path) if os.path.isdir(os.path.join(path, subpath))]
class ThemeWatcher(assets.SassWatcher):
def __init__(self, theme_dir):
super(ThemeWatcher, self).__init__()
self.theme_dir = theme_dir
#pylint: disable=arguments-differ
def register(self, observer):
return super(ThemeWatcher, self).register(observer, [self.theme_dir])
@assets.debounce()
def on_any_event(self, event):
components = os.path.relpath(event.src_path, self.theme_dir).split('/')
try:
theme = components[0]
system = components[1]
except IndexError:
return
try:
print("Detected change:", event.src_path)
print("\tRecompiling {} theme for {}".format(theme, system))
assets._compile_sass(system, Path(self.theme_dir) / theme, False, False, [])
print("\tDone recompiling {} theme for {}".format(theme, system))
except Exception: # pylint: disable=broad-except
traceback.print_exc()
if __name__ == '__main__':
main()

2
openedx/settings/cms/assets.py
View File

@ -6,7 +6,7 @@ from openedx.core.lib.derived import derive_settings
COMPREHENSIVE_THEME_DIRS.append('/openedx/themes')
STATIC_ROOT_BASE = '/openedx/data/staticfiles'
STATIC_ROOT_BASE = '/openedx/staticfiles'
STATIC_ROOT = path(STATIC_ROOT_BASE) / 'studio'
WEBPACK_LOADER['DEFAULT']['STATS_FILE'] = STATIC_ROOT / "webpack-stats.json"

2
openedx/settings/lms/assets.py
View File

@ -5,7 +5,7 @@ from ..common import *
from openedx.core.lib.derived import derive_settings
COMPREHENSIVE_THEME_DIRS.append('/openedx/themes')
STATIC_ROOT_BASE = '/openedx/data/staticfiles'
STATIC_ROOT_BASE = '/openedx/staticfiles'
STATIC_ROOT = path(STATIC_ROOT_BASE)
WEBPACK_LOADER['DEFAULT']['STATS_FILE'] = STATIC_ROOT / "webpack-stats.json"