christianlight
/
tutor
mirror of https://github.com/ChristianLight/tutor.git
7
0
Fork 0
Code Issues Releases Activity

Multiple improvements, including Ginkgo compatibility 

Docker stack now includes a studio, smtp server, rabbitmq container and
an LMS worker. There are still a couple things to do, though. (see
TODOs)
This commit is contained in:
Régis Behmo 2017-07-24 11:32:50 +02:00
parent 34b185b0eb
commit e357e763d7
13 changed files with 320 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Makefile
View File

@ -1,4 +1,12 @@
directories:
mkdir -p ./data/edxapp/logs ./data/edxapp/uploads ./data/edxapp/staticfiles
migrate:
docker-compose run lms ./manage.py lms migrate --settings=production
docker-compose run cms ./manage.py cms migrate --settings=production
assets:
docker-compose run lms paver update_assets lms --settings=production
docker-compose run cms paver update_assets cms --settings=production
lms-shell:
docker-compose run lms ./manage.py lms shell --settings=production
cms-shell:
docker-compose run lms ./manage.py cms shell --settings=production

35
README.md
View File

@ -2,33 +2,48 @@
This is a work-in-progress.
The production stack is composed of Nginx, MySQL, MongoDB, Memcache and an LMS container.
The production stack is sufficient for a minimal production deployment of Open edX.
## Lauch a production stack
docker-compose up --build
Prepare build:
The LMS will be reachable at the following url: [http://openedx.localhost](http://openedx.localhost).
make directories
Build and run:
docker-compose build # go get a coffee
docker-compose up
The LMS will be reachable at [http://openedxdemo.overhang.io](http://openedxdemo.overhang.io).
The CMS will be reachable at [http://studio.openedxdemo.overhang.io](http://studio.openedxdemo.overhang.io).
For local development, you should point to http://localhost:8800.
On the first run you will need to migrate the database and collect static assets:
make migrate
make assets
## Development tips & tricks
To daemonize:
docker-compose up -d
## Development
Open a bash in the lms:
docker-compose run lms bash
How to find the IP address of a running docker:
Open a python shell in the lms or the cms:
docker container ls
docker inspect a0fc4cc602f8
make lms-shell
make cms-shell
## TODO
- Add a CMS container
- Add rabbitmq and celery worker containers
- Make sure that secret keys are not shared with the entire world
- Fix TODOs
- Add arguments to set domain name, platform name, etc.
- Add documentation on host Nginx
- Better readme

64
docker-compose.yml
View File

@ -1,6 +1,8 @@
version: "3"
services:
############# External services
memcached:
image: memcached:1.4.38
@ -25,26 +27,66 @@ services:
nginx:
build: ./nginx
restart: on-failure
ports:
- "8800:80"
volumes:
- ./data/lms/course_static:/openedx/course_static:ro
- ./data/lms/staticfiles:/openedx/staticfiles:ro
- ./data/lms/uploads:/openedx/uploads:ro
- ./data/edxapp:/openedx/data:ro
depends_on:
- lms
rabbitmq:
image: rabbitmq:3.6.10
volumes:
- ./data/rabbitmq:/var/lib/rabbitmq
# Simple SMTP server
smtp:
image: namshi/smtp
environment:
PORT: 9025
############# LMS and CMS
lms:
build:
context: ./lms
context: ./edxapp
args:
service_variant: lms
restart: on-failure
volumes:
- ./data/lms/course_static:/openedx/course_static
- ./data/lms/data:/openedx/data
- ./data/lms/logs:/openedx/logs
- ./data/lms/staticfiles:/openedx/staticfiles
- ./data/lms/uploads:/openedx/uploads
- ./data/edxapp:/openedx/data
depends_on:
- memcached
- mongodb
- mysql
- rabbitmq
- smtp
# TODO rabbitmq and celery workers
cms:
build:
context: ./edxapp
args:
service_variant: cms
restart: on-failure
volumes:
- ./data/edxapp:/openedx/data
depends_on:
- lms
############# LMS and CMS workers
# TODO one service per queue?
lms_worker:
build:
context: ./edxapp
args:
service_variant: lms
command: ./manage.py lms --settings=production celery worker --loglevel=info --hostname=edx.lms.core.default.%%h --maxtasksperchild 100
environment:
C_FORCE_ROOT: "1" # run celery tasks as root #nofear
restart: on-failure
volumes:
- ./data/edxapp:/openedx/data
depends_on:
- lms

34
lms/Dockerfile → edxapp/Dockerfile
View File

@ -1,33 +1,29 @@
FROM ubuntu:16.04
############ common to lms & cms
# Install system requirements
RUN apt update
RUN apt upgrade -y
# Global requirements
RUN apt install -y language-pack-en git python-virtualenv build-essential software-properties-common curl git-core libxml2-dev libxslt1-dev python-pip libmysqlclient-dev python-apt python-dev libxmlsec1-dev libfreetype6-dev swig gcc g++
# lms requirements
# edxapp requirements
RUN apt install -y gettext gfortran graphviz graphviz-dev libffi-dev libfreetype6-dev libgeos-dev libjpeg8-dev liblapack-dev libpng12-dev libxml2-dev libxmlsec1-dev libxslt1-dev nodejs npm ntp pkg-config
# Install symlink so that we have access to 'node' binary without virtualenv.
# This replaces the "nodeenv" install.
RUN apt install -y nodejs-legacy
# Create necessary folders
RUN mkdir /openedx
RUN mkdir /openedx/data
RUN mkdir /openedx/logs
RUN mkdir /openedx/uploads
RUN mkdir /openedx/staticfiles
# Static assets will reside in /openedx/data and edx-platform will be
# checked-out in /openedx
VOLUME /openedx/data
VOLUME /openedx/logs
VOLUME /openedx/staticfiles
VOLUME /openedx/uploads
WORKDIR /openedx
# Checkout edx-platform code
RUN git clone https://github.com/edx/edx-platform.git
WORKDIR /openedx/edx-platform
RUN git checkout open-release/ficus.master
RUN git fetch && \
git checkout open-release/ginkgo.master
# Install python requirements
RUN pip install pip==8.1.2
@ -44,13 +40,21 @@ RUN paver install_prereqs
# Copy configuration files
COPY ./config/lms.env.json /openedx/
COPY ./config/cms.env.json /openedx/
COPY ./config/lms.auth.json /openedx/
COPY ./config/production.py /openedx/edx-platform/lms/envs/
COPY ./config/cms.auth.json /openedx/
COPY ./config/production_lms.py /openedx/edx-platform/lms/envs/production.py
COPY ./config/production_cms.py /openedx/edx-platform/cms/envs/production.py
############ End of code common to lms & cms
# service variang is "lms" or "cms"
ARG service_variant
# Configure environment
ENV DJANGO_SETTINGS_MODULE lms.envs.production
ENV SERVICE_VARIANT lms
ENV DJANGO_SETTINGS_MODULE ${service_variant}.envs.production
ENV SERVICE_VARIANT ${service_variant}
# Run server
EXPOSE 8000
CMD gunicorn --name lms --bind=0.0.0.0:8000 --max-requests=1000 lms.wsgi:application
CMD gunicorn --name ${SERVICE_VARIANT} --bind=0.0.0.0:8000 --max-requests=1000 ${SERVICE_VARIANT}.wsgi:application

0
lms/config/lms.auth.json → edxapp/config/cms.auth.json
View File

69
edxapp/config/cms.env.json Normal file
View File

@ -0,0 +1,69 @@
{
"SITE_NAME": "studio.openedxdemo.overhang.io",
"BOOK_URL": "",
"LOG_DIR": "/openedx/data/logs",
"LOGGING_ENV": "sandbox",
"OAUTH_OIDC_ISSUER": "http://localhost:8000/oauth2",
"PLATFORM_NAME": "Open edX Studio Demo Site (Ginkgo)",
"FEATURES": {
"PREVIEW_LMS_BASE": "localhost:8000"
},
"LMS_ROOT_URL": "http://openedxdemo.overhang.io",
"CMS_ROOT_URL": "http://studio.openedxdemo.overhang.io",
"CMS_BASE": "studio.openedxdemo.overhang.io",
"LMS_BASE": "openedxdemo.overhang.io",
"CELERY_BROKER_HOSTNAME": "rabbitmq",
"CELERY_BROKER_TRANSPORT": "amqp",
"MEDIA_ROOT": "/openedx/data/uploads/",
"STATIC_ROOT_BASE": "/openedx/data/staticfiles",
"EMAIL_HOST": "smtp",
"EMAIL_PORT": 9025,
"CACHES": {
"default": {
"KEY_PREFIX": "default",
"VERSION": "1",
"BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
"KEY_FUNCTION": "util.memcache.safe_key",
"LOCATION": "memcached:11211"
},
"general": {
"KEY_PREFIX": "general",
"BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
"KEY_FUNCTION": "util.memcache.safe_key",
"LOCATION": "memcached:11211"
},
"mongo_metadata_inheritance": {
"KEY_PREFIX": "mongo_metadata_inheritance",
"TIMEOUT": 300,
"BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
"KEY_FUNCTION": "util.memcache.safe_key",
"LOCATION": "memcached:11211"
},
"staticfiles": {
"KEY_PREFIX": "staticfiles_general",
"BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
"KEY_FUNCTION": "util.memcache.safe_key",
"LOCATION": "memcached:11211"
},
"configuration": {
"KEY_PREFIX": "configuration",
"BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
"KEY_FUNCTION": "util.memcache.safe_key",
"LOCATION": "memcached:11211"
},
"celery": {
"KEY_PREFIX": "celery",
"TIMEOUT": "7200",
"BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
"KEY_FUNCTION": "util.memcache.safe_key",
"LOCATION": "memcached:11211"
},
"course_structure_cache": {
"KEY_PREFIX": "course_structure",
"TIMEOUT": "7200",
"BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
"KEY_FUNCTION": "util.memcache.safe_key",
"LOCATION": "memcached:11211"
}
}
}

35
edxapp/config/lms.auth.json Normal file
View File

@ -0,0 +1,35 @@
{
"SECRET_KEY": "7i#nri2i@--brp0sri9qf@ewlj1qxghv0%af$sk4ntn9pv$8t#",
"AWS_ACCESS_KEY_ID": "",
"AWS_SECRET_ACCESS_KEY": "",
"XQUEUE_INTERFACE": {
"basic_auth": ["edx", "edx"],
"django_auth": {
"username": "lms",
"password": "password"
},
"url": "http://localhost:18040"
},
"CONTENTSTORE": {
"ENGINE": "xmodule.contentstore.mongo.MongoContentStore",
"DOC_STORE_CONFIG": {
"db": "edxapp",
"host": "mongodb"
}
},
"DOC_STORE_CONFIG": {
"db": "edxapp",
"host": "mongodb"
},
"DATABASES": {
"default": {
"ENGINE": "django.db.backends.mysql",
"NAME": "openedx",
"USER": "openedx",
"PASSWORD": "password",
"HOST": "mysql",
"PORT": "3306",
"ATOMIC_REQUESTS": true
}
}
}

21
lms/config/lms.env.json → edxapp/config/lms.env.json
View File

@ -1,20 +1,23 @@
{
"SITE_NAME": "openedx.localhost",
"SITE_NAME": "openedxdemo.overhang.io",
"BOOK_URL": "",
"LOG_DIR": "/openedx/logs",
"LOG_DIR": "/openedx/data/logs",
"LOGGING_ENV": "sandbox",
"OAUTH_OIDC_ISSUER": "http://localhost:8000/oauth2",
"PLATFORM_NAME": "My Open edX",
"PLATFORM_NAME": "Open edX Demo Site (Ginkgo)",
"FEATURES": {
"PREVIEW_LMS_BASE": "localhost:8000"
},
"LMS_ROOT_URL": "http://openedx.localhost",
"CMS_ROOT_URL": "http://studio.openedx.localhost",
"CMS_BASE": "studio.openedx.localhost",
"LMS_BASE": "openedx.localhost",
"CELERY_BROKER_HOSTNAME": "localhost",
"LMS_ROOT_URL": "http://openedxdemo.overhang.io",
"CMS_ROOT_URL": "http://studio.openedxdemo.overhang.io",
"CMS_BASE": "studio.openedxdemo.overhang.io",
"LMS_BASE": "openedxdemo.overhang.io",
"CELERY_BROKER_HOSTNAME": "rabbitmq",
"CELERY_BROKER_TRANSPORT": "amqp",
"MEDIA_ROOT": "/openedx/uploads/",
"MEDIA_ROOT": "/openedx/data/uploads/",
"STATIC_ROOT_BASE": "/openedx/data/staticfiles",
"EMAIL_HOST": "smtp",
"EMAIL_PORT": 9025,
"CACHES": {
"default": {
"KEY_PREFIX": "default",

6
lms/config/production.py → edxapp/config/production_cms.py
View File

@ -7,10 +7,12 @@ FEATURES['ENABLE_DISCUSSION_SERVICE'] = False
ALLOWED_HOSTS = [
'*',
ENV_TOKENS.get('LMS_BASE'),
FEATURES['PREVIEW_LMS_BASE'],
ENV_TOKENS.get('CMS_BASE'),
]
# Don't rely on AWS for sending email
EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend"
# We need to activate dev_env for logging, otherwise rsyslog is required (but
# it is not available in docker).
LOGGING = get_logger_config(LOG_DIR,

23
edxapp/config/production_lms.py Normal file
View File

@ -0,0 +1,23 @@
from .aws import *
update_module_store_settings(MODULESTORE, doc_store_settings=DOC_STORE_CONFIG)
MEDIA_ROOT = "/openedx/data/uploads/"
FEATURES['ENABLE_DISCUSSION_SERVICE'] = False
ALLOWED_HOSTS = [
'*',# TODO really?
ENV_TOKENS.get('LMS_BASE'),
FEATURES['PREVIEW_LMS_BASE'],
]
# Don't rely on AWS for sending email
EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend"
# We need to activate dev_env for logging, otherwise rsyslog is required (but
# it is not available in docker).
LOGGING = get_logger_config(LOG_DIR,
logging_env=ENV_TOKENS['LOGGING_ENV'],
debug=False,
dev_env=True,
service_variant=SERVICE_VARIANT)

15
nginx/Dockerfile
View File

@ -1,15 +1,6 @@
FROM nginx:1.13
RUN mkdir /openedx
RUN mkdir /openedx/course_static
RUN mkdir /openedx/static_files
RUN mkdir /openedx/uploads
VOLUME /openedx/data
VOLUME /openedx/course_static
VOLUME /openedx/staticfiles
VOLUME /openedx/uploads
# Wait until LMS becomes available
# TODO we shouldn't have to wait
RUN sleep 10
COPY lms.conf /etc/nginx/conf.d/lms.conf
COPY ./config/lms.conf /etc/nginx/conf.d/lms.conf
COPY ./config/cms.conf /etc/nginx/conf.d/cms.conf

67
nginx/config/cms.conf Normal file
View File

@ -0,0 +1,67 @@
upstream cms-backend {
server cms:8000 fail_timeout=0;
}
server {
listen 80;
server_name studio.openedxdemo.overhang.io;
# Prevent invalid display courseware in IE 10+ with high privacy settings
add_header P3P 'CP="Open edX does not have a P3P policy."';
# Nginx does not support nested condition or or conditions so
# there is an unfortunate mix of conditonals here.
client_max_body_size 100M;
rewrite ^(.*)/favicon.ico$ /static/images/favicon.ico last;
# Disables server version feedback on pages and in headers
server_tokens off;
location @proxy_to_cms_app {
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-Port $server_port;
proxy_set_header X-Forwarded-For $remote_addr;
proxy_set_header Host $http_host;
proxy_redirect off;
proxy_pass http://cms-backend;
}
location / {
try_files $uri @proxy_to_cms_app;
}
location ~ ^/static/(?P<file>.*) {
root /openedx/data;
try_files /staticfiles/$file /course_static/$file =404;
# return a 403 for static files that shouldn't be
# in the staticfiles directory
location ~ ^/static/(?:.*)(?:\.xml|\.json|README.TXT) {
return 403;
}
# Set django-pipelined files to maximum cache time
location ~ "/static/(?P<collected>.*\.[0-9a-f]{12}\..*)" {
expires max;
# Without this try_files, files that have been run through
# django-pipeline return 404s
try_files /staticfiles/$collected /course_static/$collected =404;
}
# Set django-pipelined files for studio to maximum cache time
location ~ "/static/(?P<collected>[0-9a-f]{7}/.*)" {
expires max;
# Without this try_files, files that have been run through
# django-pipeline return 404s
try_files /staticfiles/$collected /course_static/$collected =404;
}
# Expire other static files immediately (there should be very few / none of these)
expires epoch;
}
}

4
nginx/lms.conf → nginx/config/lms.conf
View File

@ -4,7 +4,7 @@ upstream lms-backend {
server {
listen 80;
server_name openedx.localhost;
server_name openedxdemo.overhang.io;
# Prevent invalid display courseware in IE 10+ with high privacy settings
add_header P3P 'CP="Open edX does not have a P3P policy."';
@ -57,7 +57,7 @@ server {
}
location ~ ^/static/(?P<file>.*) {
root /openedx;
root /openedx/data;
try_files /staticfiles/$file /course_static/$file =404;
# return a 403 for static files that shouldn't be