mirror of
https://github.com/ChristianLight/tutor.git
synced 2024-11-05 12:57:52 +00:00
Add xqueue services
Xqueue containers consist of two services: a gunicorn service, that receives requests from LMS/CMS, and a worker service. I guess the worker service receives orders from the gunicorn service, through rabbitmq. (but I'm less than certain about that). While adding xqueue containers, we refactored the way mysql databases are created, and how the root password is loaded. Also, we silenced some options from the configure script.
This commit is contained in:
parent
4dc20e3d2b
commit
43a97e5fe3
6
.gitignore
vendored
6
.gitignore
vendored
@ -1,7 +1,7 @@
|
||||
.*.swp
|
||||
config/openedx/*.json
|
||||
config/openedx/*.sh
|
||||
config/mysql/*.env
|
||||
config/nginx/*.conf
|
||||
config/android/*.yaml
|
||||
mysql/config/database
|
||||
mysql/config/username
|
||||
mysql/config/password
|
||||
config/xqueue/*.py
|
||||
|
15
Makefile
15
Makefile
@ -11,7 +11,7 @@ endif
|
||||
DOCKER_COMPOSE_RUN_LMS = $(DOCKER_COMPOSE_RUN_OPENEDX) -p 8000:8000 lms
|
||||
DOCKER_COMPOSE_RUN_CMS = $(DOCKER_COMPOSE_RUN_OPENEDX) -p 8001:8001 cms
|
||||
|
||||
all: configure update migrate migrate-forum assets daemon
|
||||
all: configure update migrate assets daemon
|
||||
|
||||
##################### Bootstrapping
|
||||
|
||||
@ -21,7 +21,10 @@ configure:
|
||||
update:
|
||||
docker-compose pull
|
||||
|
||||
migrate:
|
||||
provision:
|
||||
$(DOCKER_COMPOSE_RUN_OPENEDX) lms bash /openedx/config/provision.sh
|
||||
|
||||
migrate-openedx:
|
||||
$(DOCKER_COMPOSE_RUN_OPENEDX) lms bash -c "wait-for-greenlight.sh && ./manage.py lms migrate"
|
||||
$(DOCKER_COMPOSE_RUN_OPENEDX) cms bash -c "wait-for-greenlight.sh && ./manage.py cms migrate"
|
||||
|
||||
@ -29,6 +32,11 @@ migrate-forum:
|
||||
$(DOCKER_COMPOSE_RUN) forum bash -c "bundle exec rake search:initialize && \
|
||||
bundle exec rake search:rebuild_index"
|
||||
|
||||
migrate-xqueue:
|
||||
$(DOCKER_COMPOSE_RUN) xqueue bash -c "./manage.py migrate"
|
||||
|
||||
migrate: provision migrate-openedx migrate-forum migrate-xqueue
|
||||
|
||||
assets:
|
||||
$(DOCKER_COMPOSE_RUN_OPENEDX) lms paver update_assets lms --settings=$(EDX_PLATFORM_SETTINGS)
|
||||
$(DOCKER_COMPOSE_RUN_OPENEDX) cms paver update_assets cms --settings=$(EDX_PLATFORM_SETTINGS)
|
||||
@ -90,11 +98,14 @@ build:
|
||||
# We need to build with docker, as long as docker-compose cannot push to dockerhub
|
||||
docker build -t regis/openedx:latest -t regis/openedx:ginkgo openedx/
|
||||
docker build -t regis/openedx-forum:latest -t regis/openedx-forum:ginkgo forum/
|
||||
docker build -t regis/openedx-xqueue:latest -t regis/openedx-xqueue:ginkgo xqueue/
|
||||
|
||||
push:
|
||||
docker push regis/openedx:ginkgo
|
||||
docker push regis/openedx:latest
|
||||
docker push regis/openedx-forum:ginkgo
|
||||
docker push regis/openedx-forum:latest
|
||||
docker push regis/openedx-xqueue:ginkgo
|
||||
docker push regis/openedx-xqueue:latest
|
||||
|
||||
dockerhub: build push
|
||||
|
1
config/mysql/templates/auth.env.templ
Normal file
1
config/mysql/templates/auth.env.templ
Normal file
@ -0,0 +1 @@
|
||||
MYSQL_ROOT_PASSWORD: ${MYSQL_PASSWORD}
|
@ -3,12 +3,11 @@
|
||||
"AWS_ACCESS_KEY_ID": "",
|
||||
"AWS_SECRET_ACCESS_KEY": "",
|
||||
"XQUEUE_INTERFACE": {
|
||||
"basic_auth": ["edx", "edx"],
|
||||
"django_auth": {
|
||||
"username": "lms",
|
||||
"password": "password"
|
||||
"username": "${XQUEUE_AUTH_USERNAME}",
|
||||
"password": "${XQUEUE_AUTH_PASSWORD}"
|
||||
},
|
||||
"url": "http://localhost:18040"
|
||||
"url": "http://xqueue:8040"
|
||||
},
|
||||
"CONTENTSTORE": {
|
||||
"ENGINE": "xmodule.contentstore.mongo.MongoContentStore",
|
||||
|
@ -3,12 +3,11 @@
|
||||
"AWS_ACCESS_KEY_ID": "",
|
||||
"AWS_SECRET_ACCESS_KEY": "",
|
||||
"XQUEUE_INTERFACE": {
|
||||
"basic_auth": ["edx", "edx"],
|
||||
"django_auth": {
|
||||
"username": "lms",
|
||||
"password": "password"
|
||||
"username": "${XQUEUE_AUTH_USERNAME}",
|
||||
"password": "${XQUEUE_AUTH_PASSWORD}"
|
||||
},
|
||||
"url": "http://localhost:18040"
|
||||
"url": "http://xqueue:8040"
|
||||
},
|
||||
"CONTENTSTORE": {
|
||||
"ENGINE": "xmodule.contentstore.mongo.MongoContentStore",
|
||||
|
5
config/openedx/templates/provision.sh.templ
Executable file
5
config/openedx/templates/provision.sh.templ
Executable file
@ -0,0 +1,5 @@
|
||||
mysql -u root --password="${MYSQL_PASSWORD}" --host "mysql" -e 'CREATE DATABASE IF NOT EXISTS ${MYSQL_DATABASE};'
|
||||
mysql -u root --password="${MYSQL_PASSWORD}" --host "mysql" -e 'GRANT ALL ON ${MYSQL_DATABASE}.* TO "${MYSQL_USERNAME}"@"%" IDENTIFIED BY "${MYSQL_PASSWORD}";'
|
||||
|
||||
mysql -u root --password="${MYSQL_PASSWORD}" --host "mysql" -e 'CREATE DATABASE IF NOT EXISTS ${XQUEUE_MYSQL_DATABASE};'
|
||||
mysql -u root --password="${MYSQL_PASSWORD}" --host "mysql" -e 'GRANT ALL ON ${XQUEUE_MYSQL_DATABASE}.* TO "${XQUEUE_MYSQL_USERNAME}"@"%" IDENTIFIED BY "${XQUEUE_MYSQL_PASSWORD}";'
|
25
config/xqueue/templates/universal.py.templ
Normal file
25
config/xqueue/templates/universal.py.templ
Normal file
@ -0,0 +1,25 @@
|
||||
from .settings import *
|
||||
|
||||
DATABASES = {
|
||||
'default': {
|
||||
'ENGINE': 'django.db.backends.mysql',
|
||||
'NAME': '${XQUEUE_MYSQL_DATABASE}',
|
||||
'USER': '${XQUEUE_MYSQL_USERNAME}',
|
||||
'PASSWORD': '${XQUEUE_MYSQL_PASSWORD}',
|
||||
'HOST': 'mysql',
|
||||
}
|
||||
}
|
||||
|
||||
LOGGING = get_logger_config(
|
||||
log_dir="/openedx/data/",
|
||||
logging_env="universal",
|
||||
dev_env=True,
|
||||
)
|
||||
|
||||
RABBIT_HOST = 'rabbitmq'
|
||||
RABBIT_PORT = 5672
|
||||
SECRET_KEY = '${XQUEUE_SECRET_KEY}'
|
||||
|
||||
XQUEUE_USERS = {
|
||||
'${XQUEUE_AUTH_USERNAME}': '${XQUEUE_AUTH_PASSWORD}'
|
||||
}
|
58
configure
vendored
58
configure
vendored
@ -25,8 +25,10 @@ class Configurator(object):
|
||||
def as_dict(self):
|
||||
return dict(self.__values)
|
||||
|
||||
def add(self, name, question, default=""):
|
||||
def add(self, name, question="", default=""):
|
||||
default = self.__default_overrides.get(name, default)
|
||||
value = default
|
||||
if question:
|
||||
message = question + " (default: \"{}\"): ".format(default)
|
||||
value = self.ask(message, default)
|
||||
self.set(name, value)
|
||||
@ -116,19 +118,30 @@ def main():
|
||||
'LMS_HOST', "Your website domain name for students (LMS). Set 'localhost' for local testing.", 'www.myopenedx.com'
|
||||
).add(
|
||||
'CMS_HOST', "Your website domain name for teachers (CMS). Set 'studio.localhost' for local testing.", 'studio.myopenedx.com'
|
||||
).add(
|
||||
'SECRET_KEY', "Secret key -- if you don't know what this is, you can safely accept the default",
|
||||
"".join([random.choice(string.ascii_letters + string.digits) for _ in range(24)])
|
||||
).add(
|
||||
'PLATFORM_NAME', "Platform name/title", "My Open edX"
|
||||
).add(
|
||||
'MYSQL_DATABASE', "MySQL database name", 'openedx'
|
||||
'SECRET_KEY', "", random_string(24)
|
||||
).add(
|
||||
'MYSQL_USERNAME', "MySQL database username", 'openedx'
|
||||
'MYSQL_DATABASE', "", 'openedx'
|
||||
).add(
|
||||
'MYSQL_PASSWORD', "MySQL database password", 'password'
|
||||
'MYSQL_USERNAME', "", 'openedx'
|
||||
).add(
|
||||
'MONGODB_DATABASE', "MongoDb database name", 'openedx'
|
||||
'MYSQL_PASSWORD', "", random_string(8),
|
||||
).add(
|
||||
'MONGODB_DATABASE', "", 'openedx'
|
||||
).add(
|
||||
'XQUEUE_AUTH_USERNAME', "", 'lms'
|
||||
).add(
|
||||
'XQUEUE_AUTH_PASSWORD', "", random_string(8),
|
||||
).add(
|
||||
'XQUEUE_MYSQL_DATABASE', "", 'xqueue',
|
||||
).add(
|
||||
'XQUEUE_MYSQL_USERNAME', "", 'xqueue',
|
||||
).add(
|
||||
'XQUEUE_MYSQL_PASSWORD', "", random_string(8),
|
||||
).add(
|
||||
'XQUEUE_SECRET_KEY', "", random_string(24),
|
||||
)
|
||||
|
||||
# Save values
|
||||
@ -157,21 +170,23 @@ def main():
|
||||
os.path.join('config', 'openedx', 'cms.auth.json'),
|
||||
**configurator.as_dict()
|
||||
)
|
||||
substitute(
|
||||
os.path.join('config', 'openedx', 'templates', 'provision.sh.templ'),
|
||||
os.path.join('config', 'openedx', 'provision.sh'),
|
||||
**configurator.as_dict()
|
||||
)
|
||||
|
||||
# Xqueue
|
||||
substitute(
|
||||
os.path.join('config', 'xqueue', 'templates', 'universal.py.templ'),
|
||||
os.path.join('config', 'xqueue', 'universal.py'),
|
||||
**configurator.as_dict()
|
||||
)
|
||||
|
||||
# MySQL
|
||||
substitute(
|
||||
os.path.join('mysql', 'config', 'templates', 'username.templ'),
|
||||
os.path.join('mysql', 'config', 'username'),
|
||||
**configurator.as_dict()
|
||||
)
|
||||
substitute(
|
||||
os.path.join('mysql', 'config', 'templates', 'password.templ'),
|
||||
os.path.join('mysql', 'config', 'password'),
|
||||
**configurator.as_dict()
|
||||
)
|
||||
substitute(
|
||||
os.path.join('mysql', 'config', 'templates', 'database.templ'),
|
||||
os.path.join('mysql', 'config', 'database'),
|
||||
os.path.join('config', 'mysql', 'templates', 'auth.env.templ'),
|
||||
os.path.join('config', 'mysql', 'auth.env'),
|
||||
**configurator.as_dict()
|
||||
)
|
||||
|
||||
@ -199,5 +214,8 @@ def main():
|
||||
print("\nConfiguration files were successfuly generated. You may now run the app containers.")
|
||||
|
||||
|
||||
def random_string(length):
|
||||
return "".join([random.choice(string.ascii_letters + string.digits) for _ in range(length)])
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
1
data/.gitignore
vendored
1
data/.gitignore
vendored
@ -7,3 +7,4 @@ elasticsearch/
|
||||
mysql/
|
||||
mongodb/
|
||||
rabbitmq/
|
||||
xqueue/
|
||||
|
@ -21,13 +21,7 @@ services:
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- ./data/mysql:/var/lib/mysql
|
||||
- ./mysql/config/:/etc/mysql/conf.d/openedx
|
||||
environment:
|
||||
# Load values from files generated by configurator
|
||||
MYSQL_DATABASE_FILE: /etc/mysql/conf.d/openedx/database
|
||||
MYSQL_USER_FILE: /etc/mysql/conf.d/openedx/username
|
||||
MYSQL_PASSWORD_FILE: /etc/mysql/conf.d/openedx/password
|
||||
MYSQL_ROOT_PASSWORD_FILE: /etc/mysql/conf.d/openedx/password
|
||||
env_file: ./config/mysql/auth.env
|
||||
|
||||
elasticsearch:
|
||||
image: elasticsearch:1.5.2
|
||||
@ -150,3 +144,28 @@ services:
|
||||
- ./data/cms_worker:/openedx/data
|
||||
depends_on:
|
||||
- cms
|
||||
|
||||
############# Xqueue: external grading of Open edX problems
|
||||
xqueue:
|
||||
image: regis/openedx-xqueue:ginkgo
|
||||
build:
|
||||
context: ./xqueue
|
||||
volumes:
|
||||
- ./config/xqueue:/openedx/config
|
||||
- ./data/xqueue:/openedx/data
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
- mysql
|
||||
|
||||
xqueue_consumer:
|
||||
image: regis/openedx-xqueue:ginkgo
|
||||
build:
|
||||
context: ./xqueue
|
||||
volumes:
|
||||
- ./config/xqueue:/openedx/config
|
||||
- ./data/xqueue:/openedx/data
|
||||
restart: unless-stopped
|
||||
# Run 12 workers per queue
|
||||
command: ./manage.py run_consumer 12
|
||||
depends_on:
|
||||
- mysql
|
||||
|
@ -1 +0,0 @@
|
||||
${MYSQL_DATABASE}
|
@ -1 +0,0 @@
|
||||
${MYSQL_PASSWORD}
|
@ -1 +0,0 @@
|
||||
${MYSQL_USERNAME}
|
@ -8,7 +8,7 @@ RUN apt update && \
|
||||
# Global requirements
|
||||
apt install -y language-pack-en git python-virtualenv build-essential software-properties-common curl git-core libxml2-dev libxslt1-dev python-pip libmysqlclient-dev python-apt python-dev libxmlsec1-dev libfreetype6-dev swig gcc g++ && \
|
||||
# openedx requirements
|
||||
apt install -y gettext gfortran graphviz graphviz-dev libffi-dev libfreetype6-dev libgeos-dev libjpeg8-dev liblapack-dev libpng12-dev libxml2-dev libxmlsec1-dev libxslt1-dev nodejs npm ntp pkg-config && \
|
||||
apt install -y gettext gfortran libffi-dev libfreetype6-dev libgeos-dev libjpeg8-dev liblapack-dev libpng12-dev libxml2-dev libxmlsec1-dev libxslt1-dev nodejs npm ntp pkg-config && \
|
||||
# Our requirements
|
||||
apt install -y mysql-client
|
||||
|
||||
|
18
xqueue/Dockerfile
Normal file
18
xqueue/Dockerfile
Normal file
@ -0,0 +1,18 @@
|
||||
FROM ubuntu:16.04
|
||||
|
||||
RUN apt update && \
|
||||
apt upgrade -y && \
|
||||
apt install -y language-pack-en git git-core python-pip libmysqlclient-dev
|
||||
|
||||
RUN mkdir /openedx
|
||||
RUN git clone https://github.com/edx/xqueue --branch open-release/ginkgo.master --depth 1 /openedx/xqueue
|
||||
WORKDIR /openedx/xqueue
|
||||
|
||||
RUN pip install -r pre-requirements.txt
|
||||
RUN pip install -r requirements.txt
|
||||
|
||||
ENV DJANGO_SETTINGS_MODULE xqueue.universal
|
||||
RUN ln -s /openedx/config/universal.py xqueue/universal.py
|
||||
|
||||
EXPOSE 8040
|
||||
CMD gunicorn --name xqueue --bind=0.0.0.0:8040 --max-requests=1000 xqueue.wsgi:application
|
Loading…
Reference in New Issue
Block a user