diff --git a/.gitignore b/.gitignore index d6a3789..c78e740 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ .*.swp config/openedx/*.json +config/openedx/*.sh +config/mysql/*.env config/nginx/*.conf config/android/*.yaml -mysql/config/database -mysql/config/username -mysql/config/password +config/xqueue/*.py diff --git a/Makefile b/Makefile index 2c65570..efc826f 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ endif DOCKER_COMPOSE_RUN_LMS = $(DOCKER_COMPOSE_RUN_OPENEDX) -p 8000:8000 lms DOCKER_COMPOSE_RUN_CMS = $(DOCKER_COMPOSE_RUN_OPENEDX) -p 8001:8001 cms -all: configure update migrate migrate-forum assets daemon +all: configure update migrate assets daemon ##################### Bootstrapping @@ -21,7 +21,10 @@ configure: update: docker-compose pull -migrate: +provision: + $(DOCKER_COMPOSE_RUN_OPENEDX) lms bash /openedx/config/provision.sh + +migrate-openedx: $(DOCKER_COMPOSE_RUN_OPENEDX) lms bash -c "wait-for-greenlight.sh && ./manage.py lms migrate" $(DOCKER_COMPOSE_RUN_OPENEDX) cms bash -c "wait-for-greenlight.sh && ./manage.py cms migrate" @@ -29,6 +32,11 @@ migrate-forum: $(DOCKER_COMPOSE_RUN) forum bash -c "bundle exec rake search:initialize && \ bundle exec rake search:rebuild_index" +migrate-xqueue: + $(DOCKER_COMPOSE_RUN) xqueue bash -c "./manage.py migrate" + +migrate: provision migrate-openedx migrate-forum migrate-xqueue + assets: $(DOCKER_COMPOSE_RUN_OPENEDX) lms paver update_assets lms --settings=$(EDX_PLATFORM_SETTINGS) $(DOCKER_COMPOSE_RUN_OPENEDX) cms paver update_assets cms --settings=$(EDX_PLATFORM_SETTINGS) @@ -90,11 +98,14 @@ build: # We need to build with docker, as long as docker-compose cannot push to dockerhub docker build -t regis/openedx:latest -t regis/openedx:ginkgo openedx/ docker build -t regis/openedx-forum:latest -t regis/openedx-forum:ginkgo forum/ + docker build -t regis/openedx-xqueue:latest -t regis/openedx-xqueue:ginkgo xqueue/ push: docker push regis/openedx:ginkgo docker push regis/openedx:latest docker push regis/openedx-forum:ginkgo docker push regis/openedx-forum:latest + docker push regis/openedx-xqueue:ginkgo + docker push regis/openedx-xqueue:latest dockerhub: build push diff --git a/config/mysql/templates/auth.env.templ b/config/mysql/templates/auth.env.templ new file mode 100644 index 0000000..8faa532 --- /dev/null +++ b/config/mysql/templates/auth.env.templ @@ -0,0 +1 @@ +MYSQL_ROOT_PASSWORD: ${MYSQL_PASSWORD} diff --git a/config/openedx/templates/cms.auth.json.templ b/config/openedx/templates/cms.auth.json.templ index 6505288..c043603 100644 --- a/config/openedx/templates/cms.auth.json.templ +++ b/config/openedx/templates/cms.auth.json.templ @@ -3,12 +3,11 @@ "AWS_ACCESS_KEY_ID": "", "AWS_SECRET_ACCESS_KEY": "", "XQUEUE_INTERFACE": { - "basic_auth": ["edx", "edx"], "django_auth": { - "username": "lms", - "password": "password" + "username": "${XQUEUE_AUTH_USERNAME}", + "password": "${XQUEUE_AUTH_PASSWORD}" }, - "url": "http://localhost:18040" + "url": "http://xqueue:8040" }, "CONTENTSTORE": { "ENGINE": "xmodule.contentstore.mongo.MongoContentStore", diff --git a/config/openedx/templates/lms.auth.json.templ b/config/openedx/templates/lms.auth.json.templ index 108b66b..9bfb456 100644 --- a/config/openedx/templates/lms.auth.json.templ +++ b/config/openedx/templates/lms.auth.json.templ @@ -3,12 +3,11 @@ "AWS_ACCESS_KEY_ID": "", "AWS_SECRET_ACCESS_KEY": "", "XQUEUE_INTERFACE": { - "basic_auth": ["edx", "edx"], "django_auth": { - "username": "lms", - "password": "password" + "username": "${XQUEUE_AUTH_USERNAME}", + "password": "${XQUEUE_AUTH_PASSWORD}" }, - "url": "http://localhost:18040" + "url": "http://xqueue:8040" }, "CONTENTSTORE": { "ENGINE": "xmodule.contentstore.mongo.MongoContentStore", diff --git a/config/openedx/templates/provision.sh.templ b/config/openedx/templates/provision.sh.templ new file mode 100755 index 0000000..453c008 --- /dev/null +++ b/config/openedx/templates/provision.sh.templ @@ -0,0 +1,5 @@ +mysql -u root --password="${MYSQL_PASSWORD}" --host "mysql" -e 'CREATE DATABASE IF NOT EXISTS ${MYSQL_DATABASE};' +mysql -u root --password="${MYSQL_PASSWORD}" --host "mysql" -e 'GRANT ALL ON ${MYSQL_DATABASE}.* TO "${MYSQL_USERNAME}"@"%" IDENTIFIED BY "${MYSQL_PASSWORD}";' + +mysql -u root --password="${MYSQL_PASSWORD}" --host "mysql" -e 'CREATE DATABASE IF NOT EXISTS ${XQUEUE_MYSQL_DATABASE};' +mysql -u root --password="${MYSQL_PASSWORD}" --host "mysql" -e 'GRANT ALL ON ${XQUEUE_MYSQL_DATABASE}.* TO "${XQUEUE_MYSQL_USERNAME}"@"%" IDENTIFIED BY "${XQUEUE_MYSQL_PASSWORD}";' diff --git a/config/xqueue/templates/universal.py.templ b/config/xqueue/templates/universal.py.templ new file mode 100644 index 0000000..2a8576b --- /dev/null +++ b/config/xqueue/templates/universal.py.templ @@ -0,0 +1,25 @@ +from .settings import * + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.mysql', + 'NAME': '${XQUEUE_MYSQL_DATABASE}', + 'USER': '${XQUEUE_MYSQL_USERNAME}', + 'PASSWORD': '${XQUEUE_MYSQL_PASSWORD}', + 'HOST': 'mysql', + } +} + +LOGGING = get_logger_config( + log_dir="/openedx/data/", + logging_env="universal", + dev_env=True, +) + +RABBIT_HOST = 'rabbitmq' +RABBIT_PORT = 5672 +SECRET_KEY = '${XQUEUE_SECRET_KEY}' + +XQUEUE_USERS = { + '${XQUEUE_AUTH_USERNAME}': '${XQUEUE_AUTH_PASSWORD}' +} diff --git a/configure b/configure index 4bd886d..fb60275 100755 --- a/configure +++ b/configure @@ -25,10 +25,12 @@ class Configurator(object): def as_dict(self): return dict(self.__values) - def add(self, name, question, default=""): + def add(self, name, question="", default=""): default = self.__default_overrides.get(name, default) - message = question + " (default: \"{}\"): ".format(default) - value = self.ask(message, default) + value = default + if question: + message = question + " (default: \"{}\"): ".format(default) + value = self.ask(message, default) self.set(name, value) return self @@ -116,19 +118,30 @@ def main(): 'LMS_HOST', "Your website domain name for students (LMS). Set 'localhost' for local testing.", 'www.myopenedx.com' ).add( 'CMS_HOST', "Your website domain name for teachers (CMS). Set 'studio.localhost' for local testing.", 'studio.myopenedx.com' - ).add( - 'SECRET_KEY', "Secret key -- if you don't know what this is, you can safely accept the default", - "".join([random.choice(string.ascii_letters + string.digits) for _ in range(24)]) ).add( 'PLATFORM_NAME', "Platform name/title", "My Open edX" ).add( - 'MYSQL_DATABASE', "MySQL database name", 'openedx' + 'SECRET_KEY', "", random_string(24) ).add( - 'MYSQL_USERNAME', "MySQL database username", 'openedx' + 'MYSQL_DATABASE', "", 'openedx' ).add( - 'MYSQL_PASSWORD', "MySQL database password", 'password' + 'MYSQL_USERNAME', "", 'openedx' ).add( - 'MONGODB_DATABASE', "MongoDb database name", 'openedx' + 'MYSQL_PASSWORD', "", random_string(8), + ).add( + 'MONGODB_DATABASE', "", 'openedx' + ).add( + 'XQUEUE_AUTH_USERNAME', "", 'lms' + ).add( + 'XQUEUE_AUTH_PASSWORD', "", random_string(8), + ).add( + 'XQUEUE_MYSQL_DATABASE', "", 'xqueue', + ).add( + 'XQUEUE_MYSQL_USERNAME', "", 'xqueue', + ).add( + 'XQUEUE_MYSQL_PASSWORD', "", random_string(8), + ).add( + 'XQUEUE_SECRET_KEY', "", random_string(24), ) # Save values @@ -157,21 +170,23 @@ def main(): os.path.join('config', 'openedx', 'cms.auth.json'), **configurator.as_dict() ) + substitute( + os.path.join('config', 'openedx', 'templates', 'provision.sh.templ'), + os.path.join('config', 'openedx', 'provision.sh'), + **configurator.as_dict() + ) + + # Xqueue + substitute( + os.path.join('config', 'xqueue', 'templates', 'universal.py.templ'), + os.path.join('config', 'xqueue', 'universal.py'), + **configurator.as_dict() + ) # MySQL substitute( - os.path.join('mysql', 'config', 'templates', 'username.templ'), - os.path.join('mysql', 'config', 'username'), - **configurator.as_dict() - ) - substitute( - os.path.join('mysql', 'config', 'templates', 'password.templ'), - os.path.join('mysql', 'config', 'password'), - **configurator.as_dict() - ) - substitute( - os.path.join('mysql', 'config', 'templates', 'database.templ'), - os.path.join('mysql', 'config', 'database'), + os.path.join('config', 'mysql', 'templates', 'auth.env.templ'), + os.path.join('config', 'mysql', 'auth.env'), **configurator.as_dict() ) @@ -199,5 +214,8 @@ def main(): print("\nConfiguration files were successfuly generated. You may now run the app containers.") +def random_string(length): + return "".join([random.choice(string.ascii_letters + string.digits) for _ in range(length)]) + if __name__ == '__main__': main() diff --git a/data/.gitignore b/data/.gitignore index 74f2870..ec8b969 100644 --- a/data/.gitignore +++ b/data/.gitignore @@ -7,3 +7,4 @@ elasticsearch/ mysql/ mongodb/ rabbitmq/ +xqueue/ diff --git a/docker-compose.yml b/docker-compose.yml index a57b500..c07c649 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,13 +21,7 @@ services: restart: unless-stopped volumes: - ./data/mysql:/var/lib/mysql - - ./mysql/config/:/etc/mysql/conf.d/openedx - environment: - # Load values from files generated by configurator - MYSQL_DATABASE_FILE: /etc/mysql/conf.d/openedx/database - MYSQL_USER_FILE: /etc/mysql/conf.d/openedx/username - MYSQL_PASSWORD_FILE: /etc/mysql/conf.d/openedx/password - MYSQL_ROOT_PASSWORD_FILE: /etc/mysql/conf.d/openedx/password + env_file: ./config/mysql/auth.env elasticsearch: image: elasticsearch:1.5.2 @@ -150,3 +144,28 @@ services: - ./data/cms_worker:/openedx/data depends_on: - cms + + ############# Xqueue: external grading of Open edX problems + xqueue: + image: regis/openedx-xqueue:ginkgo + build: + context: ./xqueue + volumes: + - ./config/xqueue:/openedx/config + - ./data/xqueue:/openedx/data + restart: unless-stopped + depends_on: + - mysql + + xqueue_consumer: + image: regis/openedx-xqueue:ginkgo + build: + context: ./xqueue + volumes: + - ./config/xqueue:/openedx/config + - ./data/xqueue:/openedx/data + restart: unless-stopped + # Run 12 workers per queue + command: ./manage.py run_consumer 12 + depends_on: + - mysql diff --git a/mysql/config/templates/database.templ b/mysql/config/templates/database.templ deleted file mode 100644 index 4e54088..0000000 --- a/mysql/config/templates/database.templ +++ /dev/null @@ -1 +0,0 @@ -${MYSQL_DATABASE} diff --git a/mysql/config/templates/password.templ b/mysql/config/templates/password.templ deleted file mode 100644 index ee16543..0000000 --- a/mysql/config/templates/password.templ +++ /dev/null @@ -1 +0,0 @@ -${MYSQL_PASSWORD} diff --git a/mysql/config/templates/username.templ b/mysql/config/templates/username.templ deleted file mode 100644 index 24e45d6..0000000 --- a/mysql/config/templates/username.templ +++ /dev/null @@ -1 +0,0 @@ -${MYSQL_USERNAME} diff --git a/openedx/Dockerfile b/openedx/Dockerfile index b78bf08..1f759c4 100644 --- a/openedx/Dockerfile +++ b/openedx/Dockerfile @@ -8,7 +8,7 @@ RUN apt update && \ # Global requirements apt install -y language-pack-en git python-virtualenv build-essential software-properties-common curl git-core libxml2-dev libxslt1-dev python-pip libmysqlclient-dev python-apt python-dev libxmlsec1-dev libfreetype6-dev swig gcc g++ && \ # openedx requirements - apt install -y gettext gfortran graphviz graphviz-dev libffi-dev libfreetype6-dev libgeos-dev libjpeg8-dev liblapack-dev libpng12-dev libxml2-dev libxmlsec1-dev libxslt1-dev nodejs npm ntp pkg-config && \ + apt install -y gettext gfortran libffi-dev libfreetype6-dev libgeos-dev libjpeg8-dev liblapack-dev libpng12-dev libxml2-dev libxmlsec1-dev libxslt1-dev nodejs npm ntp pkg-config && \ # Our requirements apt install -y mysql-client diff --git a/xqueue/Dockerfile b/xqueue/Dockerfile new file mode 100644 index 0000000..a9ef759 --- /dev/null +++ b/xqueue/Dockerfile @@ -0,0 +1,18 @@ +FROM ubuntu:16.04 + +RUN apt update && \ + apt upgrade -y && \ + apt install -y language-pack-en git git-core python-pip libmysqlclient-dev + +RUN mkdir /openedx +RUN git clone https://github.com/edx/xqueue --branch open-release/ginkgo.master --depth 1 /openedx/xqueue +WORKDIR /openedx/xqueue + +RUN pip install -r pre-requirements.txt +RUN pip install -r requirements.txt + +ENV DJANGO_SETTINGS_MODULE xqueue.universal +RUN ln -s /openedx/config/universal.py xqueue/universal.py + +EXPOSE 8040 +CMD gunicorn --name xqueue --bind=0.0.0.0:8040 --max-requests=1000 xqueue.wsgi:application