Add xqueue services

Xqueue containers consist of two services: a gunicorn service, that receives requests from LMS/CMS, and a worker service. I guess the worker service receives orders from the gunicorn service, through rabbitmq. (but I'm less than certain about that). While adding xqueue containers, we refactored the way mysql databases are created, and how the root password is loaded. Also, we silenced some options from the configure script.
2018-06-01 21:29:53 -04:00 · 2018-06-01 21:29:53 -04:00 · 43a97e5fe3
parent 4dc20e3d2b
commit 43a97e5fe3
15 changed files with 139 additions and 46 deletions
--- a/.gitignore
+++ b/.gitignore
@ -1,7 +1,7 @@
 .*.swp
 config/openedx/*.json
+config/openedx/*.sh
+config/mysql/*.env
 config/nginx/*.conf
 config/android/*.yaml
-mysql/config/database
-mysql/config/username
-mysql/config/password
+config/xqueue/*.py
--- a/15
+++ b/15
@ -11,7 +11,7 @@ endif
 DOCKER_COMPOSE_RUN_LMS = $(DOCKER_COMPOSE_RUN_OPENEDX) -p 8000:8000 lms
 DOCKER_COMPOSE_RUN_CMS = $(DOCKER_COMPOSE_RUN_OPENEDX) -p 8001:8001 cms

-all: configure update migrate migrate-forum assets daemon
+all: configure update migrate assets daemon

 ##################### Bootstrapping

@ -21,7 +21,10 @@ configure:
 update:
 	docker-compose pull

-migrate:
+provision:
+	$(DOCKER_COMPOSE_RUN_OPENEDX) lms bash /openedx/config/provision.sh
+
+migrate-openedx:
 	$(DOCKER_COMPOSE_RUN_OPENEDX) lms bash -c "wait-for-greenlight.sh && ./manage.py lms migrate"
 	$(DOCKER_COMPOSE_RUN_OPENEDX) cms bash -c "wait-for-greenlight.sh && ./manage.py cms migrate"

@ -29,6 +32,11 @@ migrate-forum:
 	$(DOCKER_COMPOSE_RUN) forum bash -c "bundle exec rake search:initialize && \
 		bundle exec rake search:rebuild_index"

+migrate-xqueue:
+	$(DOCKER_COMPOSE_RUN) xqueue bash -c "./manage.py migrate"
+
+migrate: provision migrate-openedx migrate-forum migrate-xqueue
+
 assets:
 	$(DOCKER_COMPOSE_RUN_OPENEDX) lms paver update_assets lms --settings=$(EDX_PLATFORM_SETTINGS)
 	$(DOCKER_COMPOSE_RUN_OPENEDX) cms paver update_assets cms --settings=$(EDX_PLATFORM_SETTINGS)
@ -90,11 +98,14 @@ build:
 	# We need to build with docker, as long as docker-compose cannot push to dockerhub
 	docker build -t regis/openedx:latest -t regis/openedx:ginkgo openedx/
 	docker build -t regis/openedx-forum:latest -t regis/openedx-forum:ginkgo forum/
+	docker build -t regis/openedx-xqueue:latest -t regis/openedx-xqueue:ginkgo xqueue/

 push:
 	docker push regis/openedx:ginkgo
 	docker push regis/openedx:latest
 	docker push regis/openedx-forum:ginkgo
 	docker push regis/openedx-forum:latest
+	docker push regis/openedx-xqueue:ginkgo
+	docker push regis/openedx-xqueue:latest

 dockerhub: build push
--- a/config/mysql/templates/auth.env.templ
+++ b/config/mysql/templates/auth.env.templ
@ -0,0 +1 @@
+MYSQL_ROOT_PASSWORD: ${MYSQL_PASSWORD}
--- a/config/openedx/templates/cms.auth.json.templ
+++ b/config/openedx/templates/cms.auth.json.templ
@ -3,12 +3,11 @@
  "AWS_ACCESS_KEY_ID": "",
  "AWS_SECRET_ACCESS_KEY": "",
  "XQUEUE_INTERFACE": {
-    "basic_auth": ["edx", "edx"],
    "django_auth": {
-      "username": "lms",
-      "password": "password"
+      "username": "${XQUEUE_AUTH_USERNAME}",
+      "password": "${XQUEUE_AUTH_PASSWORD}"
    },
-    "url": "http://localhost:18040"
+    "url": "http://xqueue:8040"
  },
  "CONTENTSTORE": { 
    "ENGINE": "xmodule.contentstore.mongo.MongoContentStore",                       
--- a/config/openedx/templates/lms.auth.json.templ
+++ b/config/openedx/templates/lms.auth.json.templ
@ -3,12 +3,11 @@
  "AWS_ACCESS_KEY_ID": "",
  "AWS_SECRET_ACCESS_KEY": "",
  "XQUEUE_INTERFACE": {
-    "basic_auth": ["edx", "edx"],
    "django_auth": {
-      "username": "lms",
-      "password": "password"
+      "username": "${XQUEUE_AUTH_USERNAME}",
+      "password": "${XQUEUE_AUTH_PASSWORD}"
    },
-    "url": "http://localhost:18040"
+    "url": "http://xqueue:8040"
  },
  "CONTENTSTORE": { 
    "ENGINE": "xmodule.contentstore.mongo.MongoContentStore",                       
--- a/config/openedx/templates/provision.sh.templ
+++ b/config/openedx/templates/provision.sh.templ
@ -0,0 +1,5 @@
+mysql -u root --password="${MYSQL_PASSWORD}" --host "mysql" -e 'CREATE DATABASE IF NOT EXISTS ${MYSQL_DATABASE};'
+mysql -u root --password="${MYSQL_PASSWORD}" --host "mysql" -e 'GRANT ALL ON ${MYSQL_DATABASE}.* TO "${MYSQL_USERNAME}"@"%" IDENTIFIED BY "${MYSQL_PASSWORD}";'
+
+mysql -u root --password="${MYSQL_PASSWORD}" --host "mysql" -e 'CREATE DATABASE IF NOT EXISTS ${XQUEUE_MYSQL_DATABASE};'
+mysql -u root --password="${MYSQL_PASSWORD}" --host "mysql" -e 'GRANT ALL ON ${XQUEUE_MYSQL_DATABASE}.* TO "${XQUEUE_MYSQL_USERNAME}"@"%" IDENTIFIED BY "${XQUEUE_MYSQL_PASSWORD}";'
--- a/config/xqueue/templates/universal.py.templ
+++ b/config/xqueue/templates/universal.py.templ
@ -0,0 +1,25 @@
+from .settings import *
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.mysql',
+        'NAME': '${XQUEUE_MYSQL_DATABASE}',
+        'USER': '${XQUEUE_MYSQL_USERNAME}',
+        'PASSWORD': '${XQUEUE_MYSQL_PASSWORD}',
+        'HOST': 'mysql',
+    }
+}
+
+LOGGING = get_logger_config(
+    log_dir="/openedx/data/",
+    logging_env="universal",
+    dev_env=True,
+)
+
+RABBIT_HOST = 'rabbitmq'
+RABBIT_PORT = 5672
+SECRET_KEY = '${XQUEUE_SECRET_KEY}'
+
+XQUEUE_USERS = {
+    '${XQUEUE_AUTH_USERNAME}': '${XQUEUE_AUTH_PASSWORD}'
+}
--- a/62
+++ b/62
@ -25,10 +25,12 @@ class Configurator(object):
    def as_dict(self):
        return dict(self.__values)

-    def add(self, name, question, default=""):
+    def add(self, name, question="", default=""):
        default = self.__default_overrides.get(name, default)
-        message = question + " (default: \"{}\"): ".format(default)
-        value = self.ask(message, default)
+        value = default
+        if question:
+            message = question + " (default: \"{}\"): ".format(default)
+            value = self.ask(message, default)
        self.set(name, value)

        return self
@ -116,19 +118,30 @@ def main():
        'LMS_HOST', "Your website domain name for students (LMS). Set 'localhost' for local testing.", 'www.myopenedx.com'
    ).add(
        'CMS_HOST', "Your website domain name for teachers (CMS). Set 'studio.localhost' for local testing.", 'studio.myopenedx.com'
-    ).add(
-        'SECRET_KEY', "Secret key -- if you don't know what this is, you can safely accept the default",
-        "".join([random.choice(string.ascii_letters + string.digits) for _ in range(24)])
    ).add(
        'PLATFORM_NAME', "Platform name/title", "My Open edX"
    ).add(
-        'MYSQL_DATABASE', "MySQL database name", 'openedx'
+        'SECRET_KEY', "", random_string(24)
    ).add(
-        'MYSQL_USERNAME', "MySQL database username", 'openedx'
+        'MYSQL_DATABASE', "", 'openedx'
    ).add(
-        'MYSQL_PASSWORD', "MySQL database password", 'password'
+        'MYSQL_USERNAME', "", 'openedx'
    ).add(
-        'MONGODB_DATABASE', "MongoDb database name", 'openedx'
+        'MYSQL_PASSWORD', "", random_string(8),
+    ).add(
+        'MONGODB_DATABASE', "", 'openedx'
+    ).add(
+        'XQUEUE_AUTH_USERNAME', "", 'lms'
+    ).add(
+        'XQUEUE_AUTH_PASSWORD', "", random_string(8),
+    ).add(
+        'XQUEUE_MYSQL_DATABASE', "", 'xqueue',
+    ).add(
+        'XQUEUE_MYSQL_USERNAME', "", 'xqueue',
+    ).add(
+        'XQUEUE_MYSQL_PASSWORD', "", random_string(8),
+    ).add(
+        'XQUEUE_SECRET_KEY', "", random_string(24),
    )

    # Save values
@ -157,21 +170,23 @@ def main():
        os.path.join('config', 'openedx', 'cms.auth.json'),
        **configurator.as_dict()
    )
+    substitute(
+        os.path.join('config', 'openedx', 'templates', 'provision.sh.templ'),
+        os.path.join('config', 'openedx', 'provision.sh'),
+        **configurator.as_dict()
+    )
+
+    # Xqueue
+    substitute(
+        os.path.join('config', 'xqueue', 'templates', 'universal.py.templ'),
+        os.path.join('config', 'xqueue', 'universal.py'),
+        **configurator.as_dict()
+    )

    # MySQL
    substitute(
-        os.path.join('mysql', 'config', 'templates', 'username.templ'),
-        os.path.join('mysql', 'config', 'username'),
-        **configurator.as_dict()
-    )
-    substitute(
-        os.path.join('mysql', 'config', 'templates', 'password.templ'),
-        os.path.join('mysql', 'config', 'password'),
-        **configurator.as_dict()
-    )
-    substitute(
-        os.path.join('mysql', 'config', 'templates', 'database.templ'),
-        os.path.join('mysql', 'config', 'database'),
+        os.path.join('config', 'mysql', 'templates', 'auth.env.templ'),
+        os.path.join('config', 'mysql', 'auth.env'),
        **configurator.as_dict()
    )

@ -199,5 +214,8 @@ def main():
    print("\nConfiguration files were successfuly generated. You may now run the app containers.")


+def random_string(length):
+    return "".join([random.choice(string.ascii_letters + string.digits) for _ in range(length)])
+
 if __name__ == '__main__':
    main()
--- a/data/.gitignore
+++ b/data/.gitignore
@ -7,3 +7,4 @@ elasticsearch/
 mysql/
 mongodb/
 rabbitmq/
+xqueue/
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -21,13 +21,7 @@ services:
    restart: unless-stopped
    volumes:
      - ./data/mysql:/var/lib/mysql
-      - ./mysql/config/:/etc/mysql/conf.d/openedx
-    environment:
-      # Load values from files generated by configurator
-      MYSQL_DATABASE_FILE: /etc/mysql/conf.d/openedx/database
-      MYSQL_USER_FILE: /etc/mysql/conf.d/openedx/username
-      MYSQL_PASSWORD_FILE: /etc/mysql/conf.d/openedx/password
-      MYSQL_ROOT_PASSWORD_FILE: /etc/mysql/conf.d/openedx/password
+    env_file: ./config/mysql/auth.env

  elasticsearch:
    image: elasticsearch:1.5.2
@ -150,3 +144,28 @@ services:
      - ./data/cms_worker:/openedx/data
    depends_on:
      - cms
+
+  ############# Xqueue: external grading of Open edX problems
+  xqueue:
+    image: regis/openedx-xqueue:ginkgo
+    build:
+      context: ./xqueue
+    volumes:
+      - ./config/xqueue:/openedx/config
+      - ./data/xqueue:/openedx/data
+    restart: unless-stopped
+    depends_on:
+      - mysql
+
+  xqueue_consumer:
+    image: regis/openedx-xqueue:ginkgo
+    build:
+      context: ./xqueue
+    volumes:
+      - ./config/xqueue:/openedx/config
+      - ./data/xqueue:/openedx/data
+    restart: unless-stopped
+    # Run 12 workers per queue
+    command: ./manage.py run_consumer 12
+    depends_on:
+      - mysql
--- a/mysql/config/templates/database.templ
+++ b/mysql/config/templates/database.templ
@ -1 +0,0 @@
-${MYSQL_DATABASE}
--- a/mysql/config/templates/password.templ
+++ b/mysql/config/templates/password.templ
@ -1 +0,0 @@
-${MYSQL_PASSWORD}
--- a/mysql/config/templates/username.templ
+++ b/mysql/config/templates/username.templ
@ -1 +0,0 @@
-${MYSQL_USERNAME}
--- a/openedx/Dockerfile
+++ b/openedx/Dockerfile
@ -8,7 +8,7 @@ RUN apt update && \
  # Global requirements
  apt install -y language-pack-en git python-virtualenv build-essential software-properties-common curl git-core libxml2-dev libxslt1-dev python-pip libmysqlclient-dev python-apt python-dev libxmlsec1-dev libfreetype6-dev swig gcc g++ && \
  # openedx requirements
-  apt install -y gettext gfortran graphviz graphviz-dev libffi-dev libfreetype6-dev libgeos-dev libjpeg8-dev liblapack-dev libpng12-dev libxml2-dev libxmlsec1-dev libxslt1-dev nodejs npm ntp pkg-config && \
+  apt install -y gettext gfortran libffi-dev libfreetype6-dev libgeos-dev libjpeg8-dev liblapack-dev libpng12-dev libxml2-dev libxmlsec1-dev libxslt1-dev nodejs npm ntp pkg-config && \
  # Our requirements
  apt install -y mysql-client

--- a/xqueue/Dockerfile
+++ b/xqueue/Dockerfile
@ -0,0 +1,18 @@
+FROM ubuntu:16.04
+
+RUN apt update && \
+  apt upgrade -y && \
+  apt install -y language-pack-en git git-core python-pip libmysqlclient-dev
+
+RUN mkdir /openedx
+RUN git clone https://github.com/edx/xqueue --branch open-release/ginkgo.master --depth 1 /openedx/xqueue 
+WORKDIR /openedx/xqueue
+
+RUN pip install -r pre-requirements.txt
+RUN pip install -r requirements.txt
+
+ENV DJANGO_SETTINGS_MODULE xqueue.universal
+RUN ln -s /openedx/config/universal.py xqueue/universal.py
+
+EXPOSE 8040
+CMD gunicorn --name xqueue --bind=0.0.0.0:8040 --max-requests=1000 xqueue.wsgi:application