Make it possible to configure data services host/port

In the future, we want to allow users to rely on third-party services
for data storage, such as hosted MySQL and such. To do so, we need to be
able to configure the host/port of these services, which we do here.

This is to address part of #114.

tutor/scripts.py

@@ -1,20 +1,20 @@
-create_databases = """dockerize -wait tcp://mysql:3306 -timeout 20s
+create_databases = """dockerize -wait tcp://{{ MYSQL_HOST }}:{{ MYSQL_PORT }} -timeout 20s
-mysql -u root --password="{{ MYSQL_PASSWORD }}" --host "mysql" -e 'CREATE DATABASE IF NOT EXISTS {{ MYSQL_DATABASE }};'
+mysql -u root --password="{{ MYSQL_PASSWORD }}" --host "{{ MYSQL_HOST }}" -e 'CREATE DATABASE IF NOT EXISTS {{ MYSQL_DATABASE }};'
-mysql -u root --password="{{ MYSQL_PASSWORD }}" --host "mysql" -e 'GRANT ALL ON {{ MYSQL_DATABASE }}.* TO "{{ MYSQL_USERNAME }}"@"%" IDENTIFIED BY "{{ MYSQL_PASSWORD }}";'
+mysql -u root --password="{{ MYSQL_PASSWORD }}" --host "{{ MYSQL_HOST }}" -e 'GRANT ALL ON {{ MYSQL_DATABASE }}.* TO "{{ MYSQL_USERNAME }}"@"%" IDENTIFIED BY "{{ MYSQL_PASSWORD }}";'
 
 {% if ACTIVATE_NOTES %}
-mysql -u root --password="{{ MYSQL_PASSWORD }}" --host "mysql" -e 'CREATE DATABASE IF NOT EXISTS {{ NOTES_MYSQL_DATABASE }};'
+mysql -u root --password="{{ MYSQL_PASSWORD }}" --host "{{ MYSQL_HOST }}" -e 'CREATE DATABASE IF NOT EXISTS {{ NOTES_MYSQL_DATABASE }};'
-mysql -u root --password="{{ MYSQL_PASSWORD }}" --host "mysql" -e 'GRANT ALL ON {{ NOTES_MYSQL_DATABASE }}.* TO "{{ NOTES_MYSQL_USERNAME }}"@"%" IDENTIFIED BY "{{ NOTES_MYSQL_PASSWORD }}";'
+mysql -u root --password="{{ MYSQL_PASSWORD }}" --host "{{ MYSQL_HOST }}" -e 'GRANT ALL ON {{ NOTES_MYSQL_DATABASE }}.* TO "{{ NOTES_MYSQL_USERNAME }}"@"%" IDENTIFIED BY "{{ NOTES_MYSQL_PASSWORD }}";'
 {% endif %}
 
 {% if ACTIVATE_XQUEUE %}
-mysql -u root --password="{{ MYSQL_PASSWORD }}" --host "mysql" -e 'CREATE DATABASE IF NOT EXISTS {{ XQUEUE_MYSQL_DATABASE }};'
+mysql -u root --password="{{ MYSQL_PASSWORD }}" --host "{{ MYSQL_HOST }}" -e 'CREATE DATABASE IF NOT EXISTS {{ XQUEUE_MYSQL_DATABASE }};'
-mysql -u root --password="{{ MYSQL_PASSWORD }}" --host "mysql" -e 'GRANT ALL ON {{ XQUEUE_MYSQL_DATABASE }}.* TO "{{ XQUEUE_MYSQL_USERNAME }}"@"%" IDENTIFIED BY "{{ XQUEUE_MYSQL_PASSWORD }}";'
+mysql -u root --password="{{ MYSQL_PASSWORD }}" --host "{{ MYSQL_HOST }}" -e 'GRANT ALL ON {{ XQUEUE_MYSQL_DATABASE }}.* TO "{{ XQUEUE_MYSQL_USERNAME }}"@"%" IDENTIFIED BY "{{ XQUEUE_MYSQL_PASSWORD }}";'
 {% endif %}
 """
 
-migrate_lms = "dockerize -wait tcp://mysql:3306 -timeout 20s && ./manage.py lms migrate"
+migrate_lms = "dockerize -wait tcp://{{ MYSQL_HOST }}:{{ MYSQL_PORT }} -timeout 20s && ./manage.py lms migrate"
-migrate_cms = "dockerize -wait tcp://mysql:3306 -timeout 20s && ./manage.py cms migrate"
+migrate_cms = "dockerize -wait tcp://{{ MYSQL_HOST }}:{{ MYSQL_PORT }} -timeout 20s && ./manage.py cms migrate"
 migrate_forum = "bundle exec rake search:initialize && bundle exec rake search:rebuild_index"
 migrate_notes = "./manage.py migrate"
 migrate_xqueue = "./manage.py migrate"

tutor/templates/apps/notes/settings/tutor.py

@@ -6,10 +6,11 @@ ALLOWED_HOSTS = ['localhost', 'notes', 'notes.openedx', 'notes.localhost', 'note
 DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.mysql',
+        'HOST': '{{ MYSQL_HOST }}',
+        'PORT': {{ MYSQL_PORT }},
         'NAME': '{{ NOTES_MYSQL_DATABASE }}',
         'USER': '{{ NOTES_MYSQL_USERNAME }}',
         'PASSWORD': '{{ NOTES_MYSQL_PASSWORD }}',
-        'HOST': 'mysql',
     }
 }
 
@@ -19,7 +20,7 @@ CLIENT_SECRET = '{{ NOTES_OAUTH2_SECRET }}'
 HAYSTACK_CONNECTIONS = {
     'default': {
         'ENGINE': 'notesserver.highlight.ElasticsearchSearchEngine',
-        'URL': 'http://elasticsearch:9200/',
+        'URL': 'http://{{ ELASTICSEARCH_HOST }}:{{ ELASTICSEARCH_PORT }}/',
         'INDEX_NAME': 'notes',
     },
 }

tutor/templates/apps/openedx/config/cms.auth.json

@@ -13,21 +13,21 @@
     "ENGINE": "xmodule.contentstore.mongo.MongoContentStore",                       
     "DOC_STORE_CONFIG": {                                                           
       "db": "{{ MONGODB_DATABASE }}",
-      "host": "mongodb"
+      "host": "{{ MONGODB_HOST }}"
     } 
   },  
   "DOC_STORE_CONFIG": {
     "db": "{{ MONGODB_DATABASE }}",
-    "host": "mongodb"  
+    "host": "{{ MONGODB_HOST }}"  
   },
   "DATABASES": {
     "default": {
       "ENGINE": "django.db.backends.mysql",
+      "HOST": "{{ MYSQL_HOST }}",
+      "PORT": {{ MYSQL_PORT }},
       "NAME": "{{ MYSQL_DATABASE }}",
       "USER": "{{ MYSQL_USERNAME }}",
       "PASSWORD": "{{ MYSQL_PASSWORD }}",
-      "HOST": "mysql",
-      "PORT": "3306",
       "ATOMIC_REQUESTS": true
     }
   }

tutor/templates/apps/openedx/config/cms.env.json

@@ -16,14 +16,14 @@
   "CMS_BASE": "{{ CMS_HOST }}",
   "LMS_BASE": "{{ LMS_HOST }}",
   "CONTACT_EMAIL": "{{ CONTACT_EMAIL }}",
-  "CELERY_BROKER_HOSTNAME": "rabbitmq",
+  "CELERY_BROKER_HOSTNAME": "{{ RABBITMQ_HOST }}",
   "CELERY_BROKER_TRANSPORT": "amqp",
   "COMPREHENSIVE_THEME_DIRS": ["/openedx/themes"],
   "MEDIA_ROOT": "/openedx/data/uploads/",
   "STATIC_ROOT_BASE": "/openedx/staticfiles",
   "ELASTIC_SEARCH_CONFIG": [{
-    "host": "elasticsearch",
+    "host": "{{ ELASTICSEARCH_HOST }}",
-    "port": 9200
+    "port": {{ ELASTICSEARCH_PORT }}
   }],
   "EMAIL_BACKEND": "django.core.mail.backends.smtp.EmailBackend",
   "EMAIL_HOST": "smtp",
@@ -34,46 +34,46 @@
       "VERSION": "1",
       "BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
       "KEY_FUNCTION": "util.memcache.safe_key",
-      "LOCATION": "memcached:11211"
+      "LOCATION": "{{ MEMCACHED_HOST }}:{{ MEMCACHED_PORT }}"
     },
     "general": {
       "KEY_PREFIX":  "general",
       "BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
       "KEY_FUNCTION": "util.memcache.safe_key",
-      "LOCATION": "memcached:11211"
+      "LOCATION": "{{ MEMCACHED_HOST }}:{{ MEMCACHED_PORT }}"
     },
     "mongo_metadata_inheritance": {
       "KEY_PREFIX": "mongo_metadata_inheritance",
       "TIMEOUT": 300,
       "BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
       "KEY_FUNCTION": "util.memcache.safe_key",
-      "LOCATION": "memcached:11211"
+      "LOCATION": "{{ MEMCACHED_HOST }}:{{ MEMCACHED_PORT }}"
     },
     "staticfiles": {
       "KEY_PREFIX": "staticfiles_cms",
       "BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
       "KEY_FUNCTION": "util.memcache.safe_key",
-      "LOCATION": "memcached:11211"
+      "LOCATION": "{{ MEMCACHED_HOST }}:{{ MEMCACHED_PORT }}"
     },
     "configuration": {
       "KEY_PREFIX": "configuration",
       "BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
       "KEY_FUNCTION": "util.memcache.safe_key",
-      "LOCATION": "memcached:11211"
+      "LOCATION": "{{ MEMCACHED_HOST }}:{{ MEMCACHED_PORT }}"
     },
     "celery": {
       "KEY_PREFIX":  "celery",
       "TIMEOUT": "7200",
       "BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
       "KEY_FUNCTION": "util.memcache.safe_key",
-      "LOCATION": "memcached:11211"
+      "LOCATION": "{{ MEMCACHED_HOST }}:{{ MEMCACHED_PORT }}"
     },
     "course_structure_cache": {
       "KEY_PREFIX": "course_structure",
       "TIMEOUT": "7200",
       "BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
       "KEY_FUNCTION": "util.memcache.safe_key",
-      "LOCATION": "memcached:11211"
+      "LOCATION": "{{ MEMCACHED_HOST }}:{{ MEMCACHED_PORT }}"
     }
   }
 }

tutor/templates/apps/openedx/config/lms.auth.json

@@ -13,21 +13,21 @@
     "ENGINE": "xmodule.contentstore.mongo.MongoContentStore",                       
     "DOC_STORE_CONFIG": {                                                           
       "db": "{{ MONGODB_DATABASE }}",
-      "host": "mongodb"
+      "host": "{{ MONGODB_HOST }}"
     } 
   },  
   "DOC_STORE_CONFIG": {
     "db": "{{ MONGODB_DATABASE }}",
-    "host": "mongodb"  
+    "host": "{{ MONGODB_HOST }}"  
   },
   "DATABASES": {
     "default": {
       "ENGINE": "django.db.backends.mysql",
+      "HOST": "{{ MYSQL_HOST }}",
+      "PORT": {{ MYSQL_PORT }},
       "NAME": "{{ MYSQL_DATABASE }}",
       "USER": "{{ MYSQL_USERNAME }}",
       "PASSWORD": "{{ MYSQL_PASSWORD }}",
-      "HOST": "mysql",
-      "PORT": "3306",
       "ATOMIC_REQUESTS": true
     }
   }

tutor/templates/apps/openedx/config/lms.env.json

@@ -21,7 +21,7 @@
   "CMS_BASE": "{{ CMS_HOST }}",
   "LMS_BASE": "{{ LMS_HOST }}",
   "CONTACT_EMAIL": "{{ CONTACT_EMAIL }}",
-  "CELERY_BROKER_HOSTNAME": "rabbitmq",
+  "CELERY_BROKER_HOSTNAME": "{{ RABBITMQ_HOST }}",
   "CELERY_BROKER_TRANSPORT": "amqp",
   "COMMENTS_SERVICE_URL": "http://forum:4567",
   "COMMENTS_SERVICE_KEY": "forumapikey",
@@ -29,8 +29,8 @@
   "MEDIA_ROOT": "/openedx/data/uploads/",
   "STATIC_ROOT_BASE": "/openedx/staticfiles",
   "ELASTIC_SEARCH_CONFIG": [{
-    "host": "elasticsearch",
+    "host": "{{ ELASTICSEARCH_HOST }}",
-    "port": 9200
+    "port": {{ ELASTICSEARCH_PORT }}
   }],
   "EMAIL_BACKEND": "django.core.mail.backends.smtp.EmailBackend",
   "EMAIL_HOST": "smtp",
@@ -45,52 +45,52 @@
       "VERSION": "1",
       "BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
       "KEY_FUNCTION": "util.memcache.safe_key",
-      "LOCATION": "memcached:11211"
+      "LOCATION": "{{ MEMCACHED_HOST }}:{{ MEMCACHED_PORT }}"
     },
     "general": {
       "KEY_PREFIX":  "general",
       "BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
       "KEY_FUNCTION": "util.memcache.safe_key",
-      "LOCATION": "memcached:11211"
+      "LOCATION": "{{ MEMCACHED_HOST }}:{{ MEMCACHED_PORT }}"
     },
     "mongo_metadata_inheritance": {
       "KEY_PREFIX": "mongo_metadata_inheritance",
       "TIMEOUT": 300,
       "BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
       "KEY_FUNCTION": "util.memcache.safe_key",
-      "LOCATION": "memcached:11211"
+      "LOCATION": "{{ MEMCACHED_HOST }}:{{ MEMCACHED_PORT }}"
     },
     "staticfiles": {
       "KEY_PREFIX": "staticfiles_lms",
       "BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
       "KEY_FUNCTION": "util.memcache.safe_key",
-      "LOCATION": "memcached:11211"
+      "LOCATION": "{{ MEMCACHED_HOST }}:{{ MEMCACHED_PORT }}"
     },
     "configuration": {
       "KEY_PREFIX": "configuration",
       "BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
       "KEY_FUNCTION": "util.memcache.safe_key",
-      "LOCATION": "memcached:11211"
+      "LOCATION": "{{ MEMCACHED_HOST }}:{{ MEMCACHED_PORT }}"
     },
     "celery": {
       "KEY_PREFIX":  "celery",
       "TIMEOUT": "7200",
       "BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
       "KEY_FUNCTION": "util.memcache.safe_key",
-      "LOCATION": "memcached:11211"
+      "LOCATION": "{{ MEMCACHED_HOST }}:{{ MEMCACHED_PORT }}"
     },
     "course_structure_cache": {
       "KEY_PREFIX": "course_structure",
       "TIMEOUT": "7200",
       "BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
       "KEY_FUNCTION": "util.memcache.safe_key",
-      "LOCATION": "memcached:11211"
+      "LOCATION": "{{ MEMCACHED_HOST }}:{{ MEMCACHED_PORT }}"
     },
     "ora2-storage": {
       "KEY_PREFIX":  "ora2-storage",
       "BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
       "KEY_FUNCTION": "util.memcache.safe_key",
-      "LOCATION": "memcached:11211"
+      "LOCATION": "{{ MEMCACHED_HOST }}:{{ MEMCACHED_PORT }}"
     }
   }
 }

tutor/templates/apps/xqueue/settings/tutor.py

@@ -3,11 +3,11 @@ from .settings import *
 DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.mysql',
+        'HOST': '{{ MYSQL_HOST }}',
+        'PORT': {{ MYSQL_PORT }},
         'NAME': '{{ XQUEUE_MYSQL_DATABASE }}',
         'USER': '{{ XQUEUE_MYSQL_USERNAME }}',
         'PASSWORD': '{{ XQUEUE_MYSQL_PASSWORD }}',
-        'HOST': 'mysql',
-        'PORT': '3306',
     }
 }
 
@@ -17,7 +17,7 @@ LOGGING = get_logger_config(
     dev_env=True,
 )
 
-RABBIT_HOST = 'rabbitmq'
+RABBIT_HOST = '{{ RABBITMQ_HOST }}'
 RABBIT_PORT = 5672
 SECRET_KEY = '{{ XQUEUE_SECRET_KEY }}'
 

tutor/templates/config-defaults.yml

@@ -1,4 +1,7 @@
 ---
+ANDROID_RELEASE_STORE_PASSWORD: "android store password"
+ANDROID_RELEASE_KEY_PASSWORD: "android release key password"
+ANDROID_RELEASE_KEY_ALIAS: "android release key alias"
 DOCKER_IMAGE_OPENEDX: "regis/openedx:hawthorn"
 DOCKER_IMAGE_ANDROID: "regis/openedx-android:hawthorn"
 DOCKER_IMAGE_FORUM: "regis/openedx-forum:hawthorn"
@@ -12,16 +15,21 @@ DOCKER_IMAGE_NGINX: "nginx:1.13"
 DOCKER_IMAGE_RABBITMQ: "rabbitmq:3.6.10"
 DOCKER_IMAGE_NAMSHI: "namshi/smtp:latest"
 DOCKER_REGISTRY: ""
+ELASTICSEARCH_HOST: "elasticsearch"
+ELASTICSEARCH_PORT: 9200
+MEMCACHED_HOST: "memcached"
+MEMCACHED_PORT: 11211
+MONGODB_HOST: "mongodb"
+MONGODB_DATABASE: "openedx"
+MYSQL_HOST: "mysql"
+MYSQL_PORT: 3306
 MYSQL_DATABASE: "openedx"
 MYSQL_USERNAME: "openedx"
-MONGODB_DATABASE: "openedx"
+NGINX_HTTP_PORT: 80
+NGINX_HTTPS_PORT: 443
 NOTES_MYSQL_DATABASE: "notes"
 NOTES_MYSQL_USERNAME: "notes"
+RABBITMQ_HOST: "rabbitmq"
 XQUEUE_AUTH_USERNAME: "lms"
 XQUEUE_MYSQL_DATABASE: "xqueue"
 XQUEUE_MYSQL_USERNAME: "xqueue"
-NGINX_HTTP_PORT: 80
-NGINX_HTTPS_PORT: 443
-ANDROID_RELEASE_STORE_PASSWORD: "android store password"
-ANDROID_RELEASE_KEY_PASSWORD: "android release key password"
-ANDROID_RELEASE_KEY_ALIAS: "android release key alias"

tutor/templates/k8s/deployments.yml

@@ -66,6 +66,11 @@ spec:
           ports:
             - containerPort: 4567
           imagePullPolicy: Always
+          env:
+            - name: SEARCH_SERVER
+              value: "http://{{ ELASTICSEARCH_HOST }}:{{ ELASTICSEARCH_PORT }}"
+            - name: MONGOHQ_URL
+              value: "mongodb://{{ MONGODB_HOST }}/cs_comments_service"
 
 ---
 apiVersion: apps/v1

tutor/templates/local/docker-compose.yml

@@ -75,6 +75,9 @@ services:
 
   forum:
     image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_FORUM }}
+    environment:
+      SEARCH_SERVER: "http://{{ ELASTICSEARCH_HOST }}:{{ ELASTICSEARCH_PORT }}"
+      MONGOHQ_URL: "mongodb://{{ MONGODB_HOST }}/cs_comments_service"
     restart: unless-stopped
     depends_on:
       - elasticsearch