Better mysql initialization

For initializing the mysql database, we now run the mysql client from
the mysql container. That means that the mysql client is no longer
required in the openedx container, as previously suggested in PR #209.

CHANGELOG.md

@@ -6,6 +6,7 @@
 - [Bugfix] Fix installing a locally cloned requirement repository
 - [Improvement] Add `--no-cache` option to `images build`
 - [Improvement] Make it possible to configure the notes service hostname
+- [Improvement] Better, more robust MySQL initialization
 
 ## 3.3.10 (2019-05-15)
 

tutor/local.py

@@ -1,7 +1,4 @@
-import os
-import subprocess
 from textwrap import indent
-from time import sleep
 
 import click
 
@@ -148,42 +145,9 @@ def execute(root, service, command, args):
 @click.command(help="Create databases and run database migrations")
 @opts.root
 def databases(root):
-    init_mysql(root)
     scripts.migrate(root, run_sh)
 
 
-def init_mysql(root):
-    config = tutor_config.load(root)
-    if not config["ACTIVATE_MYSQL"]:
-        return
-    mysql_data_path = tutor_env.data_path(root, "mysql", "mysql")
-    if os.path.exists(mysql_data_path):
-        return
-    click.echo(fmt.info("Initializing MySQL database..."))
-    docker_compose(root, config, "up", "-d", "mysql")
-    while True:
-        click.echo(fmt.info("    waiting for mysql initialization"))
-        # TODO this is duplicate code with the docker_compose function. We
-        # should rely on a dedicated function in utils module.
-        mysql_logs = subprocess.check_output(
-            [
-                "docker-compose",
-                "-f",
-                tutor_env.pathjoin(root, "local", "docker-compose.yml"),
-                "--project-name",
-                config["LOCAL_PROJECT_NAME"],
-                "logs",
-                "mysql",
-            ]
-        )
-        # pylint: disable=unsupported-membership-test
-        if b"MySQL init process done. Ready for start up." in mysql_logs:
-            click.echo(fmt.info("MySQL database initialized"))
-            docker_compose(root, config, "stop", "mysql")
-            return
-        sleep(4)
-
-
 @click.group(help="Manage https certificates")
 def https():
     pass

tutor/scripts.py

@@ -9,8 +9,7 @@ def migrate(root, run_func):
     config = tutor_config.load(root)
 
     click.echo(fmt.info("Creating all databases..."))
-    # Note: run this only when lms is activated?
+    run_template(root, config, "mysql-client", "create_databases.sh", run_func)
-    run_template(root, config, "lms", "create_databases.sh", run_func)
 
     if config["ACTIVATE_LMS"]:
         click.echo(fmt.info("Running lms migrations..."))

tutor/templates/build/openedx/Dockerfile

@@ -9,8 +9,6 @@ RUN apt update && \
     apt install -y language-pack-en git python-virtualenv build-essential software-properties-common curl git-core libxml2-dev libxslt1-dev python-virtualenv libmysqlclient-dev python-apt python-dev libxmlsec1-dev libfreetype6-dev swig gcc g++ \
     # openedx requirements
     gettext gfortran graphviz graphviz-dev libffi-dev libfreetype6-dev libgeos-dev libjpeg8-dev liblapack-dev libpng12-dev libsqlite3-dev libxml2-dev libxmlsec1-dev libxslt1-dev lynx nodejs npm ntp pkg-config \
-    # Our requirements
-    mysql-client \
     && rm -rf /var/lib/apt/lists/*
 
 # Dockerize will be useful to wait for mysql DB availability

tutor/templates/local/docker-compose.yml

@@ -28,6 +28,12 @@ services:
       - ../../data/mysql:/var/lib/mysql
     env_file: ../apps/mysql/auth.env
   {% endif %}
+  mysql-client:
+    image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_MYSQL }}
+    command: echo "mysql client ready"
+    restart: "no"
+    {% if ACTIVATE_MYSQL %}depends_on:
+      - mysql{% endif %}
 
   {% if ACTIVATE_ELASTICSEARCH %}
   elasticsearch:

tutor/templates/scripts/create_databases.sh

@@ -1,13 +1,28 @@
-dockerize -wait tcp://{{ MYSQL_HOST }}:{{ MYSQL_PORT }} -timeout 20s
+echo "Initializing MySQL..."
-mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" -e 'CREATE DATABASE IF NOT EXISTS {{ OPENEDX_MYSQL_DATABASE }};'
+mysql_connection_max_attempts=10
-mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" -e 'GRANT ALL ON {{ OPENEDX_MYSQL_DATABASE }}.* TO "{{ OPENEDX_MYSQL_USERNAME }}"@"%" IDENTIFIED BY "{{ OPENEDX_MYSQL_PASSWORD }}";'
+mysql_connection_attempt=0
+until mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" --port {{ MYSQL_PORT }} -e 'exit'
+do
+    mysql_connection_attempt=$(expr $mysql_connection_attempt + 1)
+    echo "    [$mysql_connection_attempt/$mysql_connection_max_attempts] Waiting for MySQL service (this may take a while)..."
+    if [ $mysql_connection_attempt -eq $mysql_connection_max_attempts ]
+    then
+      echo "MySQL initialization error" 1>&2
+      exit 1
+    fi
+    sleep 10
+done
+echo "MySQL is up and running"
+
+mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" --port {{ MYSQL_PORT }} -e 'CREATE DATABASE IF NOT EXISTS {{ OPENEDX_MYSQL_DATABASE }};'
+mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" --port {{ MYSQL_PORT }} -e 'GRANT ALL ON {{ OPENEDX_MYSQL_DATABASE }}.* TO "{{ OPENEDX_MYSQL_USERNAME }}"@"%" IDENTIFIED BY "{{ OPENEDX_MYSQL_PASSWORD }}";'
 
 {% if ACTIVATE_NOTES %}
-mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" -e 'CREATE DATABASE IF NOT EXISTS {{ NOTES_MYSQL_DATABASE }};'
+mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" --port {{ MYSQL_PORT }} -e 'CREATE DATABASE IF NOT EXISTS {{ NOTES_MYSQL_DATABASE }};'
-mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" -e 'GRANT ALL ON {{ NOTES_MYSQL_DATABASE }}.* TO "{{ NOTES_MYSQL_USERNAME }}"@"%" IDENTIFIED BY "{{ NOTES_MYSQL_PASSWORD }}";'
+mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" --port {{ MYSQL_PORT }} -e 'GRANT ALL ON {{ NOTES_MYSQL_DATABASE }}.* TO "{{ NOTES_MYSQL_USERNAME }}"@"%" IDENTIFIED BY "{{ NOTES_MYSQL_PASSWORD }}";'
 {% endif %}
 
 {% if ACTIVATE_XQUEUE %}
-mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" -e 'CREATE DATABASE IF NOT EXISTS {{ XQUEUE_MYSQL_DATABASE }};'
+mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" --port {{ MYSQL_PORT }} -e 'CREATE DATABASE IF NOT EXISTS {{ XQUEUE_MYSQL_DATABASE }};'
-mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" -e 'GRANT ALL ON {{ XQUEUE_MYSQL_DATABASE }}.* TO "{{ XQUEUE_MYSQL_USERNAME }}"@"%" IDENTIFIED BY "{{ XQUEUE_MYSQL_PASSWORD }}";'
+mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" --port {{ MYSQL_PORT }} -e 'GRANT ALL ON {{ XQUEUE_MYSQL_DATABASE }}.* TO "{{ XQUEUE_MYSQL_USERNAME }}"@"%" IDENTIFIED BY "{{ XQUEUE_MYSQL_PASSWORD }}";'
 {% endif %}