From badb28b361895be22df2dda6942d2a8c51beae7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Thu, 9 May 2019 08:56:08 +0200 Subject: [PATCH] Better mysql initialization For initializing the mysql database, we now run the mysql client from the mysql container. That means that the mysql client is no longer required in the openedx container, as previously suggested in PR #209. --- CHANGELOG.md | 1 + tutor/local.py | 36 --------------------- tutor/scripts.py | 3 +- tutor/templates/build/openedx/Dockerfile | 2 -- tutor/templates/local/docker-compose.yml | 6 ++++ tutor/templates/scripts/create_databases.sh | 29 +++++++++++++---- 6 files changed, 30 insertions(+), 47 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 186d902..208c672 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - [Bugfix] Fix installing a locally cloned requirement repository - [Improvement] Add `--no-cache` option to `images build` - [Improvement] Make it possible to configure the notes service hostname +- [Improvement] Better, more robust MySQL initialization ## 3.3.10 (2019-05-15) diff --git a/tutor/local.py b/tutor/local.py index 35e95a3..a2fe485 100644 --- a/tutor/local.py +++ b/tutor/local.py @@ -1,7 +1,4 @@ -import os -import subprocess from textwrap import indent -from time import sleep import click @@ -148,42 +145,9 @@ def execute(root, service, command, args): @click.command(help="Create databases and run database migrations") @opts.root def databases(root): - init_mysql(root) scripts.migrate(root, run_sh) -def init_mysql(root): - config = tutor_config.load(root) - if not config["ACTIVATE_MYSQL"]: - return - mysql_data_path = tutor_env.data_path(root, "mysql", "mysql") - if os.path.exists(mysql_data_path): - return - click.echo(fmt.info("Initializing MySQL database...")) - docker_compose(root, config, "up", "-d", "mysql") - while True: - click.echo(fmt.info(" waiting for mysql initialization")) - # TODO this is duplicate code with the docker_compose function. We - # should rely on a dedicated function in utils module. - mysql_logs = subprocess.check_output( - [ - "docker-compose", - "-f", - tutor_env.pathjoin(root, "local", "docker-compose.yml"), - "--project-name", - config["LOCAL_PROJECT_NAME"], - "logs", - "mysql", - ] - ) - # pylint: disable=unsupported-membership-test - if b"MySQL init process done. Ready for start up." in mysql_logs: - click.echo(fmt.info("MySQL database initialized")) - docker_compose(root, config, "stop", "mysql") - return - sleep(4) - - @click.group(help="Manage https certificates") def https(): pass diff --git a/tutor/scripts.py b/tutor/scripts.py index 30c1d38..ae62469 100644 --- a/tutor/scripts.py +++ b/tutor/scripts.py @@ -9,8 +9,7 @@ def migrate(root, run_func): config = tutor_config.load(root) click.echo(fmt.info("Creating all databases...")) - # Note: run this only when lms is activated? - run_template(root, config, "lms", "create_databases.sh", run_func) + run_template(root, config, "mysql-client", "create_databases.sh", run_func) if config["ACTIVATE_LMS"]: click.echo(fmt.info("Running lms migrations...")) diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index 0dd73fb..dc751bb 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -9,8 +9,6 @@ RUN apt update && \ apt install -y language-pack-en git python-virtualenv build-essential software-properties-common curl git-core libxml2-dev libxslt1-dev python-virtualenv libmysqlclient-dev python-apt python-dev libxmlsec1-dev libfreetype6-dev swig gcc g++ \ # openedx requirements gettext gfortran graphviz graphviz-dev libffi-dev libfreetype6-dev libgeos-dev libjpeg8-dev liblapack-dev libpng12-dev libsqlite3-dev libxml2-dev libxmlsec1-dev libxslt1-dev lynx nodejs npm ntp pkg-config \ - # Our requirements - mysql-client \ && rm -rf /var/lib/apt/lists/* # Dockerize will be useful to wait for mysql DB availability diff --git a/tutor/templates/local/docker-compose.yml b/tutor/templates/local/docker-compose.yml index cf7c571..fe0f588 100644 --- a/tutor/templates/local/docker-compose.yml +++ b/tutor/templates/local/docker-compose.yml @@ -28,6 +28,12 @@ services: - ../../data/mysql:/var/lib/mysql env_file: ../apps/mysql/auth.env {% endif %} + mysql-client: + image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_MYSQL }} + command: echo "mysql client ready" + restart: "no" + {% if ACTIVATE_MYSQL %}depends_on: + - mysql{% endif %} {% if ACTIVATE_ELASTICSEARCH %} elasticsearch: diff --git a/tutor/templates/scripts/create_databases.sh b/tutor/templates/scripts/create_databases.sh index 5a5d128..100ed08 100644 --- a/tutor/templates/scripts/create_databases.sh +++ b/tutor/templates/scripts/create_databases.sh @@ -1,13 +1,28 @@ -dockerize -wait tcp://{{ MYSQL_HOST }}:{{ MYSQL_PORT }} -timeout 20s -mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" -e 'CREATE DATABASE IF NOT EXISTS {{ OPENEDX_MYSQL_DATABASE }};' -mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" -e 'GRANT ALL ON {{ OPENEDX_MYSQL_DATABASE }}.* TO "{{ OPENEDX_MYSQL_USERNAME }}"@"%" IDENTIFIED BY "{{ OPENEDX_MYSQL_PASSWORD }}";' +echo "Initializing MySQL..." +mysql_connection_max_attempts=10 +mysql_connection_attempt=0 +until mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" --port {{ MYSQL_PORT }} -e 'exit' +do + mysql_connection_attempt=$(expr $mysql_connection_attempt + 1) + echo " [$mysql_connection_attempt/$mysql_connection_max_attempts] Waiting for MySQL service (this may take a while)..." + if [ $mysql_connection_attempt -eq $mysql_connection_max_attempts ] + then + echo "MySQL initialization error" 1>&2 + exit 1 + fi + sleep 10 +done +echo "MySQL is up and running" + +mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" --port {{ MYSQL_PORT }} -e 'CREATE DATABASE IF NOT EXISTS {{ OPENEDX_MYSQL_DATABASE }};' +mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" --port {{ MYSQL_PORT }} -e 'GRANT ALL ON {{ OPENEDX_MYSQL_DATABASE }}.* TO "{{ OPENEDX_MYSQL_USERNAME }}"@"%" IDENTIFIED BY "{{ OPENEDX_MYSQL_PASSWORD }}";' {% if ACTIVATE_NOTES %} -mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" -e 'CREATE DATABASE IF NOT EXISTS {{ NOTES_MYSQL_DATABASE }};' -mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" -e 'GRANT ALL ON {{ NOTES_MYSQL_DATABASE }}.* TO "{{ NOTES_MYSQL_USERNAME }}"@"%" IDENTIFIED BY "{{ NOTES_MYSQL_PASSWORD }}";' +mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" --port {{ MYSQL_PORT }} -e 'CREATE DATABASE IF NOT EXISTS {{ NOTES_MYSQL_DATABASE }};' +mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" --port {{ MYSQL_PORT }} -e 'GRANT ALL ON {{ NOTES_MYSQL_DATABASE }}.* TO "{{ NOTES_MYSQL_USERNAME }}"@"%" IDENTIFIED BY "{{ NOTES_MYSQL_PASSWORD }}";' {% endif %} {% if ACTIVATE_XQUEUE %} -mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" -e 'CREATE DATABASE IF NOT EXISTS {{ XQUEUE_MYSQL_DATABASE }};' -mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" -e 'GRANT ALL ON {{ XQUEUE_MYSQL_DATABASE }}.* TO "{{ XQUEUE_MYSQL_USERNAME }}"@"%" IDENTIFIED BY "{{ XQUEUE_MYSQL_PASSWORD }}";' +mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" --port {{ MYSQL_PORT }} -e 'CREATE DATABASE IF NOT EXISTS {{ XQUEUE_MYSQL_DATABASE }};' +mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" --port {{ MYSQL_PORT }} -e 'GRANT ALL ON {{ XQUEUE_MYSQL_DATABASE }}.* TO "{{ XQUEUE_MYSQL_USERNAME }}"@"%" IDENTIFIED BY "{{ XQUEUE_MYSQL_PASSWORD }}";' {% endif %}