diff --git a/CHANGELOG.md b/CHANGELOG.md index 186d902..208c672 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - [Bugfix] Fix installing a locally cloned requirement repository - [Improvement] Add `--no-cache` option to `images build` - [Improvement] Make it possible to configure the notes service hostname +- [Improvement] Better, more robust MySQL initialization ## 3.3.10 (2019-05-15) diff --git a/tutor/local.py b/tutor/local.py index 35e95a3..a2fe485 100644 --- a/tutor/local.py +++ b/tutor/local.py @@ -1,7 +1,4 @@ -import os -import subprocess from textwrap import indent -from time import sleep import click @@ -148,42 +145,9 @@ def execute(root, service, command, args): @click.command(help="Create databases and run database migrations") @opts.root def databases(root): - init_mysql(root) scripts.migrate(root, run_sh) -def init_mysql(root): - config = tutor_config.load(root) - if not config["ACTIVATE_MYSQL"]: - return - mysql_data_path = tutor_env.data_path(root, "mysql", "mysql") - if os.path.exists(mysql_data_path): - return - click.echo(fmt.info("Initializing MySQL database...")) - docker_compose(root, config, "up", "-d", "mysql") - while True: - click.echo(fmt.info(" waiting for mysql initialization")) - # TODO this is duplicate code with the docker_compose function. We - # should rely on a dedicated function in utils module. - mysql_logs = subprocess.check_output( - [ - "docker-compose", - "-f", - tutor_env.pathjoin(root, "local", "docker-compose.yml"), - "--project-name", - config["LOCAL_PROJECT_NAME"], - "logs", - "mysql", - ] - ) - # pylint: disable=unsupported-membership-test - if b"MySQL init process done. Ready for start up." in mysql_logs: - click.echo(fmt.info("MySQL database initialized")) - docker_compose(root, config, "stop", "mysql") - return - sleep(4) - - @click.group(help="Manage https certificates") def https(): pass diff --git a/tutor/scripts.py b/tutor/scripts.py index 30c1d38..ae62469 100644 --- a/tutor/scripts.py +++ b/tutor/scripts.py @@ -9,8 +9,7 @@ def migrate(root, run_func): config = tutor_config.load(root) click.echo(fmt.info("Creating all databases...")) - # Note: run this only when lms is activated? - run_template(root, config, "lms", "create_databases.sh", run_func) + run_template(root, config, "mysql-client", "create_databases.sh", run_func) if config["ACTIVATE_LMS"]: click.echo(fmt.info("Running lms migrations...")) diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index 0dd73fb..dc751bb 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -9,8 +9,6 @@ RUN apt update && \ apt install -y language-pack-en git python-virtualenv build-essential software-properties-common curl git-core libxml2-dev libxslt1-dev python-virtualenv libmysqlclient-dev python-apt python-dev libxmlsec1-dev libfreetype6-dev swig gcc g++ \ # openedx requirements gettext gfortran graphviz graphviz-dev libffi-dev libfreetype6-dev libgeos-dev libjpeg8-dev liblapack-dev libpng12-dev libsqlite3-dev libxml2-dev libxmlsec1-dev libxslt1-dev lynx nodejs npm ntp pkg-config \ - # Our requirements - mysql-client \ && rm -rf /var/lib/apt/lists/* # Dockerize will be useful to wait for mysql DB availability diff --git a/tutor/templates/local/docker-compose.yml b/tutor/templates/local/docker-compose.yml index cf7c571..fe0f588 100644 --- a/tutor/templates/local/docker-compose.yml +++ b/tutor/templates/local/docker-compose.yml @@ -28,6 +28,12 @@ services: - ../../data/mysql:/var/lib/mysql env_file: ../apps/mysql/auth.env {% endif %} + mysql-client: + image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_MYSQL }} + command: echo "mysql client ready" + restart: "no" + {% if ACTIVATE_MYSQL %}depends_on: + - mysql{% endif %} {% if ACTIVATE_ELASTICSEARCH %} elasticsearch: diff --git a/tutor/templates/scripts/create_databases.sh b/tutor/templates/scripts/create_databases.sh index 5a5d128..100ed08 100644 --- a/tutor/templates/scripts/create_databases.sh +++ b/tutor/templates/scripts/create_databases.sh @@ -1,13 +1,28 @@ -dockerize -wait tcp://{{ MYSQL_HOST }}:{{ MYSQL_PORT }} -timeout 20s -mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" -e 'CREATE DATABASE IF NOT EXISTS {{ OPENEDX_MYSQL_DATABASE }};' -mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" -e 'GRANT ALL ON {{ OPENEDX_MYSQL_DATABASE }}.* TO "{{ OPENEDX_MYSQL_USERNAME }}"@"%" IDENTIFIED BY "{{ OPENEDX_MYSQL_PASSWORD }}";' +echo "Initializing MySQL..." +mysql_connection_max_attempts=10 +mysql_connection_attempt=0 +until mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" --port {{ MYSQL_PORT }} -e 'exit' +do + mysql_connection_attempt=$(expr $mysql_connection_attempt + 1) + echo " [$mysql_connection_attempt/$mysql_connection_max_attempts] Waiting for MySQL service (this may take a while)..." + if [ $mysql_connection_attempt -eq $mysql_connection_max_attempts ] + then + echo "MySQL initialization error" 1>&2 + exit 1 + fi + sleep 10 +done +echo "MySQL is up and running" + +mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" --port {{ MYSQL_PORT }} -e 'CREATE DATABASE IF NOT EXISTS {{ OPENEDX_MYSQL_DATABASE }};' +mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" --port {{ MYSQL_PORT }} -e 'GRANT ALL ON {{ OPENEDX_MYSQL_DATABASE }}.* TO "{{ OPENEDX_MYSQL_USERNAME }}"@"%" IDENTIFIED BY "{{ OPENEDX_MYSQL_PASSWORD }}";' {% if ACTIVATE_NOTES %} -mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" -e 'CREATE DATABASE IF NOT EXISTS {{ NOTES_MYSQL_DATABASE }};' -mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" -e 'GRANT ALL ON {{ NOTES_MYSQL_DATABASE }}.* TO "{{ NOTES_MYSQL_USERNAME }}"@"%" IDENTIFIED BY "{{ NOTES_MYSQL_PASSWORD }}";' +mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" --port {{ MYSQL_PORT }} -e 'CREATE DATABASE IF NOT EXISTS {{ NOTES_MYSQL_DATABASE }};' +mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" --port {{ MYSQL_PORT }} -e 'GRANT ALL ON {{ NOTES_MYSQL_DATABASE }}.* TO "{{ NOTES_MYSQL_USERNAME }}"@"%" IDENTIFIED BY "{{ NOTES_MYSQL_PASSWORD }}";' {% endif %} {% if ACTIVATE_XQUEUE %} -mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" -e 'CREATE DATABASE IF NOT EXISTS {{ XQUEUE_MYSQL_DATABASE }};' -mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" -e 'GRANT ALL ON {{ XQUEUE_MYSQL_DATABASE }}.* TO "{{ XQUEUE_MYSQL_USERNAME }}"@"%" IDENTIFIED BY "{{ XQUEUE_MYSQL_PASSWORD }}";' +mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" --port {{ MYSQL_PORT }} -e 'CREATE DATABASE IF NOT EXISTS {{ XQUEUE_MYSQL_DATABASE }};' +mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" --port {{ MYSQL_PORT }} -e 'GRANT ALL ON {{ XQUEUE_MYSQL_DATABASE }}.* TO "{{ XQUEUE_MYSQL_USERNAME }}"@"%" IDENTIFIED BY "{{ XQUEUE_MYSQL_PASSWORD }}";' {% endif %}