From ea2dd7c4fb5470ce8ca7412095f3a849f185ed8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Sun, 5 Aug 2018 16:26:19 +0200 Subject: [PATCH] Generate configuration from Docker We no longer run the `configure` script on the host. Instead, we run a container that generates the configuration files. This opens the way for more complex configuration templates that would be written in jinja2. More complex templates are required for feature flags, such as SSL, XQUEUE, etc. --- .travis.yml | 2 +- Makefile | 5 +- README.md | 9 +- configurator/Dockerfile | 25 +++ configurator/bin/configure.py | 172 +++++++++++++++++++ configurator/bin/docker-entrypoint.sh | 18 ++ configure | 227 -------------------------- 7 files changed, 222 insertions(+), 236 deletions(-) create mode 100644 configurator/Dockerfile create mode 100755 configurator/bin/configure.py create mode 100755 configurator/bin/docker-entrypoint.sh delete mode 100755 configure diff --git a/.travis.yml b/.travis.yml index dc653b6..786f92f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,7 @@ services: - docker script: - make build - - ./configure --silent + - make configure SILENT=1 - make migrate #- make assets # too time-consuming - docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" diff --git a/Makefile b/Makefile index aa7661f..8949dd2 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,8 @@ all: configure update migrate assets daemon ##################### Bootstrapping configure: - ./configure + docker build -t regis/openedx-configurator:latest configurator/ + docker run --rm -it --volume="$(PWD)/config:/openedx/config" -e USERID=$(USERID) -e SILENT=$(SILENT) regis/openedx-configurator update: docker-compose pull @@ -63,8 +64,6 @@ info: @echo "-------------------------" docker-compose --version @echo "-------------------------" - python --version - @echo "-------------------------" echo $$EDX_PLATFORM_PATH echo $$EDX_PLATFORM_SETTINGS diff --git a/README.md b/README.md index 829620a..dd4bc98 100644 --- a/README.md +++ b/README.md @@ -25,11 +25,10 @@ This might seem too simple to be true, but there's no magic -- just good packagi ## Requirements -The only prerequisite for running this is Python and a working docker install. You will need both docker and docker-compose. Follow the instructions from the official documentation: +The only prerequisite for running this is a working docker install. You will need both docker and docker-compose. Follow the instructions from the official documentation: -- [Python](https://www.python.org/downloads/): all versions >= 2.7 are supported. -- [Docker install](https://docs.docker.com/engine/installation/) -- [Docker compose install](https://docs.docker.com/compose/install/) +- [Docker](https://docs.docker.com/engine/installation/) +- [Docker compose](https://docs.docker.com/compose/install/) Note that the production web server container will bind to port 80, so if you already have a web server running (Apache or Nginx, for instance), you should stop it. @@ -45,7 +44,7 @@ Also, the host running the containers should be a 64 bit platform. (images are n make configure -This is the only non-automatic step in the install process. You will be asked various questions about your Open edX platform and appropriate configuration files will be generated. If you would like to automate this step then you should run `make configure` interactively once. After that, you will have a `config.json` file at the root of the repository. Just upload it to wherever you want to run Open edX and then run `./configure --silent` instead of `make configure`. All values from `config.json` will be automatically loaded. +This is the only non-automatic step in the install process. You will be asked various questions about your Open edX platform and appropriate configuration files will be generated. If you would like to automate this step then you should run `make configure` interactively once. After that, you will have a `config.json` file at the root of the repository. Just upload it to wherever you want to run Open edX and then run `make configure SILENT=1` instead of `make configure`. All values from `config.json` will be automatically loaded. ### Download diff --git a/configurator/Dockerfile b/configurator/Dockerfile new file mode 100644 index 0000000..6cdc2a1 --- /dev/null +++ b/configurator/Dockerfile @@ -0,0 +1,25 @@ +FROM ubuntu:18.04 + +RUN apt update && \ + apt install -y python3 python3-pip +RUN pip3 install jinja2 + +RUN mkdir /openedx +VOLUME /openedx/config +COPY ./bin/configure.py /openedx/configure.py +COPY ./bin/docker-entrypoint.sh /openedx/docker-entrypoint.sh +WORKDIR /openedx + +ENV SILENT='' +ENTRYPOINT ["./docker-entrypoint.sh"] +CMD ./configure.py interactive ${SILENT:+--silent} && \ + ./configure.py substitute ./config/openedx/templates/lms.env.json.templ ./config/openedx/lms.env.json && \ + ./configure.py substitute ./config/openedx/templates/cms.env.json.templ ./config/openedx/cms.env.json && \ + ./configure.py substitute ./config/openedx/templates/lms.auth.json.templ ./config/openedx/lms.auth.json && \ + ./configure.py substitute ./config/openedx/templates/cms.auth.json.templ ./config/openedx/cms.auth.json && \ + ./configure.py substitute ./config/openedx/templates/provision.sh.templ ./config/openedx/provision.sh && \ + ./configure.py substitute ./config/xqueue/templates/universal.py.templ ./config/xqueue/universal.py && \ + ./configure.py substitute ./config/mysql/templates/auth.env.templ ./config/mysql/auth.env && \ + ./configure.py substitute --delimiter '£' ./config/nginx/templates/lms.conf.templ ./config/nginx/lms.conf && \ + ./configure.py substitute --delimiter '£' ./config/nginx/templates/cms.conf.templ ./config/nginx/cms.conf && \ + ./configure.py substitute ./config/android/templates/universal.yaml.templ ./config/android/universal.yaml diff --git a/configurator/bin/configure.py b/configurator/bin/configure.py new file mode 100755 index 0000000..50451fe --- /dev/null +++ b/configurator/bin/configure.py @@ -0,0 +1,172 @@ +#! /usr/bin/env python3 +# coding: utf8 +import argparse +import codecs +import json +import os +import random +import string +import sys + +from collections import OrderedDict + + +class Configurator: + + def __init__(self, **default_overrides): + self.__values = OrderedDict() + self.__default_values = default_overrides + try: + self.__input = raw_input + except NameError: + self.__input = input + + def as_dict(self): + all_values = self.__values.copy() + for key, value in self.__default_values.items(): + all_values.setdefault(key, value) + return all_values + + def mute(self): + self.__input = None + + def add(self, name, question="", default=""): + default = self.__default_values.get(name, default) + value = default + if question: + message = question + " (default: \"{}\"): ".format(default) + value = self.ask(message, default) + self.set(name, value) + + return self + + def add_boolean(self, name, question, default=True): + default = self.__default_values.get(name, default) + message = question + " ({}) ".format("Y/n" if default else "y/N") + value = None + while value is None: + answer = self.ask(message, default) + value = { + "y": True, + "n": False, + default: default, + }.get(answer) + self.set(name, value) + return self + + def ask(self, message, default): + if self.__input: + return self.__input(message) or default + return default + + def get(self, name): + return self.__values.get(name) + + def set(self, name, value): + self.__values[name] = value + + +def main(): + parser = argparse.ArgumentParser("Config file generator for Open edX") + parser.add_argument('-c', '--config', default=os.path.join("/", "openedx", "config", "config.json"), + help="Load default values from this file. Config values will be saved there.") + subparsers = parser.add_subparsers() + + parser_interactive = subparsers.add_parser('interactive') + parser_interactive.add_argument('-s', '--silent', action='store_true', + help=( + "Be silent and accept all default values. " + "This is good for debugging and automation, but " + "probably not what you want" + )) + parser_interactive.set_defaults(func=interactive) + + parser_substitute = subparsers.add_parser('substitute') + parser_substitute.add_argument('--delimiter', default='$', help="Template file delimiter") + parser_substitute.add_argument('src', help="Template source file") + parser_substitute.add_argument('dst', help="Destination configuration file") + parser_substitute.set_defaults(func=substitute) + + args = parser.parse_args() + + # Load defaults + defaults = {} + if os.path.exists(args.config): + with open(args.config) as f: + defaults = json.load(f) + configurator = Configurator(**defaults) + + args.func(configurator, args) + + +def interactive(configurator, args): + if args.silent: + configurator.mute() + configurator.add( + 'LMS_HOST', "Your website domain name for students (LMS).", 'www.myopenedx.com' + ).add( + 'CMS_HOST', "Your website domain name for teachers (CMS).", 'studio.myopenedx.com' + ).add( + 'PLATFORM_NAME', "Platform name/title", "My Open edX" + ).add( + 'SECRET_KEY', "", random_string(24) + ).add( + 'MYSQL_DATABASE', "", 'openedx' + ).add( + 'MYSQL_USERNAME', "", 'openedx' + ).add( + 'MYSQL_PASSWORD', "", random_string(8), + ).add( + 'MONGODB_DATABASE', "", 'openedx' + ).add( + 'XQUEUE_AUTH_USERNAME', "", 'lms' + ).add( + 'XQUEUE_AUTH_PASSWORD', "", random_string(8), + ).add( + 'XQUEUE_MYSQL_DATABASE', "", 'xqueue', + ).add( + 'XQUEUE_MYSQL_USERNAME', "", 'xqueue', + ).add( + 'XQUEUE_MYSQL_PASSWORD', "", random_string(8), + ).add( + 'XQUEUE_SECRET_KEY', "", random_string(24), + ) + + # Save values + with open(args.config, 'w') as f: + json.dump(configurator.as_dict(), f, sort_keys=True, indent=4) + print("\nConfiguration values were saved to ", args.config) + + +def substitute(configurator, args): + with codecs.open(args.src, encoding='utf-8') as fi: + template = template_class(args.delimiter)(fi.read()) + try: + substituted = template.substitute(**configurator.as_dict()) + except KeyError as e: + sys.stderr.write("ERROR Missing config value '{}' for template {}\n".format(e.args[0], args.src)) + sys.exit(1) + + with codecs.open(args.dst, encoding='utf-8', mode='w') as fo: + fo.write(substituted) + + print("Generated file {} from template {}".format(args.dst, args.src)) + + +def template_class(user_delimiter='$'): + """ + The default delimiter of the python Template class is '$'. Here, we + generate a Template class with a custom delimiter. This cannot be done + after the class creation because the Template metaclass uses the delimiter + value. + """ + class Template(string.Template): + delimiter = user_delimiter + return Template + + +def random_string(length): + return "".join([random.choice(string.ascii_letters + string.digits) for _ in range(length)]) + +if __name__ == '__main__': + main() diff --git a/configurator/bin/docker-entrypoint.sh b/configurator/bin/docker-entrypoint.sh new file mode 100755 index 0000000..67f4b65 --- /dev/null +++ b/configurator/bin/docker-entrypoint.sh @@ -0,0 +1,18 @@ +#!/bin/bash -e +USERID=${USERID:=0} + +## Configure user with a different USERID if requested. +if [ "$USERID" -ne 0 ] + then + echo "creating new user 'openedx' with UID $USERID" + useradd --home-dir /openedx -u $USERID openedx + + # Change file permissions + chown --no-dereference -R openedx /openedx/config + + # Run CMD as different user + exec chroot --userspec="$USERID" --skip-chdir / env HOME=/openedx "$@" +else + # Run CMD as root (business as usual) + exec "$@" +fi diff --git a/configure b/configure deleted file mode 100755 index 960f655..0000000 --- a/configure +++ /dev/null @@ -1,227 +0,0 @@ -#! /usr/bin/env python -# coding: utf8 -from __future__ import unicode_literals, print_function -import argparse -import codecs -import json -import os -import random -import string -import sys - -class Configurator(object): - - def __init__(self, silent=False, **default_overrides): - self.__values = [] - self.__default_overrides = default_overrides - if silent: - self.__input = None - else: - try: - self.__input = raw_input - except NameError: - self.__input = input - - def as_dict(self): - return dict(self.__values) - - def add(self, name, question="", default=""): - default = self.__default_overrides.get(name, default) - value = default - if question: - message = question + " (default: \"{}\"): ".format(default) - value = self.ask(message, default) - self.set(name, value) - - return self - - def add_boolean(self, name, question, default=True): - default = self.__default_overrides.get(name, default) - message = question + " ({}) ".format("Y/n" if default else "y/N") - value = None - while value is None: - answer = self.ask(message, default) - value = { - "y": True, - "n": False, - default: default, - }.get(answer) - self.set(name, value) - return self - - def ask(self, message, default): - if self.__input: - return self.__input(message) or default - return default - - def get(self, name): - for key, value in self.__values: - if key == name: - return value - return None - - def set(self, name, value): - self.__values.append((name, value)) - - -def substitute(src, dst, delimiter='$', **values): - Template = template_class(delimiter) - - with codecs.open(src, encoding='utf-8') as fi: - template = Template(fi.read()) - try: - substituted = template.substitute(**values) - except KeyError as e: - sys.stderr.write("ERROR Missing config value '{}' for template {}\n".format(e.args[0], src)) - sys.exit(1) - - with open(dst, 'w') as fo: - fo.write(substituted) - - print("Generated config file {} (from template {})".format(dst, src)) - -def template_class(user_delimiter='$'): - """ - The default delimiter of the python Template class is '$'. Here, we - generate a Template class with a custom delimiter. This cannot be done - after the class creation because the Template metaclass uses the delimiter - value. - """ - class Template(string.Template): - delimiter = user_delimiter - return Template - - -def main(): - # Hack to handle older config.json files that used to be in a different location - # TODO remove me - config_path = os.path.join("config", "config.json") - deprecated_paths = [ - "config.json", - os.path.join("config", "openedx", "config.json"), - ] - for deprecated_path in deprecated_paths: - if os.path.exists(deprecated_path): - os.rename(deprecated_path, config_path) - - parser = argparse.ArgumentParser("Config file generator for Open edX") - parser.add_argument('-c', '--config', default=config_path, - help="Load default values from this file. Config values will be saved there.") - parser.add_argument('-s', '--silent', action='store_true', - help=( - "Be silent and accept all default values. " - "This is good for debugging, but probably not what you want" - )) - args = parser.parse_args() - - # Load defaults - defaults = {} - if os.path.exists(args.config): - with open(args.config) as f: - defaults = json.load(f) - - configurator = Configurator(silent=args.silent, **defaults).add( - 'LMS_HOST', "Your website domain name for students (LMS).", 'www.myopenedx.com' - ).add( - 'CMS_HOST', "Your website domain name for teachers (CMS).", 'studio.myopenedx.com' - ).add( - 'PLATFORM_NAME', "Platform name/title", "My Open edX" - ).add( - 'SECRET_KEY', "", random_string(24) - ).add( - 'MYSQL_DATABASE', "", 'openedx' - ).add( - 'MYSQL_USERNAME', "", 'openedx' - ).add( - 'MYSQL_PASSWORD', "", random_string(8), - ).add( - 'MONGODB_DATABASE', "", 'openedx' - ).add( - 'XQUEUE_AUTH_USERNAME', "", 'lms' - ).add( - 'XQUEUE_AUTH_PASSWORD', "", random_string(8), - ).add( - 'XQUEUE_MYSQL_DATABASE', "", 'xqueue', - ).add( - 'XQUEUE_MYSQL_USERNAME', "", 'xqueue', - ).add( - 'XQUEUE_MYSQL_PASSWORD', "", random_string(8), - ).add( - 'XQUEUE_SECRET_KEY', "", random_string(24), - ) - - # Save values - with open(args.config, 'w') as f: - json.dump(configurator.as_dict(), f, sort_keys=True, indent=4) - print("\nConfiguration values were saved to ", args.config) - - # Open edX - substitute( - os.path.join('config', 'openedx', 'templates', 'lms.env.json.templ'), - os.path.join('config', 'openedx', 'lms.env.json'), - **configurator.as_dict() - ) - substitute( - os.path.join('config', 'openedx', 'templates', 'cms.env.json.templ'), - os.path.join('config', 'openedx', 'cms.env.json'), - **configurator.as_dict() - ) - substitute( - os.path.join('config', 'openedx', 'templates', 'lms.auth.json.templ'), - os.path.join('config', 'openedx', 'lms.auth.json'), - **configurator.as_dict() - ) - substitute( - os.path.join('config', 'openedx', 'templates', 'cms.auth.json.templ'), - os.path.join('config', 'openedx', 'cms.auth.json'), - **configurator.as_dict() - ) - substitute( - os.path.join('config', 'openedx', 'templates', 'provision.sh.templ'), - os.path.join('config', 'openedx', 'provision.sh'), - **configurator.as_dict() - ) - - # Xqueue - substitute( - os.path.join('config', 'xqueue', 'templates', 'universal.py.templ'), - os.path.join('config', 'xqueue', 'universal.py'), - **configurator.as_dict() - ) - - # MySQL - substitute( - os.path.join('config', 'mysql', 'templates', 'auth.env.templ'), - os.path.join('config', 'mysql', 'auth.env'), - **configurator.as_dict() - ) - - # Nginx - # We need a different delimiter in nginx config files, because the '$' sign - # is widely used there - substitute( - os.path.join('config', 'nginx', 'templates', 'lms.conf.templ'), - os.path.join('config', 'nginx', 'lms.conf'), - delimiter='£', **configurator.as_dict() - ) - substitute( - os.path.join('config', 'nginx', 'templates', 'cms.conf.templ'), - os.path.join('config', 'nginx', 'cms.conf'), - delimiter='£', **configurator.as_dict() - ) - - # Android - substitute( - os.path.join('config', 'android', 'templates', 'universal.yaml.templ'), - os.path.join('config', 'android', 'universal.yaml'), - **configurator.as_dict() - ) - - print("\nConfiguration files were successfuly generated. You may now run the app containers.") - - -def random_string(length): - return "".join([random.choice(string.ascii_letters + string.digits) for _ in range(length)]) - -if __name__ == '__main__': - main()