From 07b3d113d4de79d07d3f610c10f944e4ddac831a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Mon, 18 Mar 2019 17:26:37 +0100 Subject: [PATCH] Simplify environment generation Environment is no longer generated separately for each target, but only once the configuration is saved. Note that the environment is automatically updated during re-configuration, based on a "version" file stored in the environment. --- .travis.yml | 2 +- CHANGELOG.md | 1 + Makefile | 30 ++++++++++++++-------------- cloud/aws.sh | 2 +- docs/configuration.rst | 4 ++-- docs/customise.rst | 21 +++++++------------- docs/local.rst | 9 --------- docs/mobile.rst | 6 +----- docs/troubleshooting.rst | 7 +++---- tutor/android.py | 12 ------------ tutor/config.py | 31 +++++++++++++++++++++++++---- tutor/env.py | 33 ++++++++++++++++++++++++++----- tutor/images.py | 42 +++++++++++++++++++++++----------------- tutor/k8s.py | 12 ------------ tutor/local.py | 12 ------------ 15 files changed, 111 insertions(+), 113 deletions(-) diff --git a/.travis.yml b/.travis.yml index e574868..e48a3b6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,8 +31,8 @@ matrix: script: - make ci-info + - make ci-config - make ci-bundle - - make ci-test deploy: # Push tutor binary to github releases diff --git a/CHANGELOG.md b/CHANGELOG.md index 62c22da..af4f2f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Latest +- [Improvement] Automatic environment re-generation after re-configuration - [Improvement] Error and interrupt handling in UI and web UI - [Bugfix] Fix missing webui env directory diff --git a/Makefile b/Makefile index 8323032..6cd5952 100644 --- a/Makefile +++ b/Makefile @@ -1,18 +1,19 @@ .DEFAULT_GOAL := help +###### Development + compile-requirements: ## Compile requirements files pip-compile -o requirements/base.txt requirements/base.in pip-compile -o requirements/dev.txt requirements/dev.in pip-compile -o requirements/docs.txt requirements/docs.in +###### Deployment + bundle: ## Bundle the tutor package in a single "dist/tutor" executable pyinstaller --onefile --name=tutor --add-data=./tutor/templates:./tutor/templates ./bin/main dist/tutor: $(MAKE) bundle -version: ## Print the current tutor version - @python -c 'import io, os; about = {}; exec(io.open(os.path.join("tutor", "__about__.py"), "rt", encoding="utf-8").read(), about); print(about["__version__"])' - tag: ## Create a release, update the "latest" tag and push them to origin $(MAKE) retag TAG=v$(shell make version) $(MAKE) retag TAG=latest @@ -25,27 +26,23 @@ retag: git push origin :$(TAG) || true git push origin $(TAG) -travis: bundle ## Run tests on travis-ci - ./dist/tutor config save --silent - ./dist/tutor images env - ./dist/tutor images build all - ./dist/tutor local databases +###### Continuous integration tasks + +ci-config: ## Generate configuration and environment + ./dist/tutor config save --silent --set ACTIVATE_NOTES=true --set ACTIVATE_XQUEUE=true ci-info: ## Print info about environment python3 --version pip3 --version -ci-bundle: ## Create bundle +ci-bundle: ## Create bundle and run basic tests pip3 install -U setuptools pip3 install -r requirements/dev.txt $(MAKE) bundle mkdir -p releases/ cp ./dist/tutor ./releases/tutor-$$(uname -s)_$$(uname -m) - -ci-test: ## Run basic tests - ./dist/tutor config save --silent - ./dist/tutor images env - ./dist/tutor local env + ./dist/tutor --version + ./dist/tutor config printroot ci-images: ## Build and push docker images to hub.docker.com python setup.py develop @@ -59,6 +56,11 @@ ci-pypi: ## Push release to pypi python setup.py sdist twine upload dist/*.tar.gz +###### Additional commands + +version: ## Print the current tutor version + @python -c 'import io, os; about = {}; exec(io.open(os.path.join("tutor", "__about__.py"), "rt", encoding="utf-8").read(), about); print(about["__version__"])' + ESCAPE =  help: ## Print this help @grep -E '^([a-zA-Z_-]+:.*?## .*|######* .+)$$' Makefile \ diff --git a/cloud/aws.sh b/cloud/aws.sh index 84af469..481f9e6 100755 --- a/cloud/aws.sh +++ b/cloud/aws.sh @@ -44,7 +44,7 @@ docker pull rabbitmq:3.6.10 docker pull namshi/smtp:latest echo "=============== Building docker images" -tutor images env +tutor config save --silent --set ACTIVATE_NOTES=true --set ACTIVATE_XQUEUE=true tutor images build all echo "=============== Create Web UI script" diff --git a/docs/configuration.rst b/docs/configuration.rst index de7893e..88ce3dc 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -15,9 +15,9 @@ Alternatively, you can set each parameter from the command line:: tutor config save --silent --set PARAM1=VALUE1 --set PARAM2=VALUE2 -After changing a configuration parameter, it will be taken into account next time the environment is generated. For instance, in a local installation:: +Once the base configuration is created or updated, the environment is automatically re-generated. The environment is the set of all files required to manage an Open edX platform: Dockerfile, ``lms.env.json``, settings files... You can view the environment files in the ``env`` folder:: - tutor local env + ls $(tutor config printroot)/env .. _docker_images: diff --git a/docs/customise.rst b/docs/customise.rst index 11b16d1..31371e8 100644 --- a/docs/customise.rst +++ b/docs/customise.rst @@ -5,11 +5,7 @@ Open edX platform customisation There are different ways you can customise your Open edX platform. For instance, optional features can be activated during configuration. But if you want to add unique features to your Open edX platform, you are going to have to modify and re-build the ``openedx`` docker image. This is the image that contains the ``edx-platform`` repository: it is in charge of running the web application for the Open edX "core". Both the LMS and the CMS run from the ``openedx`` docker image. -On a vanilla platform deployed by Tutor, the image that is run is downloaded from the `regis/openedx repository on Docker Hub `_. This is also the image that is downloaded whenever we run ``tutor local pullimages``. But you can decide to build the image locally instead of downloading it. To do so, generate the image-building environment:: - - tutor images env - -Then, build and tag the ``openedx`` image:: +On a vanilla platform deployed by Tutor, the image that is run is downloaded from the `regis/openedx repository on Docker Hub `_. This is also the image that is downloaded whenever we run ``tutor local pullimages``. But you can decide to build the image locally instead of downloading it. To do so, build and tag the ``openedx`` image:: tutor images build openedx @@ -23,7 +19,7 @@ Adding custom themes Comprehensive theming is enabled by default, but only the default theme is compiled. To compile your own theme, add it to the ``env/build/openedx/themes/`` folder:: - git clone https://github.com/me/myopenedxtheme.git env/build/openedx/themes/ + git clone https://github.com/me/myopenedxtheme.git $(tutor config printroot)/env/build/openedx/themes/ The ``themes`` folder should have the following structure:: @@ -47,7 +43,7 @@ Installing extra xblocks and requirements Would you like to include custom xblocks, or extra requirements to your Open edX platform? Additional requirements can be added to the ``env/build/openedx/requirements/private.txt`` file. For instance, to include the `polling xblock from Opencraft `_:: - echo "git+https://github.com/open-craft/xblock-poll.git" >> env/build/openedx/requirements/private.txt + echo "git+https://github.com/open-craft/xblock-poll.git" >> $(tutor config printroot)/env/build/openedx/requirements/private.txt Then, the ``openedx`` docker image must be rebuilt:: @@ -59,7 +55,7 @@ To install xblocks from a private repository that requires authentication, you m Then, declare your extra requirements with the ``-e`` flag in ``openedx/requirements/private.txt``:: - echo "-e ./myprivaterepo" >> env/build/openedx/requirements/private.txt + echo "-e ./myprivaterepo" >> $(tutor config printroot)/env/build/openedx/requirements/private.txt .. _edx_platform_fork: @@ -82,13 +78,10 @@ By default, Tutor runs the `regis/openedx `. -This value will then be used by Tutor when generating your environment. For instance, this is how you would use your image in a local deployment:: - - tutor local env - tutor local quickstart +This value will then be used by Tutor to run the platform, for instance when running ``tutor local quickstart``. diff --git a/docs/local.rst b/docs/local.rst index 06b69c8..497d38e 100644 --- a/docs/local.rst +++ b/docs/local.rst @@ -27,15 +27,6 @@ This is the only non-automatic step in the install process. You will be asked va If you want to run a fully automated install, upload the ``config.yml`` file to wherever you want to run Open edX. You can then entirely skip the configuration step. -Environment files generation ----------------------------- - -:: - - tutor local env - -This command generates environment files, such as the ``*.env.json``, ``*.auth.json`` files, the ``docker-compose.yml`` file, etc. They are generated from templates and the configuration values stored in ``config.yml``. The generated files are placed in the ``env/local`` subfolder of the project root. You may modify and delete those files at will, since they can be easily re-generated with the same ``tutor local env`` command. - Update docker images -------------------- diff --git a/docs/mobile.rst b/docs/mobile.rst index 3a56a44..4c06ffe 100644 --- a/docs/mobile.rst +++ b/docs/mobile.rst @@ -3,11 +3,7 @@ Mobile Android application ========================== -With Tutor, you can build an Android mobile application for your platform. First, generate the required environment:: - - tutor android env - -Then, build the app in debug mode:: +With Tutor, you can build an Android mobile application for your platform. To build the application in debug mode, run:: tutor android build debug diff --git a/docs/troubleshooting.rst b/docs/troubleshooting.rst index 401b1f1..d0d2174 100644 --- a/docs/troubleshooting.rst +++ b/docs/troubleshooting.rst @@ -43,12 +43,11 @@ The containerized Nginx needs to listen to ports 80 and 443 on the host. If ther However, you might not want to do that if you need a webserver for running non-Open edX related applications. In such cases... -2. Run the nginx container on different ports: to do so, indicate different ports in the ``config.yml`` file. For instance:: +2. Run the nginx container on different ports: to do so, configure different ports for the dockerized ngins:: - NGINX_HTTP_PORT: 81 - NGINX_HTTPS_PORT: 444 + tutor config save --silentn --set NGINX_HTTP_PORT=81 --set NGINX_HTTPS_PORT=444 -In this example, the nginx container ports would be mapped to 81 and 444, instead of 80 and 443. Then, re-generate the environment with ``tutor local env`` and restart nginx with ``tutor local restart nginx``. +In this example, the nginx container ports would be mapped to 81 and 444, instead of 80 and 443. Then, restart nginx with ``tutor local restart nginx``. You should note that with the latter solution, it is your responsibility to configure the webserver on the host as a proxy to the nginx container. See `this github issue `_ for http, and `this other github issue `_ for https. diff --git a/tutor/android.py b/tutor/android.py index d3fed61..da93c84 100644 --- a/tutor/android.py +++ b/tutor/android.py @@ -13,17 +13,6 @@ from . import utils def android(): pass -@click.command( - help="Generate the environment required for building the application" -) -@opts.root -def env(root): - config = tutor_config.load(root) - # sub.domain.com -> com.domain.sub - config["LMS_HOST_REVERSE"] = ".".join(config["LMS_HOST"].split(".")[::-1]) - tutor_env.render_target(root, config, "build") - tutor_env.render_target(root, config, "android") - @click.group( help="Build the application" ) @@ -66,5 +55,4 @@ def docker_run(root, *command): build.add_command(debug) build.add_command(release) android.add_command(build) -android.add_command(env) android.add_command(pullimage) diff --git a/tutor/config.py b/tutor/config.py index a9c70a1..2544fc5 100644 --- a/tutor/config.py +++ b/tutor/config.py @@ -8,6 +8,7 @@ from . import exceptions from . import env from . import fmt from . import opts +from .__about__ import __version__ @click.group( @@ -28,7 +29,10 @@ def save(root, silent, set_): config[k] = v if not silent: load_interactive(config) - save_config(config, root) + save_config(root, config) + + load_defaults(config) + save_env(root, config) @click.command( help="Print the project root", @@ -45,12 +49,27 @@ def load(root): config = {} load_files(config, root) + should_update_env = False if not os.path.exists(config_path(root)): load_interactive(config) - save_config(config, root) + should_update_env = True + save_config(root, config) load_defaults(config) + if not env.is_up_to_date(root): + click.echo(fmt.alert( + "The current environment stored at {} is not up-to-date: it is at " + "v{} while the 'tutor' binary is at v{}. The environment will be " + "upgraded now. Any change you might have made will be overwritten.".format( + env.base_dir(root), env.version(root), __version__ + ) + )) + should_update_env = True + + if should_update_env: + save_env(root, config) + return config def load_files(config, root): @@ -143,11 +162,11 @@ def convert_json2yml(root): ) with open(json_path) as fi: config = json.load(fi) - save_config(config, root) + save_config(root, config) os.remove(json_path) click.echo(fmt.info("File config.json detected in {} and converted to config.yml".format(root))) -def save_config(config, root): +def save_config(root, config): env.render_dict(config) path = config_path(root) directory = os.path.dirname(path) @@ -157,6 +176,10 @@ def save_config(config, root): yaml.dump(config, of, default_flow_style=False) click.echo(fmt.info("Configuration saved to {}".format(path))) +def save_env(root, config): + env.render_full(root, config) + click.echo(fmt.info("Environment generated in {}".format(env.base_dir(root)))) + def config_path(root): return os.path.join(root, "config.yml") diff --git a/tutor/env.py b/tutor/env.py index b96fde6..aaecb4d 100644 --- a/tutor/env.py +++ b/tutor/env.py @@ -2,15 +2,25 @@ import codecs import os import shutil -import click import jinja2 from . import exceptions -from . import fmt from . import utils +from .__about__ import __version__ TEMPLATES_ROOT = os.path.join(os.path.dirname(__file__), "templates") +VERSION_FILENAME = "version" + +def render_full(root, config): + """ + Render the full environment, including version information. + """ + for target in ["apps", "k8s", "local", "webui"]: + render_target(root, config, target) + copy_target(root, "build") + with open(pathjoin(root, VERSION_FILENAME), 'w') as f: + f.write(__version__) def render_target(root, config, target): """ @@ -23,7 +33,6 @@ def render_target(root, config, target): substituted = render_str(fi.read(), config) with open(dst, "w") as of: of.write(substituted) - click.echo(fmt.info("Environment generated in {}".format(pathjoin(root, target)))) def render_dict(config): """ @@ -70,11 +79,22 @@ def copy_target(root, target): for src, dst in walk_templates(root, target): if is_part_of_env(src): shutil.copy(src, dst) - click.echo(fmt.info("Environment generated in {}".format(pathjoin(root, target)))) def is_part_of_env(path): return not os.path.basename(path).startswith(".") +def is_up_to_date(root): + return version(root) == __version__ + +def version(root): + """ + Return the current environment version. + """ + path = pathjoin(root, VERSION_FILENAME) + if not os.path.exists(path): + return "0" + return open(path).read().strip() + def read(*path): """ Read template content located at `path`. @@ -108,4 +128,7 @@ def data_path(root, *path): return os.path.join(os.path.abspath(root), "data", *path) def pathjoin(root, target, *path): - return os.path.join(root, "env", target, *path) + return os.path.join(base_dir(root), target, *path) + +def base_dir(root): + return os.path.join(root, "env") diff --git a/tutor/images.py b/tutor/images.py index f2753ac..1669812 100644 --- a/tutor/images.py +++ b/tutor/images.py @@ -1,5 +1,6 @@ import click +from . import config as tutor_config from . import env as tutor_env from . import fmt from . import opts @@ -16,24 +17,18 @@ argument_image = click.argument( "image", type=click.Choice(["all"] + all_images), ) -@click.command( - short_help="Generate environment", - help="""Generate the environment files required to build and customise the docker images.""" -) -@opts.root -def env(root): - tutor_env.copy_target(root, "build") - @click.command( short_help="Download docker images", help=("""Download the docker images from hub.docker.com. The images will come from {namespace}/{image}:{version}.""") ) +@opts.root @option_namespace @option_version @argument_image -def download(namespace, version, image): - for image in image_list(image): +def download(root, namespace, version, image): + config = tutor_config.load(root) + for image in image_list(config, image): utils.docker('image', 'pull', get_tag(namespace, image, version)) @click.command( @@ -50,7 +45,8 @@ def download(namespace, version, image): help="Set build-time docker ARGS in the form 'myarg=value'. This option may be specified multiple times." ) def build(root, namespace, version, image, build_arg): - for image in image_list(image): + config = tutor_config.load(root) + for image in image_list(config, image): tag = get_tag(namespace, image, version) click.echo(fmt.info("Building image {}".format(tag))) command = [ @@ -66,11 +62,13 @@ def build(root, namespace, version, image, build_arg): @click.command( short_help="Pull images from hub.docker.com", ) +@opts.root @option_namespace @option_version @argument_image -def pull(namespace, version, image): - for image in image_list(image): +def pull(root, namespace, version, image): + config = tutor_config.load(root) + for image in image_list(config, image): tag = get_tag(namespace, image, version) click.echo(fmt.info("Pulling image {}".format(tag))) utils.execute("docker", "pull", tag) @@ -78,11 +76,13 @@ def pull(namespace, version, image): @click.command( short_help="Push images to hub.docker.com", ) +@opts.root @option_namespace @option_version @argument_image -def push(namespace, version, image): - for image in image_list(image): +def push(root, namespace, version, image): + config = tutor_config.load(root) + for image in image_list(config, image): tag = get_tag(namespace, image, version) click.echo(fmt.info("Pushing image {}".format(tag))) utils.execute("docker", "push", tag) @@ -96,10 +96,16 @@ def get_tag(namespace, image, version): version=version, ) -def image_list(image): - return all_images if image == "all" else [image] +def image_list(config, image): + if image == "all": + images = all_images[:] + if not config['ACTIVATE_XQUEUE']: + images.remove('xqueue') + if not config['ACTIVATE_NOTES']: + images.remove('notes') + return images + return [image] -images.add_command(env) images.add_command(download) images.add_command(build) images.add_command(pull) diff --git a/tutor/k8s.py b/tutor/k8s.py index e1f16a6..6cb56bf 100644 --- a/tutor/k8s.py +++ b/tutor/k8s.py @@ -22,7 +22,6 @@ def quickstart(root): click.echo(fmt.title("Interactive platform configuration")) tutor_config.save.callback(root, False, []) click.echo(fmt.title("Environment generation")) - env.callback(root) click.echo(fmt.title("Stopping any existing platform")) stop.callback() click.echo(fmt.title("Starting the platform")) @@ -30,16 +29,6 @@ def quickstart(root): click.echo(fmt.title("Running migrations. NOTE: this might fail. If it does, please retry 'tutor k9s databases later'")) databases.callback(root) -@click.command( - short_help="Generate environment", - help="Generate the environment files required to run Open edX", -) -@opts.root -def env(root): - config = tutor_config.load(root) - tutor_env.render_target(root, config, "apps") - tutor_env.render_target(root, config, "k8s") - @click.command(help="Run all configured Open edX services") @opts.root def start(root): @@ -177,7 +166,6 @@ def run_bash(root, service, command): K8s().execute(service, "bash", "-e", "-c", command) k8s.add_command(quickstart) -k8s.add_command(env) k8s.add_command(start) k8s.add_command(stop) k8s.add_command(delete) diff --git a/tutor/local.py b/tutor/local.py index 1eb8aa3..a9399c0 100644 --- a/tutor/local.py +++ b/tutor/local.py @@ -29,7 +29,6 @@ def quickstart(pullimages_, root): click.echo(fmt.title("Interactive platform configuration")) tutor_config.save.callback(root, False, []) click.echo(fmt.title("Environment generation")) - env.callback(root) click.echo(fmt.title("Stopping any existing platform")) stop.callback(root) if pullimages_: @@ -42,16 +41,6 @@ def quickstart(pullimages_, root): click.echo(fmt.title("Starting the platform in detached mode")) start.callback(root, True) -@click.command( - short_help="Generate environment", - help="Generate the environment files required to run Open edX", -) -@opts.root -def env(root): - config = tutor_config.load(root) - tutor_env.render_target(root, config, "apps") - tutor_env.render_target(root, config, "local") - @click.command( help="Update docker images", ) @@ -261,7 +250,6 @@ https.add_command(https_create) https.add_command(https_renew) local.add_command(quickstart) -local.add_command(env) local.add_command(pullimages) local.add_command(start) local.add_command(stop)