From 6a746eada0ed564b3b9eb44f74519a496eb21112 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Mon, 18 Mar 2019 22:39:35 +0100 Subject: [PATCH] Pull vendor images from the command line Vendor docker image versions are no longer hardcoded in environment files. --- CHANGELOG.md | 1 + tutor/images.py | 30 ++++++++++++++++-------- tutor/templates/config-defaults.yml | 7 ++++++ tutor/templates/k8s/deployments.yml | 12 +++++----- tutor/templates/local/docker-compose.yml | 14 +++++------ 5 files changed, 41 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5012fd8..46fba12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Latest +- [Improvement] `images pull` now also pulls vendor images - [Feature] Add convenient `config printvalue` command - [Feature] Customize docker registry - [Feature] Load configuration parameters from the system environment diff --git a/tutor/images.py b/tutor/images.py index 9a74fc4..9c7d4de 100644 --- a/tutor/images.py +++ b/tutor/images.py @@ -10,9 +10,13 @@ from . import utils def images(): pass -all_images = ["openedx", "forum", "notes", "xqueue", "android"] +OPENEDX_IMAGES = ["openedx", "forum", "notes", "xqueue", "android"] +VENDOR_IMAGES = ["elasticsearch", "memcached", "mongodb", "mysql", "namshi", "nginx", "rabbitmq"] +argument_openedx_image = click.argument( + "image", type=click.Choice(["all"] + OPENEDX_IMAGES), +) argument_image = click.argument( - "image", type=click.Choice(["all"] + all_images), + "image", type=click.Choice(["all"] + OPENEDX_IMAGES + VENDOR_IMAGES), ) @click.command( @@ -20,14 +24,14 @@ argument_image = click.argument( help="Build the docker images necessary for an Open edX platform." ) @opts.root -@argument_image +@argument_openedx_image @click.option( "-a", "--build-arg", multiple=True, help="Set build-time docker ARGS in the form 'myarg=value'. This option may be specified multiple times." ) def build(root, image, build_arg): config = tutor_config.load(root) - for image in image_list(config, image): + for image in openedx_image_names(config, image): tag = get_tag(config, image) click.echo(fmt.info("Building image {}".format(tag))) command = [ @@ -40,22 +44,22 @@ def build(root, image, build_arg): ] utils.docker(*command) -@click.command(short_help="Pull images from the docker registry") +@click.command(short_help="Pull images from the Docker registry") @opts.root @argument_image def pull(root, image): config = tutor_config.load(root) - for image in image_list(config, image): + for image in image_names(config, image): tag = get_tag(config, image) click.echo(fmt.info("Pulling image {}".format(tag))) utils.execute("docker", "pull", tag) @click.command(short_help="Push images to the Docker registry") @opts.root -@argument_image +@argument_openedx_image def push(root, image): config = tutor_config.load(root) - for image in image_list(config, image): + for image in openedx_image_names(config, image): tag = get_tag(config, image) click.echo(fmt.info("Pushing image {}".format(tag))) utils.execute("docker", "push", tag) @@ -67,9 +71,12 @@ def get_tag(config, name): image=image, ) -def image_list(config, image): +def image_names(config, image): + return openedx_image_names(config, image) + vendor_image_names(config, image) + +def openedx_image_names(config, image): if image == "all": - images = all_images[:] + images = OPENEDX_IMAGES[:] if not config['ACTIVATE_XQUEUE']: images.remove('xqueue') if not config['ACTIVATE_NOTES']: @@ -77,6 +84,9 @@ def image_list(config, image): return images return [image] +def vendor_image_names(config, image): + return VENDOR_IMAGES if image == "all" else image + images.add_command(build) images.add_command(pull) images.add_command(push) diff --git a/tutor/templates/config-defaults.yml b/tutor/templates/config-defaults.yml index 5fc8751..8f71c97 100644 --- a/tutor/templates/config-defaults.yml +++ b/tutor/templates/config-defaults.yml @@ -4,6 +4,13 @@ DOCKER_IMAGE_ANDROID: "regis/openedx-android:hawthorn" DOCKER_IMAGE_FORUM: "regis/openedx-forum:hawthorn" DOCKER_IMAGE_NOTES: "regis/openedx-notes:hawthorn" DOCKER_IMAGE_XQUEUE: "regis/openedx-xqueue:hawthorn" +DOCKER_IMAGE_MEMCACHED: "memcached:1.4.38" +DOCKER_IMAGE_MONGODB: "mongo:3.2.16" +DOCKER_IMAGE_MYSQL: "mysql:5.6.36" +DOCKER_IMAGE_ELASTICSEARCH: "elasticsearch:1.5.2" +DOCKER_IMAGE_NGINX: "nginx:1.13" +DOCKER_IMAGE_RABBITMQ: "rabbitmq:3.6.10" +DOCKER_IMAGE_NAMSHI: "namshi/smtp:latest" DOCKER_REGISTRY: "" MYSQL_DATABASE: "openedx" MYSQL_USERNAME: "openedx" diff --git a/tutor/templates/k8s/deployments.yml b/tutor/templates/k8s/deployments.yml index 533e114..26f5c0f 100644 --- a/tutor/templates/k8s/deployments.yml +++ b/tutor/templates/k8s/deployments.yml @@ -128,7 +128,7 @@ spec: spec: containers: - name: elasticsearch - image: {{ DOCKER_REGISTRY }}elasticsearch:1.5.2 + image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_ELASTICSEARCH }} env: - name: ES_JAVA_OPTS value: "-Xms1g -Xmx1g" @@ -163,7 +163,7 @@ spec: spec: containers: - name: memcached - image: {{ DOCKER_REGISTRY }}memcached:1.4.38 + image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_MEMCACHED }} ports: - containerPort: 11211 @@ -184,7 +184,7 @@ spec: spec: containers: - name: mongodb - image: {{ DOCKER_REGISTRY }}mongo:3.2.16 + image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_MONGODB }} command: ["mongod", "--smallfiles", "--nojournal", "--storageEngine", "wiredTiger"] ports: - containerPort: 27017 @@ -212,7 +212,7 @@ spec: spec: containers: - name: mysql - image: {{ DOCKER_REGISTRY }}mysql:5.6.36 + image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_MYSQL }} env: - name: MYSQL_ROOT_PASSWORD valueFrom: @@ -261,7 +261,7 @@ spec: imagePullPolicy: Always containers: - name: nginx - image: {{ DOCKER_REGISTRY }}nginx:1.13 + image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_NGINX }} volumeMounts: - mountPath: /etc/nginx/conf.d/ name: config @@ -302,7 +302,7 @@ spec: spec: containers: - name: rabbitmq - image: {{ DOCKER_REGISTRY }}rabbitmq:3.6.10 + image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_RABBITMQ }} ports: - containerPort: 5672 volumeMounts: diff --git a/tutor/templates/local/docker-compose.yml b/tutor/templates/local/docker-compose.yml index 5a1f793..c7a3494 100644 --- a/tutor/templates/local/docker-compose.yml +++ b/tutor/templates/local/docker-compose.yml @@ -4,11 +4,11 @@ services: ############# External services memcached: - image: {{ DOCKER_REGISTRY }}memcached:1.4.38 + image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_MEMCACHED }} restart: unless-stopped mongodb: - image: {{ DOCKER_REGISTRY }}mongo:3.2.16 + image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_MONGODB }} # Use WiredTiger in all environments, just like at edx.org command: mongod --smallfiles --nojournal --storageEngine wiredTiger restart: unless-stopped @@ -16,7 +16,7 @@ services: - ../../data/mongodb:/data/db mysql: - image: {{ DOCKER_REGISTRY }}mysql:5.6.36 + image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_MYSQL }} command: mysqld --character-set-server=utf8 --collation-server=utf8_general_ci restart: unless-stopped volumes: @@ -24,7 +24,7 @@ services: env_file: ../apps/mysql/auth.env elasticsearch: - image: {{ DOCKER_REGISTRY }}elasticsearch:1.5.2 + image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_ELASTICSEARCH }} environment: - "ES_JAVA_OPTS=-Xms1g -Xmx1g" - "cluster.name=openedx" @@ -44,7 +44,7 @@ services: command: bash -c "rm -rf /var/www/openedx/staticfiles && cp -r /openedx/staticfiles/ /var/www/openedx/" nginx: - image: {{ DOCKER_REGISTRY }}nginx:1.13 + image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_NGINX }} restart: unless-stopped ports: - "{{ NGINX_HTTP_PORT }}:80" @@ -61,14 +61,14 @@ services: {% if ACTIVATE_NOTES %}- notes{% endif %} rabbitmq: - image: {{ DOCKER_REGISTRY }}rabbitmq:3.6.10 + image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_RABBITMQ }} volumes: - ../../data/rabbitmq:/var/lib/rabbitmq restart: unless-stopped # Simple SMTP server smtp: - image: {{ DOCKER_REGISTRY }}namshi/smtp:latest + image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_NAMSHI }} restart: unless-stopped ############# Forum