From bccbd58e20e9b99bc4ce39fbe55abf9a84d5b77c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Mon, 18 Mar 2019 21:53:18 +0100 Subject: [PATCH] Easily customise the docker registry As a consequence, `images` commands lose the `--namespace/--version` arguments, which were confusing anyway. --- CHANGELOG.md | 1 + docs/configuration.rst | 9 ++++ docs/customise.rst | 13 ++---- tutor/images.py | 58 ++++++------------------ tutor/templates/config-defaults.yml | 1 + tutor/templates/k8s/deployments.yml | 22 ++++----- tutor/templates/local/docker-compose.yml | 32 ++++++------- 7 files changed, 57 insertions(+), 79 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a3f836..d009c52 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Latest +- [Feature] Customize docker registry - [Feature] Load configuration parameters from the system environment - [Improvement] Automatic environment re-generation after re-configuration - [Improvement] Error and interrupt handling in UI and web UI diff --git a/docs/configuration.rst b/docs/configuration.rst index 4210bab..bfdd533 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -38,6 +38,15 @@ These configuration parameters define which image to run for each service. - ``DOCKER_IMAGE_NOTES`` (default: ``regis/openedx-notes:hawthorn``) - ``DOCKER_IMAGE_XQUEUE`` (default: ``regis/openedx-xqueue:hawthorn``) +``DOCKER_REGISTRY`` Custom Docker registry +------------------------------------------ + +You may want to pull/push images from/to a custom docker registry. For instance, for a registry running on ``localhost:5000``, define:: + + DOCKER_REGISTRY: localhost:5000/ + +(the trailing ``/`` is important) + ``ACTIVATE_*`` Optional features -------------------------------- diff --git a/docs/customise.rst b/docs/customise.rst index 31371e8..8916f73 100644 --- a/docs/customise.rst +++ b/docs/customise.rst @@ -73,15 +73,12 @@ Note that your release must be a fork of Hawthorn in order to work. Otherwise, y Running a different ``openedx`` Docker image -------------------------------------------- -By default, Tutor runs the `regis/openedx `_ docker image from Docker Hub. If you have an account on `hub.docker.com `_ or you have a private image registry, you can build your image and push it to your registry:: - - tutor images build openedx --namespace=myusername --version=mytag - tutor images push openedx --namespace=myusername --version=mytag - -Then, set the following configuration parameter:: +By default, Tutor runs the `regis/openedx `_ docker image from Docker Hub. If you have an account on `hub.docker.com `_ or you have a private image registry, you can build your image and push it to your registry with:: tutor config save --silent --set DOCKER_IMAGE_OPENEDX=myusername/openedx:mytag + tutor images build openedx + tutor images push openedx -See the relevant :ref:`configuration parameters `. +(See the relevant :ref:`configuration parameters `.) -This value will then be used by Tutor to run the platform, for instance when running ``tutor local quickstart``. +The customised Docker image tag value will then be used by Tutor to run the platform, for instance when running ``tutor local quickstart``. diff --git a/tutor/images.py b/tutor/images.py index 1669812..9a74fc4 100644 --- a/tutor/images.py +++ b/tutor/images.py @@ -10,44 +10,25 @@ from . import utils def images(): pass -option_namespace = click.option("-n", "--namespace", default="regis", show_default=True) -option_version = click.option("-V", "--version", default="hawthorn", show_default=True) all_images = ["openedx", "forum", "notes", "xqueue", "android"] argument_image = click.argument( "image", type=click.Choice(["all"] + all_images), ) -@click.command( - short_help="Download docker images", - help=("""Download the docker images from hub.docker.com. - The images will come from {namespace}/{image}:{version}.""") -) -@opts.root -@option_namespace -@option_version -@argument_image -def download(root, namespace, version, image): - config = tutor_config.load(root) - for image in image_list(config, image): - utils.docker('image', 'pull', get_tag(namespace, image, version)) - @click.command( short_help="Build docker images", - help=("""Build the docker images necessary for an Open edX platform. - The images will be tagged as {namespace}/{image}:{version}.""") + help="Build the docker images necessary for an Open edX platform." ) @opts.root -@option_namespace -@option_version @argument_image @click.option( "-a", "--build-arg", multiple=True, help="Set build-time docker ARGS in the form 'myarg=value'. This option may be specified multiple times." ) -def build(root, namespace, version, image, build_arg): +def build(root, image, build_arg): config = tutor_config.load(root) for image in image_list(config, image): - tag = get_tag(namespace, image, version) + tag = get_tag(config, image) click.echo(fmt.info("Building image {}".format(tag))) command = [ "build", "-t", tag, @@ -59,41 +40,31 @@ def build(root, namespace, version, image, build_arg): ] utils.docker(*command) -@click.command( - short_help="Pull images from hub.docker.com", -) +@click.command(short_help="Pull images from the docker registry") @opts.root -@option_namespace -@option_version @argument_image -def pull(root, namespace, version, image): +def pull(root, image): config = tutor_config.load(root) for image in image_list(config, image): - tag = get_tag(namespace, image, version) + tag = get_tag(config, image) click.echo(fmt.info("Pulling image {}".format(tag))) utils.execute("docker", "pull", tag) -@click.command( - short_help="Push images to hub.docker.com", -) +@click.command(short_help="Push images to the Docker registry") @opts.root -@option_namespace -@option_version @argument_image -def push(root, namespace, version, image): +def push(root, image): config = tutor_config.load(root) for image in image_list(config, image): - tag = get_tag(namespace, image, version) + tag = get_tag(config, image) click.echo(fmt.info("Pushing image {}".format(tag))) utils.execute("docker", "push", tag) -def get_tag(namespace, image, version): - name = "openedx" if image == "openedx" else "openedx-{}".format(image) - return "{namespace}{sep}{image}:{version}".format( - namespace=namespace, - sep="/" if namespace else "", - image=name, - version=version, +def get_tag(config, name): + image = config["DOCKER_IMAGE_" + name.upper()] + return "{registry}{image}".format( + registry=config["DOCKER_REGISTRY"], + image=image, ) def image_list(config, image): @@ -106,7 +77,6 @@ def image_list(config, image): return images return [image] -images.add_command(download) images.add_command(build) images.add_command(pull) images.add_command(push) diff --git a/tutor/templates/config-defaults.yml b/tutor/templates/config-defaults.yml index 168b981..5fc8751 100644 --- a/tutor/templates/config-defaults.yml +++ b/tutor/templates/config-defaults.yml @@ -4,6 +4,7 @@ DOCKER_IMAGE_ANDROID: "regis/openedx-android:hawthorn" DOCKER_IMAGE_FORUM: "regis/openedx-forum:hawthorn" DOCKER_IMAGE_NOTES: "regis/openedx-notes:hawthorn" DOCKER_IMAGE_XQUEUE: "regis/openedx-xqueue:hawthorn" +DOCKER_REGISTRY: "" MYSQL_DATABASE: "openedx" MYSQL_USERNAME: "openedx" MONGODB_DATABASE: "openedx" diff --git a/tutor/templates/k8s/deployments.yml b/tutor/templates/k8s/deployments.yml index 8b97574..533e114 100644 --- a/tutor/templates/k8s/deployments.yml +++ b/tutor/templates/k8s/deployments.yml @@ -15,7 +15,7 @@ spec: spec: containers: - name: cms - image: {{ DOCKER_IMAGE_OPENEDX }} + image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_OPENEDX }} env: - name: SERVICE_VARIANT value: cms @@ -62,7 +62,7 @@ spec: spec: containers: - name: forum - image: {{ DOCKER_IMAGE_FORUM }} + image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_FORUM }} ports: - containerPort: 4567 imagePullPolicy: Always @@ -84,7 +84,7 @@ spec: spec: containers: - name: lms - image: {{ DOCKER_IMAGE_OPENEDX }} + image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_OPENEDX }} ports: - containerPort: 8000 volumeMounts: @@ -128,7 +128,7 @@ spec: spec: containers: - name: elasticsearch - image: elasticsearch:1.5.2 + image: {{ DOCKER_REGISTRY }}elasticsearch:1.5.2 env: - name: ES_JAVA_OPTS value: "-Xms1g -Xmx1g" @@ -163,7 +163,7 @@ spec: spec: containers: - name: memcached - image: memcached:1.4.38 + image: {{ DOCKER_REGISTRY }}memcached:1.4.38 ports: - containerPort: 11211 @@ -184,7 +184,7 @@ spec: spec: containers: - name: mongodb - image: mongo:3.2.16 + image: {{ DOCKER_REGISTRY }}mongo:3.2.16 command: ["mongod", "--smallfiles", "--nojournal", "--storageEngine", "wiredTiger"] ports: - containerPort: 27017 @@ -212,7 +212,7 @@ spec: spec: containers: - name: mysql - image: mysql:5.6.36 + image: {{ DOCKER_REGISTRY }}mysql:5.6.36 env: - name: MYSQL_ROOT_PASSWORD valueFrom: @@ -246,14 +246,14 @@ spec: spec: initContainers: - name: clean-openedx-staticfiles - image: {{ DOCKER_IMAGE_OPENEDX }} + image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_OPENEDX }} command: ['rm', '-rf', '/var/www/openedx/staticfiles'] volumeMounts: - mountPath: /var/www/openedx/ name: openedx-staticfiles imagePullPolicy: Always - name: init-openedx-staticfiles - image: {{ DOCKER_IMAGE_OPENEDX }} + image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_OPENEDX }} command: ['cp', '-r', '/openedx/staticfiles', '/var/www/openedx/'] volumeMounts: - mountPath: /var/www/openedx/ @@ -261,7 +261,7 @@ spec: imagePullPolicy: Always containers: - name: nginx - image: nginx:1.13 + image: {{ DOCKER_REGISTRY }}nginx:1.13 volumeMounts: - mountPath: /etc/nginx/conf.d/ name: config @@ -302,7 +302,7 @@ spec: spec: containers: - name: rabbitmq - image: rabbitmq:3.6.10 + image: {{ DOCKER_REGISTRY }}rabbitmq:3.6.10 ports: - containerPort: 5672 volumeMounts: diff --git a/tutor/templates/local/docker-compose.yml b/tutor/templates/local/docker-compose.yml index 415fc24..5a1f793 100644 --- a/tutor/templates/local/docker-compose.yml +++ b/tutor/templates/local/docker-compose.yml @@ -4,11 +4,11 @@ services: ############# External services memcached: - image: memcached:1.4.38 + image: {{ DOCKER_REGISTRY }}memcached:1.4.38 restart: unless-stopped mongodb: - image: mongo:3.2.16 + image: {{ DOCKER_REGISTRY }}mongo:3.2.16 # Use WiredTiger in all environments, just like at edx.org command: mongod --smallfiles --nojournal --storageEngine wiredTiger restart: unless-stopped @@ -16,7 +16,7 @@ services: - ../../data/mongodb:/data/db mysql: - image: mysql:5.6.36 + image: {{ DOCKER_REGISTRY }}mysql:5.6.36 command: mysqld --character-set-server=utf8 --collation-server=utf8_general_ci restart: unless-stopped volumes: @@ -24,7 +24,7 @@ services: env_file: ../apps/mysql/auth.env elasticsearch: - image: elasticsearch:1.5.2 + image: {{ DOCKER_REGISTRY }}elasticsearch:1.5.2 environment: - "ES_JAVA_OPTS=-Xms1g -Xmx1g" - "cluster.name=openedx" @@ -38,13 +38,13 @@ services: - ../../data/elasticsearch:/usr/share/elasticsearch/data openedx-assets: - image: {{ DOCKER_IMAGE_OPENEDX }} + image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_OPENEDX }} volumes: - ../../data/openedx:/var/www/openedx command: bash -c "rm -rf /var/www/openedx/staticfiles && cp -r /openedx/staticfiles/ /var/www/openedx/" nginx: - image: nginx:1.13 + image: {{ DOCKER_REGISTRY }}nginx:1.13 restart: unless-stopped ports: - "{{ NGINX_HTTP_PORT }}:80" @@ -61,20 +61,20 @@ services: {% if ACTIVATE_NOTES %}- notes{% endif %} rabbitmq: - image: rabbitmq:3.6.10 + image: {{ DOCKER_REGISTRY }}rabbitmq:3.6.10 volumes: - ../../data/rabbitmq:/var/lib/rabbitmq restart: unless-stopped # Simple SMTP server smtp: - image: namshi/smtp:latest + image: {{ DOCKER_REGISTRY }}namshi/smtp:latest restart: unless-stopped ############# Forum forum: - image: {{ DOCKER_IMAGE_FORUM }} + image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_FORUM }} restart: unless-stopped depends_on: - elasticsearch @@ -83,7 +83,7 @@ services: ############# LMS and CMS lms: - image: {{ DOCKER_IMAGE_OPENEDX }} + image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_OPENEDX }} environment: SERVICE_VARIANT: lms SETTINGS: ${EDX_PLATFORM_SETTINGS:-tutor.production} @@ -103,7 +103,7 @@ services: - smtp cms: - image: {{ DOCKER_IMAGE_OPENEDX }} + image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_OPENEDX }} environment: SERVICE_VARIANT: cms SETTINGS: ${EDX_PLATFORM_SETTINGS:-tutor.production} @@ -125,7 +125,7 @@ services: # We could probably create one service per queue here. For small instances, it is not necessary. lms_worker: - image: {{ DOCKER_IMAGE_OPENEDX }} + image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_OPENEDX }} environment: SERVICE_VARIANT: lms SETTINGS: ${EDX_PLATFORM_SETTINGS:-tutor.production} @@ -141,7 +141,7 @@ services: - lms cms_worker: - image: {{ DOCKER_IMAGE_OPENEDX }} + image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_OPENEDX }} environment: SERVICE_VARIANT: cms SETTINGS: ${EDX_PLATFORM_SETTINGS:-tutor.production} @@ -159,7 +159,7 @@ services: {% if ACTIVATE_NOTES %} ############# Notes: backend store for edX Student Notes notes: - image: {{ DOCKER_IMAGE_NOTES }} + image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_NOTES }} networks: default: aliases: @@ -177,7 +177,7 @@ services: {% if ACTIVATE_XQUEUE %} ############# Xqueue: external grading of Open edX problems xqueue: - image: {{ DOCKER_IMAGE_XQUEUE }} + image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_XQUEUE }} volumes: - ../apps/xqueue/settings/tutor.py:/openedx/xqueue/xqueue/tutor.py - ../../data/xqueue:/openedx/data @@ -188,7 +188,7 @@ services: - mysql xqueue_consumer: - image: {{ DOCKER_IMAGE_XQUEUE }} + image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_XQUEUE }} volumes: - ../apps/xqueue/settings/tutor.py:/openedx/xqueue/xqueue/tutor.py - ../../data/xqueue:/openedx/data