Make it possible to override the docker registry for individual services

Previously, it was not possible to override the docker registry for just
one or a few services. Setting the DOCKER_REGISTRY configuration
parameter would apply to all images. This was inconvenient. To resolve
this, we include the docker registry value in the DOCKER_IMAGE_*
configuration parameters. This allows users to override the docker
registry individually by defining the DOCKER_IMAGE_SERVICENAME
configuration parameter.

See https://discuss.overhang.io/t/kubernetes-ci-cd-pipeline/765/3

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 Note: Breaking changes between versions are indicated by "💥".
 
+## Unreleased
+
+- 💥[Feature] Make it possible to override the docker registry for just a few services by setting `DOCKER_IMAGE_SERVICENAME` values.
+
 ## v10.0.11 (2020-07-16)
 
 - [Feature] Upgrade all repositories to open-release/juniper.2

docs/configuration.rst

@@ -61,9 +61,9 @@ Docker
 Custom images
 *************
 
-- ``DOCKER_IMAGE_OPENEDX`` (default: ``"overhangio/openedx:{{ TUTOR_VERSION }}"``)
-- ``DOCKER_IMAGE_ANDROID`` (default: ``"overhangio/openedx-android:{{ TUTOR_VERSION }}"``)
-- ``DOCKER_IMAGE_FORUM`` (default: ``"overhangio/openedx-forum:{{ TUTOR_VERSION }}"``)
+- ``DOCKER_IMAGE_OPENEDX`` (default: ``"{{ DOCKER_REGISTRY }}overhangio/openedx:{{ TUTOR_VERSION }}"``)
+- ``DOCKER_IMAGE_ANDROID`` (default: ``"{{ DOCKER_REGISTRY }}overhangio/openedx-android:{{ TUTOR_VERSION }}"``)
+- ``DOCKER_IMAGE_FORUM`` (default: ``"{{ DOCKER_REGISTRY }}overhangio/openedx-forum:{{ TUTOR_VERSION }}"``)
 
 These configuration parameters define which image to run for each service. By default, the docker image tag matches the Tutor version it was built with.
 
@@ -309,7 +309,7 @@ With django.po containing::
     msgstr "Ça marche ! Propulsé by Open edX{registered_trademark}"
 
 And djangojs.po::
-    
+
     msgid "%(num_points)s point possible (graded, results hidden)"
     msgid_plural "%(num_points)s points possible (graded, results hidden)"
     msgstr[0] "%(num_points)s point possible (noté, résultats cachés)"
@@ -320,14 +320,14 @@ Then you will have to re-build the openedx Docker image::
     tutor images build openedx openedx-dev
 
 Beware that this will take a long time! Unfortunately it's difficult to accelerate this process, as translation files need to be compiled prior to collecting the assets. In development it's possible to accelerate the iteration loop -- but that exercise is left to the reader.
-    
+
 
 Running a different ``openedx`` Docker image
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 By default, Tutor runs the `overhangio/openedx <https://hub.docker.com/r/overhangio/openedx/>`_ docker image from Docker Hub. If you have an account on `hub.docker.com <https://hub.docker.com>`_ or you have a private image registry, you can build your image and push it to your registry with::
 
-    tutor config save --set DOCKER_IMAGE_OPENEDX=myusername/openedx:mytag
+    tutor config save --set DOCKER_IMAGE_OPENEDX=docker.io/myusername/openedx:mytag
     tutor images build openedx
     tutor images push openedx
 

docs/plugins/api.rst

@@ -17,7 +17,7 @@ The ``config`` attribute is used to modify existing and add new configuration pa
  "set" and "default" key names will be automatically prefixed with the plugin name, in upper case.
 
 Example::
-  
+
     config = {
         "add": {
             "SECRET_KEY": "{{ 8|random_string }}"

tutor/commands/images.py

@@ -95,7 +95,7 @@ def pull_image(config, image):
     for _plugin, hook in plugins.iter_hooks(config, "remote-image"):
         for img, tag in hook.items():
             if image in [img, "all"]:
-                tag = config["DOCKER_REGISTRY"] + tutor_env.render_str(config, tag)
+                tag = tutor_env.render_str(config, tag)
                 images.pull(tag)
 
 
@@ -119,7 +119,7 @@ def push_image(config, image):
     for _plugin, hook in plugins.iter_hooks(config, "remote-image"):
         for img, tag in hook.items():
             if image in [img, "all"]:
-                tag = config["DOCKER_REGISTRY"] + tutor_env.render_str(config, tag)
+                tag = tutor_env.render_str(config, tag)
                 images.push(tag)
 
 

tutor/images.py

@@ -3,8 +3,7 @@ from . import utils
 
 
 def get_tag(config, name):
-    image = config["DOCKER_IMAGE_" + name.upper().replace("-", "_")]
-    return "{registry}{image}".format(registry=config["DOCKER_REGISTRY"], image=image)
+    return config["DOCKER_IMAGE_" + name.upper().replace("-", "_")]
 
 
 def build(path, tag, no_cache=False, build_args=None):

tutor/templates/build/openedx-dev/Dockerfile

@@ -1,4 +1,4 @@
-FROM {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_OPENEDX }}
+FROM {{ DOCKER_IMAGE_OPENEDX }}
 MAINTAINER Overhang.io <contact@overhang.io>
 
 # Install useful system requirements

tutor/templates/config.yml

@@ -28,18 +28,18 @@ ANDROID_RELEASE_STORE_PASSWORD: "android store password"
 ANDROID_RELEASE_KEY_PASSWORD: "android release key password"
 ANDROID_RELEASE_KEY_ALIAS: "android release key alias"
 DEV_PROJECT_NAME: "tutor_dev"
-DOCKER_IMAGE_OPENEDX: "overhangio/openedx:{{ TUTOR_VERSION }}"
-DOCKER_IMAGE_OPENEDX_DEV: "overhangio/openedx-dev:{{ TUTOR_VERSION }}"
-DOCKER_IMAGE_ANDROID: "overhangio/openedx-android:{{ TUTOR_VERSION }}"
-DOCKER_IMAGE_FORUM: "overhangio/openedx-forum:{{ TUTOR_VERSION }}"
-DOCKER_IMAGE_MEMCACHED: "memcached:1.4.38"
-DOCKER_IMAGE_MONGODB: "mongo:3.6.18"
-DOCKER_IMAGE_MYSQL: "mysql:5.6.36"
-DOCKER_IMAGE_ELASTICSEARCH: "elasticsearch:1.5.2"
-DOCKER_IMAGE_NGINX: "nginx:1.13"
-DOCKER_IMAGE_RABBITMQ: "rabbitmq:3.6.10-management-alpine"
-DOCKER_IMAGE_SMTP: "namshi/smtp:latest"
 DOCKER_REGISTRY: "docker.io/"
+DOCKER_IMAGE_OPENEDX: "{{ DOCKER_REGISTRY }}overhangio/openedx:{{ TUTOR_VERSION }}"
+DOCKER_IMAGE_OPENEDX_DEV: "{{ DOCKER_REGISTRY }}overhangio/openedx-dev:{{ TUTOR_VERSION }}"
+DOCKER_IMAGE_ANDROID: "{{ DOCKER_REGISTRY }}overhangio/openedx-android:{{ TUTOR_VERSION }}"
+DOCKER_IMAGE_FORUM: "{{ DOCKER_REGISTRY }}overhangio/openedx-forum:{{ TUTOR_VERSION }}"
+DOCKER_IMAGE_MEMCACHED: "{{ DOCKER_REGISTRY }}memcached:1.4.38"
+DOCKER_IMAGE_MONGODB: "{{ DOCKER_REGISTRY }}mongo:3.6.18"
+DOCKER_IMAGE_MYSQL: "{{ DOCKER_REGISTRY }}mysql:5.6.36"
+DOCKER_IMAGE_ELASTICSEARCH: "{{ DOCKER_REGISTRY }}elasticsearch:1.5.2"
+DOCKER_IMAGE_NGINX: "{{ DOCKER_REGISTRY }}nginx:1.13"
+DOCKER_IMAGE_RABBITMQ: "{{ DOCKER_REGISTRY }}rabbitmq:3.6.10-management-alpine"
+DOCKER_IMAGE_SMTP: "{{ DOCKER_REGISTRY }}namshi/smtp:latest"
 LOCAL_PROJECT_NAME: "tutor_local"
 ELASTICSEARCH_HOST: "elasticsearch"
 ELASTICSEARCH_PORT: 9200

tutor/templates/dev/docker-compose.yml

@@ -2,7 +2,7 @@ version: "3.7"
 
 x-openedx-service:
   &openedx-service
-  image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_OPENEDX_DEV }}
+  image: {{ DOCKER_IMAGE_OPENEDX_DEV }}
   environment:
     SETTINGS: ${TUTOR_EDX_PLATFORM_SETTINGS:-tutor.development}
   volumes:

tutor/templates/k8s/deployments.yml

@@ -16,7 +16,7 @@ spec:
     spec:
       containers:
         - name: cms
-          image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_OPENEDX }}
+          image: {{ DOCKER_IMAGE_OPENEDX }}
           env:
           - name: SERVICE_VARIANT
             value: cms
@@ -60,7 +60,7 @@ spec:
     spec:
       containers:
         - name: cms-worker
-          image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_OPENEDX }}
+          image: {{ DOCKER_IMAGE_OPENEDX }}
           args: ["./manage.py", "cms", "celery", "worker", "--loglevel=info", "--hostname=edx.cms.core.default.%%h", "--maxtasksperchild", "100", "--exclude-queues=edx.lms.core.default"]
           env:
           - name: SERVICE_VARIANT
@@ -103,7 +103,7 @@ spec:
     spec:
       containers:
         - name: forum
-          image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_FORUM }}
+          image: {{ DOCKER_IMAGE_FORUM }}
           ports:
             - containerPort: 4567
           env:
@@ -134,7 +134,7 @@ spec:
     spec:
       containers:
         - name: lms
-          image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_OPENEDX }}
+          image: {{ DOCKER_IMAGE_OPENEDX }}
           ports:
             - containerPort: 8000
           volumeMounts:
@@ -175,7 +175,7 @@ spec:
     spec:
       containers:
         - name: lms-worker
-          image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_OPENEDX }}
+          image: {{ DOCKER_IMAGE_OPENEDX }}
           args: ["./manage.py", "lms", "celery", "worker", "--loglevel=info", "--hostname=edx.lms.core.default.%%h", "--maxtasksperchild", "100", "--exclude-queues=edx.cms.core.default"]
           env:
           - name: SERVICE_VARIANT
@@ -220,7 +220,7 @@ spec:
     spec:
       containers:
         - name: elasticsearch
-          image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_ELASTICSEARCH }}
+          image: {{ DOCKER_IMAGE_ELASTICSEARCH }}
           env:
             - name: ES_JAVA_OPTS
               value: "-Xms1g -Xmx1g"
@@ -257,7 +257,7 @@ spec:
     spec:
       containers:
         - name: memcached
-          image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_MEMCACHED }}
+          image: {{ DOCKER_IMAGE_MEMCACHED }}
           ports:
             - containerPort: 11211
 {% endif %}
@@ -282,7 +282,7 @@ spec:
     spec:
       containers:
         - name: mongodb
-          image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_MONGODB }}
+          image: {{ DOCKER_IMAGE_MONGODB }}
           args: ["mongod", "--smallfiles", "--nojournal", "--storageEngine", "wiredTiger"]
           ports:
             - containerPort: 27017
@@ -316,7 +316,7 @@ spec:
     spec:
       containers:
         - name: mysql
-          image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_MYSQL }}
+          image: {{ DOCKER_IMAGE_MYSQL }}
           args: ["mysqld", "--character-set-server=utf8", "--collation-server=utf8_general_ci"]
           env:
             - name: MYSQL_ROOT_PASSWORD
@@ -350,7 +350,7 @@ spec:
     spec:
       containers:
         - name: smtp
-          image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_SMTP }}
+          image: {{ DOCKER_IMAGE_SMTP }}
           ports:
             - containerPort: 25
 {% endif %}
@@ -372,13 +372,13 @@ spec:
     spec:
       initContainers:
         - name: clean-openedx-staticfiles
-          image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_OPENEDX }}
+          image: {{ DOCKER_IMAGE_OPENEDX }}
           args: ['rm', '-rf', '/var/www/openedx/staticfiles']
           volumeMounts:
             - mountPath: /var/www/openedx/
               name: openedx-staticfiles
         - name: init-openedx-staticfiles
-          image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_OPENEDX }}
+          image: {{ DOCKER_IMAGE_OPENEDX }}
           args: ['cp', '-r', '/openedx/staticfiles', '/var/www/openedx/']
           volumeMounts:
             - mountPath: /var/www/openedx/
@@ -386,7 +386,7 @@ spec:
         {{ patch("k8s-deployments-nginx-init-containers")|indent(8) }}
       containers:
         - name: nginx
-          image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_NGINX }}
+          image: {{ DOCKER_IMAGE_NGINX }}
           volumeMounts:
             - mountPath: /etc/nginx/conf.d/
               name: config
@@ -435,7 +435,7 @@ spec:
     spec:
       containers:
         - name: rabbitmq
-          image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_RABBITMQ }}
+          image: {{ DOCKER_IMAGE_RABBITMQ }}
           ports:
             - containerPort: 5672
           volumeMounts:

tutor/templates/k8s/jobs.yml

@@ -11,7 +11,7 @@ spec:
       restartPolicy: Never
       containers:
       - name: lms
-        image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_OPENEDX }}
+        image: {{ DOCKER_IMAGE_OPENEDX }}
         volumeMounts:
           - mountPath: /openedx/edx-platform/lms/envs/tutor/
             name: settings-lms
@@ -42,7 +42,7 @@ spec:
       restartPolicy: Never
       containers:
       - name: cms
-        image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_OPENEDX }}
+        image: {{ DOCKER_IMAGE_OPENEDX }}
         env:
         - name: SERVICE_VARIANT
           value: cms
@@ -76,7 +76,7 @@ spec:
       restartPolicy: Never
       containers:
       - name: mysql
-        image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_MYSQL }}
+        image: {{ DOCKER_IMAGE_MYSQL }}
         command: []
 ---
 apiVersion: batch/v1
@@ -91,7 +91,7 @@ spec:
       restartPolicy: Never
       containers:
       - name: forum
-        image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_FORUM }}
+        image: {{ DOCKER_IMAGE_FORUM }}
         env:
           - name: SEARCH_SERVER
             value: "{{ ELASTICSEARCH_SCHEME }}://{{ ELASTICSEARCH_HOST }}:{{ ELASTICSEARCH_PORT }}"

tutor/templates/local/docker-compose.jobs.yml

@@ -2,13 +2,13 @@ version: "3.7"
 services:
 
     mysql-job:
-      image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_MYSQL }}
+      image: {{ DOCKER_IMAGE_MYSQL }}
       entrypoint: []
       command: ["echo", "done"]
       depends_on: {{ [("mysql", ACTIVATE_MYSQL)]|list_if }}
-    
+
     lms-job:
-      image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_OPENEDX }}
+      image: {{ DOCKER_IMAGE_OPENEDX }}
       environment:
         SERVICE_VARIANT: lms
         SETTINGS: ${EDX_PLATFORM_SETTINGS:-tutor.production}
@@ -16,9 +16,9 @@ services:
         - ../apps/openedx/settings/lms/:/openedx/edx-platform/lms/envs/tutor/:ro
         - ../apps/openedx/settings/cms/:/openedx/edx-platform/cms/envs/tutor/:ro
         - ../apps/openedx/config/:/openedx/config/:ro
-    
+
     cms-job:
-      image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_OPENEDX }}
+      image: {{ DOCKER_IMAGE_OPENEDX }}
       environment:
         SERVICE_VARIANT: cms
         SETTINGS: ${EDX_PLATFORM_SETTINGS:-tutor.production}
@@ -26,13 +26,13 @@ services:
         - ../apps/openedx/settings/lms/:/openedx/edx-platform/lms/envs/tutor/:ro
         - ../apps/openedx/settings/cms/:/openedx/edx-platform/cms/envs/tutor/:ro
         - ../apps/openedx/config/:/openedx/config/:ro
-    
+
     forum-job:
-      image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_FORUM }}
+      image: {{ DOCKER_IMAGE_FORUM }}
       environment:
         SEARCH_SERVER: "{{ ELASTICSEARCH_SCHEME }}://{{ ELASTICSEARCH_HOST }}:{{ ELASTICSEARCH_PORT }}"
         MONGODB_AUTH: "{% if MONGODB_USERNAME and MONGODB_PASSWORD %}{{ MONGODB_USERNAME}}:{{ MONGODB_PASSWORD }}@{% endif %}"
         MONGODB_HOST: "{{ MONGODB_HOST }}"
         MONGODB_PORT: "{{ MONGODB_PORT }}"
-    
+
     {{ patch("local-docker-compose-jobs-services")|indent(4) }}

tutor/templates/local/docker-compose.prod.yml

@@ -1,13 +1,13 @@
 version: "3.7"
 services:
   openedx-assets:
-    image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_OPENEDX }}
+    image: {{ DOCKER_IMAGE_OPENEDX }}
     volumes:
       - ../../data/openedx:/var/www/openedx
     command: sh -c "rm -rf /var/www/openedx/staticfiles && cp -r /openedx/staticfiles/ /var/www/openedx/"
 
   nginx:
-    image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_NGINX }}
+    image: {{ DOCKER_IMAGE_NGINX }}
     restart: unless-stopped
     ports:
       - "{{ NGINX_HTTP_PORT }}:80"

tutor/templates/local/docker-compose.yml

@@ -5,13 +5,13 @@ services:
 
   {% if ACTIVATE_MEMCACHED %}
   memcached:
-    image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_MEMCACHED }}
+    image: {{ DOCKER_IMAGE_MEMCACHED }}
     restart: unless-stopped
   {% endif %}
 
   {% if ACTIVATE_MONGODB %}
   mongodb:
-    image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_MONGODB }}
+    image: {{ DOCKER_IMAGE_MONGODB }}
     # Use WiredTiger in all environments, just like at edx.org
     command: mongod --smallfiles --nojournal --storageEngine wiredTiger
     restart: unless-stopped
@@ -21,7 +21,7 @@ services:
 
   {% if ACTIVATE_MYSQL %}
   mysql:
-    image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_MYSQL }}
+    image: {{ DOCKER_IMAGE_MYSQL }}
     command: mysqld --character-set-server=utf8 --collation-server=utf8_general_ci
     restart: unless-stopped
     volumes:
@@ -32,7 +32,7 @@ services:
 
   {% if ACTIVATE_ELASTICSEARCH %}
   elasticsearch:
-    image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_ELASTICSEARCH }}
+    image: {{ DOCKER_IMAGE_ELASTICSEARCH }}
     command: ["elasticsearch", "-Xms{{ ELASTICSEARCH_HEAP_SIZE }}", "-Xmx{{ ELASTICSEARCH_HEAP_SIZE }}", "--cluster.name=openedx", "--bootstrap.mlockall=true"]
     ulimits:
       memlock:
@@ -45,7 +45,7 @@ services:
 
   {% if ACTIVATE_RABBITMQ %}
   rabbitmq:
-    image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_RABBITMQ }}
+    image: {{ DOCKER_IMAGE_RABBITMQ }}
     volumes:
       - ../../data/rabbitmq:/var/lib/rabbitmq
     restart: unless-stopped
@@ -53,7 +53,7 @@ services:
 
   {% if ACTIVATE_SMTP %}
   smtp:
-    image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_SMTP }}
+    image: {{ DOCKER_IMAGE_SMTP }}
     restart: unless-stopped
   {% endif %}
 
@@ -61,7 +61,7 @@ services:
 
   {% if ACTIVATE_FORUM %}
   forum:
-    image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_FORUM }}
+    image: {{ DOCKER_IMAGE_FORUM }}
     environment:
       SEARCH_SERVER: "{{ ELASTICSEARCH_SCHEME }}://{{ ELASTICSEARCH_HOST }}:{{ ELASTICSEARCH_PORT }}"
       MONGODB_AUTH: "{% if MONGODB_USERNAME and MONGODB_PASSWORD %}{{ MONGODB_USERNAME}}:{{ MONGODB_PASSWORD }}@{% endif %}"
@@ -75,7 +75,7 @@ services:
 
   {% if ACTIVATE_LMS %}
   lms:
-    image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_OPENEDX }}
+    image: {{ DOCKER_IMAGE_OPENEDX }}
     environment:
       SERVICE_VARIANT: lms
       GUNICORN_WORKERS: {{ OPENEDX_LMS_GUNICORN_WORKERS }}
@@ -100,7 +100,7 @@ services:
 
   {% if ACTIVATE_CMS %}
   cms:
-    image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_OPENEDX }}
+    image: {{ DOCKER_IMAGE_OPENEDX }}
     environment:
       SERVICE_VARIANT: cms
       GUNICORN_WORKERS: {{ OPENEDX_CMS_GUNICORN_WORKERS }}
@@ -127,7 +127,7 @@ services:
 
   {% if ACTIVATE_LMS %}
   lms-worker:
-    image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_OPENEDX }}
+    image: {{ DOCKER_IMAGE_OPENEDX }}
     environment:
       SERVICE_VARIANT: lms
       SETTINGS: ${EDX_PLATFORM_SETTINGS:-tutor.production}
@@ -146,7 +146,7 @@ services:
 
   {% if ACTIVATE_CMS %}
   cms-worker:
-    image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_OPENEDX }}
+    image: {{ DOCKER_IMAGE_OPENEDX }}
     environment:
       SERVICE_VARIANT: cms
       SETTINGS: ${EDX_PLATFORM_SETTINGS:-tutor.production}