Improve job running in local and k8s

Running jobs was previously done with "exec". This was because it
allowed us to avoid copying too much container specification information
from the docker-compose/deployments files to the jobs files. However,
this was limiting:

- In order to run a job, the corresponding container had to be running.
This was particularly painful in Kubernetes, where containers are
crashing as long as migrations are not correctly run.
- Containers in which we need to run jobs needed to be present in the
docker-compose/deployments files. This is unnecessary, for example when
mysql is disabled, or in the case of the certbot container.

Now, we create dedicated jobs files, both for local and k8s deployment.
This introduces a little redundancy, but not too much. Note that
dependent containers are not listed in the docker-compose.jobs.yml file,
so an actual platform is still supposed to be running when we launch the
jobs.

This also introduces a subtle change: now, jobs go through the container
entrypoint prior to running. This is probably a good thing, as it will
avoid forgetting about incorrect environment variables.

In k8s, we find ourselves interacting way too much with the kubectl
utility. Parsing output from the CLI is a pain. So we need to switch to
the native kubernetes client library.

CHANGELOG.md

@@ -5,6 +5,7 @@ Note: Breaking changes between versions are indicated by "💥".
 ## Unreleased
 
 - [Improvement] Fix tls certificate generation in k8s
+- [Improvement] Radically change the way jobs are run: we no longer "exec", but instead run a dedicated container.
 - [Improvement] Upgrade k8s certificate issuer to cert-manager.io/v1alpha2
 - [Feature] Add SCORM XBlock to default openedx docker image
 

docs/plugins/api.rst

@@ -83,6 +83,10 @@ Example::
     
 During initialisation, "myservice1" and "myservice2" will be run in sequence with the commands defined in the templates ``myplugin/hooks/myservice1/init`` and ``myplugin/hooks/myservice2/init``.
 
+To initialise a "foo" service, Tutor runs the "foo-job" service that is found in the ``env/local/docker-compose.jobs.yml`` file. By default, Tutor comes with a few services in this file: mysql-job, lms-job, cms-job, forum-job. If your plugin requires running custom services during initialisation, you will need to add them to the ``docker-compose.jobs.yml`` template. To do so, just use the "local-docker-compose-jobs-services" patch.
+
+In Kubernetes, the approach is the same, except that jobs are implemented as actual job objects in the ``k8s/jobs.yml`` template. To add your own services there, your plugin should implement the "k8s-jobs" patch.
+
 ``pre-init``
 ++++++++++++
 

requirements/base.in

@@ -2,4 +2,5 @@ appdirs
 click>=7.0
 click_repl
 jinja2>=2.9
+kubernetes
 pyyaml>=4.2b1

requirements/base.txt

@@ -5,11 +5,29 @@
 #    pip-compile requirements/base.in
 #
 appdirs==1.4.3
+cachetools==4.1.0         # via google-auth
+certifi==2020.4.5.1       # via kubernetes, requests
+chardet==3.0.4            # via requests
 click-repl==0.1.6
 click==7.1.1
+google-auth==1.14.0       # via kubernetes
+idna==2.9                 # via requests
 jinja2==2.11.1
+kubernetes==11.0.0
 markupsafe==1.1.1         # via jinja2
+oauthlib==3.1.0           # via requests-oauthlib
 prompt-toolkit==3.0.5     # via click-repl
+pyasn1-modules==0.2.8     # via google-auth
+pyasn1==0.4.8             # via pyasn1-modules, rsa
+python-dateutil==2.8.1    # via kubernetes
 pyyaml==5.3.1
-six==1.14.0               # via click-repl
+requests-oauthlib==1.3.0  # via kubernetes
+requests==2.23.0          # via kubernetes, requests-oauthlib
+rsa==4.0                  # via google-auth
+six==1.14.0               # via click-repl, google-auth, kubernetes, python-dateutil, websocket-client
+urllib3==1.25.9           # via kubernetes, requests
 wcwidth==0.1.9            # via prompt-toolkit
+websocket-client==0.57.0  # via kubernetes
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools

requirements/dev.txt

@@ -10,44 +10,54 @@ astroid==2.3.3            # via pylint
 attrs==19.3.0             # via black
 black==19.10b0
 bleach==3.1.4             # via readme-renderer
-certifi==2019.11.28       # via requests
+cachetools==4.1.0
+certifi==2020.4.5.1
 cffi==1.14.0              # via cryptography
-chardet==3.0.4            # via requests
+chardet==3.0.4
 click-repl==0.1.6
 click==7.1.1
 cryptography==2.8         # via secretstorage
 docutils==0.16            # via readme-renderer
-idna==2.9                 # via requests
+google-auth==1.14.0
+idna==2.9
 importlib-metadata==1.6.0  # via keyring, twine
 isort==4.3.21             # via pylint
 jeepney==0.4.3            # via keyring, secretstorage
 jinja2==2.11.1
 keyring==21.2.0           # via twine
+kubernetes==11.0.0
 lazy-object-proxy==1.4.3  # via astroid
 markupsafe==1.1.1
 mccabe==0.6.1             # via pylint
+oauthlib==3.1.0
 pathspec==0.7.0           # via black
 pip-tools==4.5.1
 pkginfo==1.5.0.1          # via twine
 prompt-toolkit==3.0.5
+pyasn1-modules==0.2.8
+pyasn1==0.4.8
 pycparser==2.20           # via cffi
 pygments==2.6.1           # via readme-renderer
 pyinstaller==3.6
 pylint==2.4.4
+python-dateutil==2.8.1
 pyyaml==5.3.1
 readme-renderer==25.0     # via twine
 regex==2020.2.20          # via black
+requests-oauthlib==1.3.0
 requests-toolbelt==0.9.1  # via twine
-requests==2.23.0          # via requests-toolbelt, twine
+requests==2.23.0
+rsa==4.0
 secretstorage==3.1.2      # via keyring
 six==1.14.0
 toml==0.10.0              # via black
 tqdm==4.44.1              # via twine
 twine==3.1.1
 typed-ast==1.4.1          # via astroid, black
-urllib3==1.25.8           # via requests
+urllib3==1.25.9
 wcwidth==0.1.9
 webencodings==0.5.1       # via bleach
+websocket-client==0.57.0
 wrapt==1.11.2             # via astroid
 zipp==3.1.0               # via importlib-metadata
 

requirements/docs.txt

@@ -7,22 +7,31 @@
 alabaster==0.7.12         # via sphinx
 appdirs==1.4.3
 babel==2.8.0              # via sphinx
-certifi==2019.11.28       # via requests
-chardet==3.0.4            # via requests
+cachetools==4.1.0
+certifi==2020.4.5.1
+chardet==3.0.4
 click-repl==0.1.6
 click==7.1.1
 docutils==0.16            # via sphinx
-idna==2.9                 # via requests
+google-auth==1.14.0
+idna==2.9
 imagesize==1.2.0          # via sphinx
 jinja2==2.11.1
+kubernetes==11.0.0
 markupsafe==1.1.1
+oauthlib==3.1.0
 packaging==20.3           # via sphinx
 prompt-toolkit==3.0.5
+pyasn1-modules==0.2.8
+pyasn1==0.4.8
 pygments==2.6.1           # via sphinx
 pyparsing==2.4.6          # via packaging
+python-dateutil==2.8.1
 pytz==2019.3              # via babel
 pyyaml==5.3.1
-requests==2.23.0          # via sphinx
+requests-oauthlib==1.3.0
+requests==2.23.0
+rsa==4.0
 six==1.14.0
 snowballstemmer==2.0.0    # via sphinx
 sphinx-rtd-theme==0.4.3
@@ -33,8 +42,9 @@ sphinxcontrib-htmlhelp==1.0.3  # via sphinx
 sphinxcontrib-jsmath==1.0.1  # via sphinx
 sphinxcontrib-qthelp==1.0.3  # via sphinx
 sphinxcontrib-serializinghtml==1.1.4  # via sphinx
-urllib3==1.25.8           # via requests
+urllib3==1.25.9
 wcwidth==0.1.9
+websocket-client==0.57.0
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools

tests/test_config.py

@@ -72,3 +72,9 @@ class ConfigTests(unittest.TestCase):
         self.assertNotIn("LMS_HOST", config)
         self.assertEqual("www.myopenedx.com", defaults["LMS_HOST"])
         self.assertEqual("studio.{{ LMS_HOST }}", defaults["CMS_HOST"])
+
+    def test_is_service_activated(self):
+        config = {"ACTIVATE_SERVICE1": True, "ACTIVATE_SERVICE2": False}
+
+        self.assertTrue(tutor_config.is_service_activated(config, "service1"))
+        self.assertFalse(tutor_config.is_service_activated(config, "service2"))

tests/test_scripts.py

@@ -1,14 +0,0 @@
-import unittest
-import unittest.mock
-
-from tutor import config as tutor_config
-from tutor import scripts
-
-
-class ScriptsTests(unittest.TestCase):
-    def test_is_activated(self):
-        config = {"ACTIVATE_SERVICE1": True, "ACTIVATE_SERVICE2": False}
-        runner = scripts.BaseRunner("/tmp", config)
-
-        self.assertTrue(runner.is_activated("service1"))
-        self.assertFalse(runner.is_activated("service2"))

tutor.spec

@@ -27,6 +27,7 @@ hidden_imports.append("Crypto.Hash.SHA256")
 hidden_imports.append("Crypto.PublicKey.RSA")
 hidden_imports.append("Crypto.Random")
 hidden_imports.append("Crypto.Signature.PKCS1_v1_5")
+hidden_imports.append("kubernetes")
 hidden_imports.append("uuid")
 
 

tutor/commands/compose.py

@@ -1,8 +1,10 @@
 import click
 
 from .. import config as tutor_config
+from .. import env as tutor_env
 from .. import fmt
 from .. import scripts
+from .. import serialize
 from .. import utils
 
 
@@ -11,10 +13,61 @@ class ScriptRunner(scripts.BaseRunner):
         super().__init__(root, config)
         self.docker_compose_func = docker_compose_func
 
-    def exec(self, service, command):
+    def run_job(self, service, command):
+        """
+        Run the "{{ service }}-job" service from local/docker-compose.jobs.yml with the
+        specified command. For backward-compatibility reasons, if the corresponding
+        service does not exist, run the service from good old regular
+        docker-compose.yml.
+        """
+        jobs_path = tutor_env.pathjoin(self.root, "local", "docker-compose.jobs.yml")
+        job_service_name = "{}-job".format(service)
         opts = [] if utils.is_a_tty() else ["-T"]
+        if job_service_name in serialize.load(open(jobs_path).read())["services"]:
             self.docker_compose_func(
-            self.root, self.config, "exec", *opts, service, "sh", "-e", "-c", command
+                self.root,
+                self.config,
+                "-f",
+                jobs_path,
+                "run",
+                *opts,
+                "--rm",
+                job_service_name,
+                "sh",
+                "-e",
+                "-c",
+                command,
+            )
+        else:
+            fmt.echo_alert(
+                (
+                    "The '{job_service_name}' service does not exist in {jobs_path}. "
+                    "This might be caused by an older plugin. Tutor switched to a job "
+                    "runner model for running one-time commands, such as database"
+                    " initialisation. For the record, this is the command that we are "
+                    "running:\n"
+                    "\n"
+                    "    {command}\n"
+                    "\n"
+                    "Old-style job running will be deprecated soon. Please inform "
+                    "your plugin maintainer!"
+                ).format(
+                    job_service_name=job_service_name,
+                    jobs_path=jobs_path,
+                    command=command.replace("\n", "\n    "),
+                )
+            )
+            self.docker_compose_func(
+                self.root,
+                self.config,
+                "run",
+                *opts,
+                "--rm",
+                service,
+                "sh",
+                "-e",
+                "-c",
+                command,
             )
 
 
@@ -73,7 +126,7 @@ def restart(context, services):
         pass
     else:
         for service in services:
-            if "openedx" == service:
+            if service == "openedx":
                 if config["ACTIVATE_LMS"]:
                     command += ["lms", "lms-worker"]
                 if config["ACTIVATE_CMS"]:
@@ -138,7 +191,7 @@ def run_hook(context, service, path):
     fmt.echo_info(
         "Running '{}' hook in '{}' container...".format(".".join(path), service)
     )
-    runner.run(service, *path)
+    runner.run_job_from_template(service, *path)
 
 
 @click.command(help="View output from containers")
@@ -171,11 +224,10 @@ def logs(context, follow, tail, service):
 def createuser(context, superuser, staff, password, name, email):
     config = tutor_config.load(context.root)
     runner = ScriptRunner(context.root, config, context.docker_compose)
-    runner.check_service_is_activated("lms")
     command = scripts.create_user_command(
         superuser, staff, name, email, password=password
     )
-    runner.exec("lms", command)
+    runner.run_job("lms", command)
 
 
 @click.command(

tutor/commands/dev.py

@@ -12,18 +12,17 @@ from .. import utils
 class DevContext(Context):
     @staticmethod
     def docker_compose(root, config, *command):
-        args = [
-            "-f",
-            tutor_env.pathjoin(root, "local", "docker-compose.yml"),
-        ]
-        override_path = tutor_env.pathjoin(root, "local", "docker-compose.override.yml")
-        if os.path.exists(override_path):
-            args += ["-f", override_path]
+        args = []
+        for folder in ["local", "dev"]:
+            # Add docker-compose.yml and docker-compose.override.yml (if it exists)
+            # from "local" and "dev" folders
             args += [
                 "-f",
-            tutor_env.pathjoin(root, "dev", "docker-compose.yml"),
+                tutor_env.pathjoin(root, folder, "docker-compose.yml"),
             ]
-        override_path = tutor_env.pathjoin(root, "dev", "docker-compose.override.yml")
+            override_path = tutor_env.pathjoin(
+                root, folder, "docker-compose.override.yml"
+            )
             if os.path.exists(override_path):
                 args += ["-f", override_path]
         return utils.docker_compose(

tutor/commands/k8s.py

@@ -1,10 +1,15 @@
+from datetime import datetime
+from time import sleep
+
 import click
 
 from .. import config as tutor_config
 from .. import env as tutor_env
+from .. import exceptions
 from .. import fmt
 from .. import interactive as interactive_config
 from .. import scripts
+from .. import serialize
 from .. import utils
 
 
@@ -47,6 +52,7 @@ def start(context):
         "app.kubernetes.io/component=namespace",
     )
     # Create volumes
+    # TODO: instead, we should use StatefulSets
     utils.kubectl(
         "apply",
         "--kustomize",
@@ -55,8 +61,14 @@ def start(context):
         "--selector",
         "app.kubernetes.io/component=volume",
     )
-    # Create everything else
-    utils.kubectl("apply", "--kustomize", tutor_env.pathjoin(context.root))
+    # Create everything else except jobs
+    utils.kubectl(
+        "apply",
+        "--kustomize",
+        tutor_env.pathjoin(context.root),
+        "--selector",
+        "app.kubernetes.io/component!=job",
+    )
 
 
 @click.command(help="Stop a running platform")
@@ -64,7 +76,9 @@ def start(context):
 def stop(context):
     config = tutor_config.load(context.root)
     utils.kubectl(
-        "delete", *resource_selector(config), "deployments,services,ingress,configmaps"
+        "delete",
+        *resource_selector(config),
+        "deployments,services,ingress,configmaps,jobs",
     )
 
 
@@ -108,7 +122,7 @@ def init(context):
     config = tutor_config.load(context.root)
     runner = K8sScriptRunner(context.root, config)
     for service in ["mysql", "elasticsearch", "mongodb"]:
-        if runner.is_activated(service):
+        if tutor_config.is_service_activated(config, service):
             wait_for_pod_ready(config, service)
     scripts.initialise(runner)
 
@@ -126,8 +140,6 @@ def init(context):
 @click.pass_obj
 def createuser(context, superuser, staff, password, name, email):
     config = tutor_config.load(context.root)
-    runner = K8sScriptRunner(context.root, config)
-    runner.check_service_is_activated("lms")
     command = scripts.create_user_command(
         superuser, staff, name, email, password=password
     )
@@ -189,24 +201,161 @@ def logs(context, container, follow, tail, service):
     utils.kubectl(*command)
 
 
+class K8sClients:
+    _instance = None
+
+    def __init__(self):
+        # Loading the kubernetes module here to avoid import overhead
+        from kubernetes import client, config  # pylint: disable=import-outside-toplevel
+
+        config.load_kube_config()
+        self._batch_api = None
+        self._core_api = None
+        self._client = client
+
+    @classmethod
+    def instance(cls):
+        if cls._instance is None:
+            cls._instance = cls()
+        return cls._instance
+
+    @property
+    def batch_api(self):
+        if self._batch_api is None:
+            self._batch_api = self._client.BatchV1Api()
+        return self._batch_api
+
+    @property
+    def core_api(self):
+        if self._core_api is None:
+            self._core_api = self._client.CoreV1Api()
+        return self._core_api
+
+
 class K8sScriptRunner(scripts.BaseRunner):
-    def exec(self, service, command):
-        kubectl_exec(self.config, service, command, attach=False)
+    def load_job(self, name):
+        jobs = self.render("k8s", "jobs.yml")
+        for job in serialize.load_all(jobs):
+            if job["metadata"]["name"] == name:
+                return job
+        raise ValueError("Could not find job '{}'".format(name))
+
+    def active_job_names(self):
+        """
+        Return a list of active job names
+        Docs:
+        https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#list-job-v1-batch
+        """
+        api = K8sClients.instance().batch_api
+        return [
+            job.metadata.name
+            for job in api.list_namespaced_job(self.config["K8S_NAMESPACE"]).items
+            if job.status.active
+        ]
+
+    def run_job(self, service, command):
+        job_name = "{}-job".format(service)
+        try:
+            job = self.load_job(job_name)
+        except ValueError:
+            message = (
+                "The '{job_name}' kubernetes job does not exist in the list of job "
+                "runners. This might be caused by an older plugin. Tutor switched to a"
+                " job runner model for running one-time commands, such as database"
+                " initialisation. For the record, this is the command that we are "
+                "running:\n"
+                "\n"
+                "    {command}\n"
+                "\n"
+                "Old-style job running will be deprecated soon. Please inform "
+                "your plugin maintainer!"
+            ).format(
+                job_name=job_name,
+                command=command.replace("\n", "\n    "),
+            )
+            fmt.echo_alert(message)
+            wait_for_pod_ready(self.config, service)
+            kubectl_exec(self.config, service, command)
+            return
+        # Create a unique job name to make it deduplicate jobs and make it easier to
+        # find later. Logs of older jobs will remain available for some time.
+        job_name += "-" + datetime.now().strftime("%Y%m%d%H%M%S")
+
+        # Wait until all other jobs are completed
+        while True:
+            active_jobs = self.active_job_names()
+            if not active_jobs:
+                break
+            fmt.echo_info(
+                "Waiting for active jobs to terminate: {}".format(" ".join(active_jobs))
+            )
+            sleep(5)
+
+        # Configure job
+        job["metadata"]["name"] = job_name
+        job["metadata"].setdefault("labels", {})
+        job["metadata"]["labels"]["app.kubernetes.io/name"] = job_name
+        job["spec"]["template"]["spec"]["containers"][0]["args"] = [
+            "sh",
+            "-e",
+            "-c",
+            command,
+        ]
+        job["spec"]["backoffLimit"] = 1
+        job["spec"]["ttlSecondsAfterFinished"] = 3600
+        # Save patched job to "jobs.yml" file
+        with open(tutor_env.pathjoin(self.root, "k8s", "jobs.yml"), "w") as job_file:
+            serialize.dump(job, job_file)
+        # We cannot use the k8s API to create the job: configMap and volume names need
+        # to be found with the right suffixes.
+        utils.kubectl(
+            "apply",
+            "--kustomize",
+            tutor_env.pathjoin(self.root),
+            "--selector",
+            "app.kubernetes.io/name={}".format(job_name),
+        )
+
+        message = (
+            "Job {job_name} is running. To view the logs from this job, run:\n\n"
+            """    kubectl logs --namespace={namespace} --follow $(kubectl get --namespace={namespace} pods """
+            """--selector=job-name={job_name} -o=jsonpath="{{.items[0].metadata.name}}")\n\n"""
+            "Waiting for job completion..."
+        ).format(job_name=job_name, namespace=self.config["K8S_NAMESPACE"])
+        fmt.echo_info(message)
+
+        # Wait for completion
+        field_selector = "metadata.name={}".format(job_name)
+        while True:
+            jobs = K8sClients.instance().batch_api.list_namespaced_job(
+                self.config["K8S_NAMESPACE"], field_selector=field_selector
+            )
+            if not jobs.items:
+                continue
+            job = jobs.items[0]
+            if not job.status.active:
+                if job.status.succeeded:
+                    fmt.echo_info("Job {} successful.".format(job_name))
+                    break
+                if job.status.failed:
+                    raise exceptions.TutorError(
+                        "Job {} failed. View the job logs to debug this issue.".format(
+                            job_name
+                        )
+                    )
+            sleep(5)
 
 
 def kubectl_exec(config, service, command, attach=False):
     selector = "app.kubernetes.io/name={}".format(service)
-
-    # Find pod in runner deployment
-    wait_for_pod_ready(config, service)
-    fmt.echo_info("Finding pod name for {} deployment...".format(service))
-    pod = utils.check_output(
-        "kubectl",
-        "get",
-        *resource_selector(config, selector),
-        "pods",
-        "-o=jsonpath={.items[0].metadata.name}",
+    pods = K8sClients.instance().core_api.list_namespaced_pod(
+        namespace=config["K8S_NAMESPACE"], label_selector=selector
     )
+    if not pods.items:
+        raise exceptions.TutorError(
+            "Could not find an active pod for the {} service".format(service)
+        )
+    pod_name = pods.items[0].metadata.name
 
     # Run command
     attach_opts = ["-i", "-t"] if attach else []
@@ -215,7 +364,7 @@ def kubectl_exec(config, service, command, attach=False):
         *attach_opts,
         "--namespace",
         config["K8S_NAMESPACE"],
-        pod.decode(),
+        pod_name,
         "--",
         "sh",
         "-e",

tutor/config.py

@@ -1,4 +1,3 @@
-import json
 import os
 
 from . import exceptions
@@ -128,6 +127,10 @@ def load_plugins(config, defaults):
             defaults[plugin.config_key(key)] = value
 
 
+def is_service_activated(config, service):
+    return config["ACTIVATE_" + service.upper()]
+
+
 def upgrade_obsolete(config):
     # Openedx-specific mysql passwords
     if "MYSQL_PASSWORD" in config:

tutor/scripts.py

@@ -1,5 +1,4 @@
 from . import env
-from . import exceptions
 from . import fmt
 from . import plugins
 
@@ -14,34 +13,23 @@ class BaseRunner:
         self.root = root
         self.config = config
 
-    def run(self, service, *path):
+    def run_job_from_template(self, service, *path):
         command = self.render(*path)
-        self.exec(service, command)
+        self.run_job(service, command)
 
     def render(self, *path):
         return env.render_file(self.config, *path).strip()
 
-    def exec(self, service, command):
+    def run_job(self, service, command):
         raise NotImplementedError
 
-    def check_service_is_activated(self, service):
-        if not self.is_activated(service):
-            raise exceptions.TutorError(
-                "This command may only be executed on the server where the {} is running".format(
-                    service
-                )
-            )
-
-    def is_activated(self, service):
-        return self.config["ACTIVATE_" + service.upper()]
-
     def iter_plugin_hooks(self, hook):
         yield from plugins.iter_hooks(self.config, hook)
 
 
 def initialise(runner):
     fmt.echo_info("Initialising all services...")
-    runner.run("mysql", "hooks", "mysql", "init")
+    runner.run_job_from_template("mysql", "hooks", "mysql", "init")
     for plugin_name, hook in runner.iter_plugin_hooks("pre-init"):
         for service in hook:
             fmt.echo_info(
@@ -49,17 +37,18 @@ def initialise(runner):
                     plugin_name, service
                 )
             )
-            runner.run(service, plugin_name, "hooks", service, "pre-init")
+            runner.run_job_from_template(
+                service, plugin_name, "hooks", service, "pre-init"
+            )
     for service in ["lms", "cms", "forum"]:
-        if runner.is_activated(service):
         fmt.echo_info("Initialising {}...".format(service))
-            runner.run(service, "hooks", service, "init")
+        runner.run_job_from_template(service, "hooks", service, "init")
     for plugin_name, hook in runner.iter_plugin_hooks("init"):
         for service in hook:
             fmt.echo_info(
                 "Plugin {}: running init for service {}...".format(plugin_name, service)
             )
-            runner.run(service, plugin_name, "hooks", service, "init")
+            runner.run_job_from_template(service, plugin_name, "hooks", service, "init")
     fmt.echo_info("All services initialised.")
 
 
@@ -90,8 +79,7 @@ u.save()"
 
 
 def import_demo_course(runner):
-    runner.check_service_is_activated("cms")
-    runner.run("cms", "hooks", "cms", "importdemocourse")
+    runner.run_job_from_template("cms", "hooks", "cms", "importdemocourse")
 
 
 def set_theme(theme_name, domain_name, runner):
@@ -108,5 +96,4 @@ site.themes.all().delete()
 site.themes.create(theme_dir_name='{theme_name}')"
 """
     command = command.format(theme_name=theme_name, domain_name=domain_name)
-    runner.check_service_is_activated("lms")
-    runner.exec("lms", command)
+    runner.run_job("lms", command)

tutor/serialize.py

@@ -7,6 +7,10 @@ def load(stream):
     return yaml.load(stream, Loader=yaml.SafeLoader)
 
 
+def load_all(stream):
+    return yaml.load_all(stream, Loader=yaml.SafeLoader)
+
+
 def dump(content, fileobj):
     yaml.dump(content, stream=fileobj, default_flow_style=False)
 

tutor/templates/k8s/deployments.yml

@@ -295,6 +295,7 @@ spec:
           persistentVolumeClaim:
             claimName: mongodb
 {% endif %}
+{% if ACTIVATE_MYSQL %}
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -316,12 +317,7 @@ spec:
       containers:
         - name: mysql
           image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_MYSQL }}
-          {% if ACTIVATE_MYSQL %}
           args: ["mysqld", "--character-set-server=utf8", "--collation-server=utf8_general_ci"]
-          {% else %}
-          command: ["sh", "-e", "-c"]
-          args: ["echo 'ready'; while true; do sleep 60; done"]
-          {% endif %}
           env:
             - name: MYSQL_ROOT_PASSWORD
               valueFrom:
@@ -330,7 +326,6 @@ spec:
                   key: MYSQL_ROOT_PASSWORD
           ports:
             - containerPort: 3306
-          {% if ACTIVATE_MYSQL %}
           volumeMounts:
             - mountPath: /var/lib/mysql
               name: data
@@ -338,7 +333,7 @@ spec:
         - name: data
           persistentVolumeClaim:
             claimName: mysql
-            {% endif %}
+{% endif %}
 {% if ACTIVATE_SMTP %}
 ---
 apiVersion: apps/v1

tutor/templates/k8s/ingress.yml

@@ -1,11 +1,12 @@
 ---{% set hosts = [LMS_HOST, "preview." + LMS_HOST, CMS_HOST] %}
-apiVersion: extensions/v1beta1
+apiVersion: networking.k8s.io/v1beta1
 kind: Ingress
 metadata:
   name: web
   labels:
     app.kubernetes.io/name: web
   annotations:
+    kubernetes.io/ingress.class: nginx
     nginx.ingress.kubernetes.io/proxy-body-size: 1000m
     {% if ACTIVATE_HTTPS%}kubernetes.io/tls-acme: "true"
     cert-manager.io/issuer: letsencrypt{% endif %}
@@ -25,6 +26,8 @@ spec:
       {% for host in hosts %}
       - {{ host }}{% endfor %}
       {{ patch("k8s-ingress-tls-hosts")|indent(6) }}
+    # TODO maybe we should not take care of generating certificates ourselves
+    # and here just point to a tls secret
     secretName: letsencrypt
   {%endif%}
 {% if ACTIVATE_HTTPS %}

tutor/templates/k8s/jobs.yml

@@ -0,0 +1,106 @@
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: lms-job
+  labels:
+    app.kubernetes.io/component: job
+spec:
+  template:
+    spec:
+      restartPolicy: Never
+      containers:
+      - name: lms
+        image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_OPENEDX }}
+        volumeMounts:
+          - mountPath: /openedx/edx-platform/lms/envs/tutor/
+            name: settings-lms
+          - mountPath: /openedx/edx-platform/cms/envs/tutor/
+            name: settings-cms
+          - mountPath: /openedx/config
+            name: config
+      volumes:
+      - name: settings-lms
+        configMap:
+          name: openedx-settings-lms
+      - name: settings-cms
+        configMap:
+          name: openedx-settings-cms
+      - name: config
+        configMap:
+          name: openedx-config
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: cms-job
+  labels:
+    app.kubernetes.io/component: job
+spec:
+  template:
+    spec:
+      restartPolicy: Never
+      containers:
+      - name: cms
+        image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_OPENEDX }}
+        env:
+        - name: SERVICE_VARIANT
+          value: cms
+        volumeMounts:
+          - mountPath: /openedx/edx-platform/lms/envs/tutor/
+            name: settings-lms
+          - mountPath: /openedx/edx-platform/cms/envs/tutor/
+            name: settings-cms
+          - mountPath: /openedx/config
+            name: config
+      volumes:
+      - name: settings-lms
+        configMap:
+          name: openedx-settings-lms
+      - name: settings-cms
+        configMap:
+          name: openedx-settings-cms
+      - name: config
+        configMap:
+          name: openedx-config
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: mysql-job
+  labels:
+    app.kubernetes.io/component: job
+spec:
+  template:
+    spec:
+      restartPolicy: Never
+      containers:
+      - name: mysql
+        image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_MYSQL }}
+        command: []
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: forum-job
+  labels:
+    app.kubernetes.io/component: job
+spec:
+  template:
+    spec:
+      restartPolicy: Never
+      containers:
+      - name: forum
+        image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_FORUM }}
+        env:
+          - name: SEARCH_SERVER
+            value: "{{ ELASTICSEARCH_SCHEME }}://{{ ELASTICSEARCH_HOST }}:{{ ELASTICSEARCH_PORT }}"
+          - name: MONGODB_AUTH
+            value: "{% if MONGODB_USERNAME and MONGODB_PASSWORD %}{{ MONGODB_USERNAME}}:{{ MONGODB_PASSWORD }}@{% endif %}"
+          - name: MONGODB_HOST
+            value: "{{ MONGODB_HOST }}"
+          - name: MONGODB_PORT
+            value: "{{ MONGODB_PORT }}"
+
+{{ patch("k8s-jobs") }}
+

tutor/templates/kustomization.yml

@@ -4,7 +4,9 @@ kind: Kustomization
 resources:
 - k8s/namespace.yml
 - k8s/deployments.yml
+# TODO maybe we should not take care of ingress stuff and let the administrator do it
 - k8s/ingress.yml
+- k8s/jobs.yml
 - k8s/services.yml
 - k8s/volumes.yml
 {{ patch("kustomization-resources") }}

tutor/templates/local/docker-compose.jobs.yml

@@ -0,0 +1,37 @@
+version: "3.7"
+services:
+
+    mysql-job:
+      image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_MYSQL }}
+      entrypoint: []
+      command: ["echo", "done"]
+    
+    lms-job:
+      image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_OPENEDX }}
+      environment:
+        SERVICE_VARIANT: lms
+        SETTINGS: ${EDX_PLATFORM_SETTINGS:-tutor.production}
+      volumes:
+        - ../apps/openedx/settings/lms/:/openedx/edx-platform/lms/envs/tutor/:ro
+        - ../apps/openedx/settings/cms/:/openedx/edx-platform/cms/envs/tutor/:ro
+        - ../apps/openedx/config/:/openedx/config/:ro
+    
+    cms-job:
+      image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_OPENEDX }}
+      environment:
+        SERVICE_VARIANT: cms
+        SETTINGS: ${EDX_PLATFORM_SETTINGS:-tutor.production}
+      volumes:
+        - ../apps/openedx/settings/lms/:/openedx/edx-platform/lms/envs/tutor/:ro
+        - ../apps/openedx/settings/cms/:/openedx/edx-platform/cms/envs/tutor/:ro
+        - ../apps/openedx/config/:/openedx/config/:ro
+    
+    forum-job:
+      image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_FORUM }}
+      environment:
+        SEARCH_SERVER: "{{ ELASTICSEARCH_SCHEME }}://{{ ELASTICSEARCH_HOST }}:{{ ELASTICSEARCH_PORT }}"
+        MONGODB_AUTH: "{% if MONGODB_USERNAME and MONGODB_PASSWORD %}{{ MONGODB_USERNAME}}:{{ MONGODB_PASSWORD }}@{% endif %}"
+        MONGODB_HOST: "{{ MONGODB_HOST }}"
+        MONGODB_PORT: "{{ MONGODB_PORT }}"
+    
+    {{ patch("local-docker-compose-jobs-services")|indent(4) }}

tutor/templates/local/docker-compose.yml

@@ -19,18 +19,15 @@ services:
       - ../../data/mongodb:/data/db
   {% endif %}
 
+  {% if ACTIVATE_MYSQL %}
   mysql:
     image: {{ DOCKER_REGISTRY }}{{ DOCKER_IMAGE_MYSQL }}
-    {% if ACTIVATE_MYSQL %}
     command: mysqld --character-set-server=utf8 --collation-server=utf8_general_ci
-    {% else %}
-    entrypoint: ["sh", "-e", "-c"]
-    command: ["echo 'ready'; while true; do sleep 60; done"]
-    {% endif %}
     restart: unless-stopped
     volumes:
       - ../../data/mysql:/var/lib/mysql
     env_file: ../apps/mysql/auth.env
+  {% endif %}
 
   {% if ACTIVATE_ELASTICSEARCH %}
   elasticsearch: