fix: upgrade MySQL from 5.7 to 8.1 first and then to 8.4 (#1149)

* fix: upgrade MySQL from 5.7 to 8.1 first and then to 8.4
This is required when upgrading from Tutor v15 to v18 directly
MySQL does not allow direct upgrades from v5.7 to v8.4

* fix: run MySQL 8.1 as a separate container during upgrade from Olive to Redwood (#1140)
We do this because MySQL 8.1 does not have the --mysql-native-password option
We have this option turned on for backward compatibility

changelog.d/20240726_202449_danyal.faheem_mysql_upgrade_5_7_to_8_4.md

@@ -0,0 +1,2 @@
+- [Bugfix] Do not directly upgrade MySQL from v5.7 to v8.4 when upgrading from quince as MySQL does not allow that. First, upgrade to v8.1 and then to v8.4. (by @Danyal-Faheem)
+  This process should be automatic for most users. However, if you are running a third-party MySQL (i.e., RUN_MYSQL=false), you are expected to perform this process yourself. Please refer to the third-party provider's documentation for detailed instructions. Ensuring that your MySQL version is up-to-date is crucial for maintaining compatibility and security. 

changelog.d/20241018_122745_danyal.faheem_run_mysql_8_1_as_separate_container.md

@@ -0,0 +1 @@
+- [Bugfix] Run MySQL 8.1 as a separate container during upgrade from Olive to Redwood as it crashed otherwise due to the `--mysql-native-password` option not being present. (by @Danyal-Faheem) 

tutor/commands/upgrade/common.py

@@ -1,10 +1,13 @@
 from __future__ import annotations
 
+from typing import Optional
+
 import click
+from packaging import version
 
 from tutor import config as tutor_config
 from tutor import fmt, plugins
-from tutor.types import Config
+from tutor.types import Config, get_typed
 
 
 def upgrade_from_lilac(config: Config) -> None:
@@ -60,6 +63,34 @@ def get_mongo_upgrade_parameters(
     return mongo_version, admin_command
 
 
+def get_intermediate_mysql_upgrade(config: Config) -> Optional[str]:
+    """
+    Checks if a MySQL upgrade is needed based on the Tutor version and MySQL setup.
+
+    This method ensures that MySQL is running and determines if the upgrade
+    process should proceed based on the Tutor version. It is intended for upgrades
+    from Tutor version 15 to version 18 or later. Manual upgrade steps are not
+    required for versions 16 or 17.
+
+    Returns:
+        Optional[str]: The docker image of MySQL to upgrade to or None if not applicable
+    """
+    if not get_typed(config, "RUN_MYSQL", bool):
+        fmt.echo_info(
+            "You are not running MySQL (RUN_MYSQL=false). It is your "
+            "responsibility to upgrade your MySQL instance to v8.4. There is "
+            "nothing left to do to upgrade from Olive."
+        )
+        return None
+    image_tag = get_typed(config, "DOCKER_IMAGE_MYSQL", str).split(":")[-1]
+    # If latest image, we assign a constant value to invalidate the condition
+    # as we know that the latest image will always be greater than 8.1.0
+    target_version = (
+        version.Version("8.1.1") if image_tag == "latest" else version.parse(image_tag)
+    )
+    return "docker.io/mysql:8.1.0" if target_version > version.parse("8.1.0") else None
+
+
 PALM_RENAME_ORA2_FOLDER_COMMAND = """
 if stat '/openedx/data/ora2/SET-ME-PLEASE (ex. bucket-name)' 2> /dev/null; then
     echo "Renaming ora2 folder..."

tutor/commands/upgrade/compose.py

@@ -4,8 +4,9 @@ import click
 
 from tutor import config as tutor_config
 from tutor import env as tutor_env
+from tutor import hooks
 from tutor import fmt
-from tutor.commands import compose
+from tutor.commands import compose, jobs
 from tutor.types import Config
 
 from . import common as common_upgrade
@@ -158,6 +159,63 @@ def upgrade_from_olive(context: click.Context, config: Config) -> None:
     upgrade_mongodb(context, config, "4.2.17", "4.2")
     upgrade_mongodb(context, config, "4.4.22", "4.4")
 
+    intermediate_mysql_docker_image = common_upgrade.get_intermediate_mysql_upgrade(
+        config
+    )
+    if not intermediate_mysql_docker_image:
+        return
+
+    click.echo(fmt.title(f"Upgrading MySQL to {intermediate_mysql_docker_image}"))
+
+    # We start up a mysql-8.1 container to build data dictionary to preserve
+    # the upgrade order of 5.7 -> 8.1 -> 8.4
+    # Use the mysql-8.1 context so that we can clear these filters later on
+    with hooks.Contexts.app("mysql-8.1").enter():
+        hooks.Filters.ENV_PATCHES.add_items(
+            [
+                (
+                    "local-docker-compose-services",
+                    """
+mysql-8.1:
+  extends: mysql
+  image: docker.io/mysql:8.1.0
+  command: >
+    mysqld
+    --character-set-server=utf8mb3
+    --collation-server=utf8mb3_general_ci
+    --binlog-expire-logs-seconds=259200
+    """,
+                ),
+                (
+                    "local-docker-compose-jobs-services",
+                    """
+mysql-8.1-job:
+  image: docker.io/mysql:8.1.0
+  depends_on: {{ [("mysql-8.1", RUN_MYSQL)]|list_if }}
+        """,
+                ),
+            ]
+        )
+        hooks.Filters.CONFIG_DEFAULTS.add_item(("MYSQL_HOST", "mysql-8.1"))
+
+        hooks.Filters.CLI_DO_INIT_TASKS.add_item(
+            ("mysql-8.1", tutor_env.read_core_template_file("jobs", "init", "mysql.sh"))
+        )
+
+    tutor_env.save(context.obj.root, config)
+
+    # Run the init command to make sure MySQL is ready for connections
+    context.invoke(jobs.initialise, limit="mysql-8.1")
+    context.invoke(compose.stop, services=["mysql-8.1"])
+
+    # Clear the filters added for mysql-8.1 as we don't need them anymore
+    hooks.clear_all(context="app:mysql-8.1")
+
+    # Save environment and run init for mysql 8.4 to make sure MySQL is ready
+    tutor_env.save(context.obj.root, config)
+    context.invoke(jobs.initialise, limit="mysql")
+    context.invoke(compose.stop, services=["mysql"])
+
 
 def upgrade_from_quince(context: click.Context, config: Config) -> None:
     click.echo(fmt.title("Upgrading from Quince"))

tutor/commands/upgrade/k8s.py

@@ -2,7 +2,7 @@ import click
 
 from tutor import config as tutor_config
 from tutor import env as tutor_env
-from tutor import fmt
+from tutor import fmt, hooks
 from tutor.commands import k8s
 from tutor.commands.context import Context
 from tutor.types import Config
@@ -39,7 +39,7 @@ def upgrade_from(context: click.Context, from_release: str) -> None:
         running_release = "olive"
 
     if running_release == "olive":
-        upgrade_from_olive(context.obj, config)
+        upgrade_from_olive(context, config)
         running_release = "palm"
 
     if running_release == "palm":
@@ -148,11 +148,11 @@ def upgrade_from_maple(context: Context, config: Config) -> None:
     )
 
 
-def upgrade_from_olive(context: Context, config: Config) -> None:
+def upgrade_from_olive(context: click.Context, config: Config) -> None:
     # Note that we need to exec because the ora2 folder is not bind-mounted in the job
     # services.
     k8s.kubectl_apply(
-        context.root,
+        context.obj.root,
         "--selector",
         "app.kubernetes.io/name=lms",
     )
@@ -165,6 +165,144 @@ def upgrade_from_olive(context: Context, config: Config) -> None:
     upgrade_mongodb(config, "4.2.17", "4.2")
     upgrade_mongodb(config, "4.4.22", "4.4")
 
+    intermediate_mysql_docker_image = common_upgrade.get_intermediate_mysql_upgrade(
+        config
+    )
+    if not intermediate_mysql_docker_image:
+        return
+
+    click.echo(fmt.title(f"Upgrading MySQL to {intermediate_mysql_docker_image}"))
+
+    # We start up a mysql-8.1 container to build data dictionary to preserve
+    # the upgrade order of 5.7 -> 8.1 -> 8.4
+    # Use the mysql-8.1 context so that we can clear these filters later on
+    with hooks.Contexts.app("mysql-8.1").enter():
+        hooks.Filters.ENV_PATCHES.add_items(
+            [
+                (
+                    "k8s-deployments",
+                    """
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mysql-81
+  labels:
+    app.kubernetes.io/name: mysql-81
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: mysql-81
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: mysql-81
+    spec:
+      securityContext:
+        runAsUser: 999
+        runAsGroup: 999
+        fsGroup: 999
+        fsGroupChangePolicy: "OnRootMismatch"
+      containers:
+        - name: mysql-81
+          image: docker.io/mysql:8.1.0
+          args:
+            - "mysqld"
+            - "--character-set-server=utf8mb3"
+            - "--collation-server=utf8mb3_general_ci"
+            - "--binlog-expire-logs-seconds=259200"
+          env:
+            - name: MYSQL_ROOT_PASSWORD
+              value: "{{ MYSQL_ROOT_PASSWORD }}"
+          ports:
+            - containerPort: 3306
+          volumeMounts:
+            - mountPath: /var/lib/mysql
+              name: data
+          securityContext:
+            allowPrivilegeEscalation: false
+      volumes:
+        - name: data
+          persistentVolumeClaim:
+            claimName: mysql
+    """,
+                ),
+                (
+                    "k8s-jobs",
+                    """
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: mysql-81-job
+  labels:
+    app.kubernetes.io/component: job
+spec:
+  template:
+    spec:
+      restartPolicy: Never
+      containers:
+      - name: mysql-81
+        image: docker.io/mysql:8.1.0
+        """,
+                ),
+            ]
+        )
+        hooks.Filters.ENV_PATCHES.add_item(
+            (
+                "k8s-services",
+                """
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: mysql-81
+  labels:
+    app.kubernetes.io/name: mysql-81
+spec:
+  type: ClusterIP
+  ports:
+    - port: 3306
+      protocol: TCP
+  selector:
+    app.kubernetes.io/name: mysql-81
+        """,
+            )
+        )
+        hooks.Filters.CONFIG_DEFAULTS.add_item(("MYSQL_HOST", "mysql-81"))
+
+        hooks.Filters.CLI_DO_INIT_TASKS.add_item(
+            ("mysql-81", tutor_env.read_core_template_file("jobs", "init", "mysql.sh"))
+        )
+
+    tutor_env.save(context.obj.root, config)
+
+    # Run the init command to make sure MySQL is ready for connections
+    k8s.kubectl_apply(
+        context.obj.root,
+        "--selector",
+        "app.kubernetes.io/name=mysql-81",
+    )
+    k8s.wait_for_deployment_ready(config, "mysql-81")
+    context.invoke(k8s.do.commands["init"], limit="mysql-8.1")
+    context.invoke(k8s.stop, names=["mysql-81"])
+
+    # Clear the filters added for mysql-8.1 as we don't need them anymore
+    hooks.clear_all(context="app:mysql-8.1")
+
+    # Save environment and run init for mysql 8.4 to make sure MySQL is ready
+    tutor_env.save(context.obj.root, config)
+    k8s.kubectl_apply(
+        context.obj.root,
+        "--selector",
+        "app.kubernetes.io/name=mysql",
+    )
+    k8s.wait_for_deployment_ready(config, "mysql")
+    context.invoke(k8s.do.commands["init"], limit="mysql")
+    context.invoke(k8s.stop, names=["mysql"])
+
 
 def upgrade_from_quince(config: Config) -> None:
     click.echo(fmt.title("Upgrading from Quince"))