Merge branch 'master' into nightly

CHANGELOG.md

@@ -20,6 +20,11 @@ instructions, because git commits are used to generate release notes:
 
 <!-- scriv-insert-here -->
 
+<a id='changelog-16.0.5'></a>
+## v16.0.5 (2023-08-09)
+
+- [Improvement] Upgrade the Open edX default version to open-release/palm.2. (by @regisb)
+
 <a id='changelog-16.0.4'></a>
 ## v16.0.4 (2023-08-03)
 
@@ -248,6 +253,7 @@ pen-release/palm.master --repo-dir=test-course/course`. (by @regisb)
 - [Bugfix] Build openedx-dev Docker image even when the host user is root, for instance on Windows. (by @regisb)
 - [Bugfix] Patch nutmeg.1 release with [LTI 1.3 fix](https://github.com/openedx/edx-platform/pull/30716). (by @ormsbee)
 - [Improvement] Make it possible to override k8s resources in plugins using `k8s-override` patch. (by @foadlind)
+- [Bugfix] Fix a race condition that could prevent a newly provisioned Studio container from starting due to a FileExistsError when creating logs directory.
 
 ## v14.0.2 (2022-06-27)
 

changelog.d/20230731_110301_regis_fix_non_buildkit_build.md

@@ -0,0 +1,4 @@
+- [Improvement] Improve support of legacy non-BuildKit mode: (by @regisb)
+  - [Bugfix] Fix building of openedx Docker image.
+  - [Improvement] Remove `--cache-from` build option.
+  - [Improvement] Add a warning concerning the lack of support of the `--build-context` option.

changelog.d/20230815_064652_regis_fix_mysql_upgrade_charset.md

@@ -0,0 +1,2 @@
+- 💥[Bugfix] Fix mysql crash after upgrade to Palm. After an upgrade to Palm, the mysql client run by Django defaults to a utf8mb4 character set and collation, but the mysql server still runs with utf8mb3. This causes broken data during migration from Olive to Palm, and more generally when data is written to the database. To resolve this issue, we explicitely set the utf8mb3 charset and collation in the client. Users who were running Palm might have to fix their data manually. In the future we will upgrade the mysql server to utf8mb4. (by @regisb)
+- [Improvement] We upgrade to MySQL 8.1.0 to avoid having to restart the server after the upgrade.

changelog.d/20230816_184458_regis_fix_local_non_prod.md

@@ -0,0 +1 @@
+- [Bugfix] Ask whether user wants to run locally during `tutor local launch`. (by @regisb)

docs/configuration.rst

@@ -81,7 +81,7 @@ This configuration parameter defines which MongoDB Docker image to use.
 
 .. https://hub.docker.com/_/mysql/tags?page=1&name=8.0
 
-- ``DOCKER_IMAGE_MYSQL`` (default: ``"docker.io/mysql:8.0.33"``)
+- ``DOCKER_IMAGE_MYSQL`` (default: ``"docker.io/mysql:8.1.0"``)
 
 This configuration parameter defines which MySQL Docker image to use.
 
@@ -136,7 +136,7 @@ Open edX customisation
 
 This defines the git repository from which you install Open edX platform code. If you run an Open edX fork with custom patches, set this to your own git repository. You may also override this configuration parameter at build time, by providing a ``--build-arg`` option.
 
-- ``OPENEDX_COMMON_VERSION`` (default: ``"open-release/palm.1"``)
+- ``OPENEDX_COMMON_VERSION`` (default: ``"open-release/palm.2"``)
 
 This defines the default version that will be pulled from all Open edX git repositories.
 

tests/commands/test_images.py

@@ -49,7 +49,7 @@ class ImagesTests(PluginsTestCase, TestCommandMixin):
         self.assertIsNone(result.exception)
         self.assertEqual(0, result.exit_code)
         # Note: we should update this tag whenever the mysql image is updated
-        image_pull.assert_called_once_with("docker.io/mysql:8.0.33")
+        image_pull.assert_called_once_with("docker.io/mysql:8.1.0")
 
     def test_images_printtag_image(self) -> None:
         result = self.invoke(["images", "printtag", "openedx"])
@@ -147,7 +147,6 @@ class ImagesTests(PluginsTestCase, TestCommandMixin):
                 "--target",
                 "target",
                 "docker_args",
-                "--cache-from=type=registry,ref=service1:1.0.0-cache",
             ],
             list(image_build.call_args[0][1:]),
         )

tutor/__about__.py

@@ -2,7 +2,7 @@ import os
 
 # Increment this version number to trigger a new release. See
 # docs/tutor.html#versioning for information on the versioning scheme.
-__version__ = "16.0.4"
+__version__ = "16.0.5"
 
 # The version suffix will be appended to the actual version, separated by a
 # dash. Use this suffix to differentiate between the actual released version and

tutor/commands/compose.py

@@ -90,13 +90,13 @@ def launch(
     skip_build: bool,
 ) -> None:
     context_name = context.obj.NAME
-    run_for_prod = context_name != "dev"
+    run_for_prod = False if context_name == "dev" else None
 
     utils.warn_macos_docker_memory()
 
     # Upgrade has to run before configuration
-    interactive_upgrade(context, not non_interactive, run_for_prod)
+    interactive_upgrade(context, not non_interactive, run_for_prod=run_for_prod)
-    interactive_configuration(context, not non_interactive, run_for_prod)
+    interactive_configuration(context, not non_interactive, run_for_prod=run_for_prod)
 
     config = tutor_config.load(context.obj.root)
 
@@ -136,7 +136,7 @@ def launch(
 
 
 def interactive_upgrade(
-    context: click.Context, interactive: bool, run_for_prod: bool
+    context: click.Context, interactive: bool, run_for_prod: t.Optional[bool]
 ) -> None:
     """
     Piece of code that is only used in launch.
@@ -187,7 +187,7 @@ Are you sure you want to continue?"""
 
 
 def interactive_configuration(
-    context: click.Context, interactive: bool, run_for_prod: bool
+    context: click.Context, interactive: bool, run_for_prod: t.Optional[bool] = None
 ) -> None:
     click.echo(fmt.title("Interactive platform configuration"))
     config = tutor_config.load_minimal(context.obj.root)

tutor/commands/images.py

@@ -223,8 +223,11 @@ def build(
             image_build_args = [*command_args, *custom_args]
 
             # Registry cache
+            if utils.is_buildkit_enabled():
                 if not no_registry_cache:
-                image_build_args.append(f"--cache-from=type=registry,ref={tag}-cache")
+                    image_build_args.append(
+                        f"--cache-from=type=registry,ref={tag}-cache"
+                    )
                 if cache_to_registry:
                     image_build_args.append(
                         f"--cache-to=type=registry,mode=max,ref={tag}-cache"
@@ -232,7 +235,15 @@ def build(
 
             # Build contexts
             for host_path, stage_name in build_contexts.get(name, []):
+                if utils.is_buildkit_enabled():
+                    fmt.echo_info(
+                        f"Adding {host_path} to the build context '{stage_name}' of image '{image}'"
+                    )
                     image_build_args.append(f"--build-context={stage_name}={host_path}")
+                else:
+                    fmt.echo_alert(
+                        f"Unable to add {host_path} to the build context '{stage_name}' of image '{host_path}' because BuildKit is disabled."
+                    )
 
             # Build
             images.build(
@@ -257,9 +268,6 @@ def get_image_build_contexts(config: Config) -> dict[str, list[tuple[str, str]]]
         for image_name, stage_name in hooks.Filters.IMAGES_BUILD_MOUNTS.iterate(
             user_mount
         ):
-            fmt.echo_info(
-                f"Adding {user_mount} to the build context '{stage_name}' of image '{image_name}'"
-            )
             if image_name not in build_contexts:
                 build_contexts[image_name] = []
             build_contexts[image_name].append((user_mount, stage_name))

tutor/commands/jobs.py

@@ -241,7 +241,7 @@ def sqlshell(args: list[str]) -> t.Iterable[tuple[str, str]]:
     Extra arguments will be passed to the `mysql` command verbatim. For instance, to
     show tables from the "openedx" database, run `do sqlshell openedx -e 'show tables'`.
     """
-    command = "mysql --user={{ MYSQL_ROOT_USERNAME }} --password={{ MYSQL_ROOT_PASSWORD }} --host={{ MYSQL_HOST }} --port={{ MYSQL_PORT }}"
+    command = "mysql --user={{ MYSQL_ROOT_USERNAME }} --password={{ MYSQL_ROOT_PASSWORD }} --host={{ MYSQL_HOST }} --port={{ MYSQL_PORT }} --default-character-set=utf8mb3"
     if args:
         command += " " + shlex.join(args)  # pylint: disable=protected-access
     yield ("lms", command)

tutor/hooks/catalog.py

@@ -147,11 +147,11 @@ class Filters:
     instance, you can add a "hello" to the init task of the lms container by modifying
     the :py:data:`CLI_DO_INIT_TASKS` filter::
 
-        hooks.CLI_DO_INIT_TASKS.add_item(("lms", "echo hello"))
+        hooks.Filters.CLI_DO_INIT_TASKS.add_item(("lms", "echo hello"))
 
     To add multiple items at a time, use ``add_items``::
 
-        hooks.CLI_DO_INIT_TASKS.add_items(
+        hooks.Filters.CLI_DO_INIT_TASKS.add_items(
             ("lms", "echo 'hello from lms'"),
             ("cms", "echo 'hello from cms'"),
         )

tutor/templates/apps/openedx/config/partials/auth.yml

@@ -17,5 +17,8 @@ DATABASES:
     ATOMIC_REQUESTS: true
     OPTIONS:
       init_command: "SET sql_mode='STRICT_TRANS_TABLES'"
+      {%- if RUN_MYSQL %}
+      charset: "utf8mb3"
+      {%- endif %}
 EMAIL_HOST_USER: "{{ SMTP_USERNAME }}"
 EMAIL_HOST_PASSWORD: "{{ SMTP_PASSWORD }}"

tutor/templates/apps/openedx/settings/partials/common_cms.py

@@ -23,7 +23,7 @@ FRONTEND_REGISTER_URL = LMS_ROOT_URL + '/register'
 # Create folders if necessary
 for folder in [LOG_DIR, MEDIA_ROOT, STATIC_ROOT_BASE]:
     if not os.path.exists(folder):
-        os.makedirs(folder)
+        os.makedirs(folder, exist_ok=True)
 
 {{ patch("openedx-cms-common-settings") }}
 

tutor/templates/build/openedx/Dockerfile

@@ -89,6 +89,9 @@ RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,
     setuptools==67.6.1 pip==23.0.1. wheel==0.40.0
 
 # Install base requirements
+{% if not is_buildkit_enabled() %}
+COPY --from=edx-platform /requirements/edx/base.txt /openedx/edx-platform/requirements/edx/base.txt
+{% endif %}
 RUN {% if is_buildkit_enabled() %}--mount=type=bind,from=edx-platform,source=/requirements/edx/base.txt,target=/openedx/edx-platform/requirements/edx/base.txt \
     --mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install -r /openedx/edx-platform/requirements/edx/base.txt
 
@@ -123,6 +126,10 @@ RUN nodeenv /openedx/nodeenv --node=16.14.0 --prebuilt
 # Install nodejs requirements
 ARG NPM_REGISTRY={{ NPM_REGISTRY }}
 WORKDIR /openedx/edx-platform
+{% if not is_buildkit_enabled() %}
+COPY --from=edx-platform /package.json /openedx/edx-platform/package.json
+COPY --from=edx-platform /package-lock.json /openedx/edx-platform/package-lock.json
+{% endif %}
 RUN {% if is_buildkit_enabled() %}--mount=type=bind,from=edx-platform,source=/package.json,target=/openedx/edx-platform/package.json \
     --mount=type=bind,from=edx-platform,source=/package-lock.json,target=/openedx/edx-platform/package-lock.json \
     --mount=type=bind,from=edx-platform,source=/scripts/copy-node-modules.sh,target=/openedx/edx-platform/scripts/copy-node-modules.sh \

tutor/templates/config/defaults.yml

@@ -15,7 +15,7 @@ DOCKER_IMAGE_OPENEDX_DEV: "openedx-dev:{{ TUTOR_VERSION }}"
 DOCKER_IMAGE_CADDY: "docker.io/caddy:2.6.4"
 DOCKER_IMAGE_ELASTICSEARCH: "docker.io/elasticsearch:7.17.9"
 DOCKER_IMAGE_MONGODB: "docker.io/mongo:4.4.22"
-DOCKER_IMAGE_MYSQL: "docker.io/mysql:8.0.33"
+DOCKER_IMAGE_MYSQL: "docker.io/mysql:8.1.0"
 DOCKER_IMAGE_PERMISSIONS: "{{ DOCKER_REGISTRY }}overhangio/openedx-permissions:{{ TUTOR_VERSION }}"
 DOCKER_IMAGE_REDIS: "docker.io/redis:7.0.11"
 DOCKER_IMAGE_SMTP: "docker.io/devture/exim-relay:4.96-r1-0"

tutor/templates/local/docker-compose.yml

@@ -40,7 +40,7 @@ services:
   {% if RUN_MYSQL -%}
   mysql:
     image: {{ DOCKER_IMAGE_MYSQL }}
-    command: mysqld --character-set-server=utf8 --collation-server=utf8_general_ci
+    command: mysqld --character-set-server=utf8mb3 --collation-server=utf8mb3_general_ci
     restart: unless-stopped
     user: "999:999"
     volumes: