From a6338e2880700b57ad6574a41de883de63505272 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Wed, 9 Aug 2023 22:41:34 +0200 Subject: [PATCH 1/7] feat: upgrade to open-release/palm.2 --- changelog.d/20230809_221139_regis.md | 1 + docs/configuration.rst | 2 +- tutor/templates/build/openedx/Dockerfile | 6 ------ tutor/templates/config/defaults.yml | 2 +- 4 files changed, 3 insertions(+), 8 deletions(-) create mode 100644 changelog.d/20230809_221139_regis.md diff --git a/changelog.d/20230809_221139_regis.md b/changelog.d/20230809_221139_regis.md new file mode 100644 index 0000000..dce9938 --- /dev/null +++ b/changelog.d/20230809_221139_regis.md @@ -0,0 +1 @@ +- [Improvement] Upgrade the Open edX default version to open-release/palm.2. (by @regisb) diff --git a/docs/configuration.rst b/docs/configuration.rst index c636939..15543af 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -136,7 +136,7 @@ Open edX customisation This defines the git repository from which you install Open edX platform code. If you run an Open edX fork with custom patches, set this to your own git repository. You may also override this configuration parameter at build time, by providing a ``--build-arg`` option. -- ``OPENEDX_COMMON_VERSION`` (default: ``"open-release/palm.1"``) +- ``OPENEDX_COMMON_VERSION`` (default: ``"open-release/palm.2"``) This defines the default version that will be pulled from all Open edX git repositories. diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index 097d749..abfe968 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -50,12 +50,6 @@ RUN git config --global user.email "tutor@overhang.io" \ {{ patch("openedx-dockerfile-git-patches-default") }} {%- else %} # Patch edx-platform -# Security advisory: https://github.com/openedx/edx-platform/security/advisories/GHSA-3q74-3rfh-g37j -# https://github.com/openedx/edx-platform/pull/32838 -RUN curl -fsSL https://github.com/openedx/edx-platform/commit/163259779297a7dccb28e1f8c3dfa4d2cbdb9655.patch | git am -# Fix discussion units when forum is not enabled -# https://github.com/openedx/edx-platform/pull/32464 -RUN curl -fsSL https://github.com/openedx/edx-platform/commit/a9f66705503288c360055ab80c7c3bfb884f75fe.patch | git am {%- endif %} {# Example: RUN curl -fsSL https://github.com/openedx/edx-platform/commit/.patch | git am #} diff --git a/tutor/templates/config/defaults.yml b/tutor/templates/config/defaults.yml index 29996e4..5a8399b 100644 --- a/tutor/templates/config/defaults.yml +++ b/tutor/templates/config/defaults.yml @@ -52,7 +52,7 @@ OPENEDX_CMS_UWSGI_WORKERS: 2 OPENEDX_LMS_UWSGI_WORKERS: 2 OPENEDX_MYSQL_DATABASE: "openedx" OPENEDX_MYSQL_USERNAME: "openedx" -OPENEDX_COMMON_VERSION: "open-release/palm.1" +OPENEDX_COMMON_VERSION: "open-release/palm.2" OPENEDX_EXTRA_PIP_REQUIREMENTS: - "openedx-scorm-xblock>=16.0.0,<17.0.0" MYSQL_HOST: "mysql" From f6507f995ada4f5dcd5fb74743ebf51d68091e7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Wed, 9 Aug 2023 22:42:32 +0200 Subject: [PATCH 2/7] v16.0.5 --- CHANGELOG.md | 5 +++++ changelog.d/20230809_221139_regis.md | 1 - tutor/__about__.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/20230809_221139_regis.md diff --git a/CHANGELOG.md b/CHANGELOG.md index deda386..96d429c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,11 @@ instructions, because git commits are used to generate release notes: + +## v16.0.5 (2023-08-09) + +- [Improvement] Upgrade the Open edX default version to open-release/palm.2. (by @regisb) + ## v16.0.4 (2023-08-03) diff --git a/changelog.d/20230809_221139_regis.md b/changelog.d/20230809_221139_regis.md deleted file mode 100644 index dce9938..0000000 --- a/changelog.d/20230809_221139_regis.md +++ /dev/null @@ -1 +0,0 @@ -- [Improvement] Upgrade the Open edX default version to open-release/palm.2. (by @regisb) diff --git a/tutor/__about__.py b/tutor/__about__.py index 194be8a..24e0913 100644 --- a/tutor/__about__.py +++ b/tutor/__about__.py @@ -2,7 +2,7 @@ import os # Increment this version number to trigger a new release. See # docs/tutor.html#versioning for information on the versioning scheme. -__version__ = "16.0.4" +__version__ = "16.0.5" # The version suffix will be appended to the actual version, separated by a # dash. Use this suffix to differentiate between the actual released version and From ef30dbc193334e91a3c725ceb8e052ea2c0be453 Mon Sep 17 00:00:00 2001 From: Emad Rad Date: Fri, 11 Aug 2023 14:17:03 +0330 Subject: [PATCH 3/7] chore: left out "Filters" word added --- tutor/hooks/catalog.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tutor/hooks/catalog.py b/tutor/hooks/catalog.py index 20d3e32..e1eac6d 100644 --- a/tutor/hooks/catalog.py +++ b/tutor/hooks/catalog.py @@ -147,11 +147,11 @@ class Filters: instance, you can add a "hello" to the init task of the lms container by modifying the :py:data:`CLI_DO_INIT_TASKS` filter:: - hooks.CLI_DO_INIT_TASKS.add_item(("lms", "echo hello")) + hooks.Filters.CLI_DO_INIT_TASKS.add_item(("lms", "echo hello")) To add multiple items at a time, use ``add_items``:: - hooks.CLI_DO_INIT_TASKS.add_items( + hooks.Filters.CLI_DO_INIT_TASKS.add_items( ("lms", "echo 'hello from lms'"), ("cms", "echo 'hello from cms'"), ) From 67009fb85776605e6771d3e2ba66766cebd33bb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Mon, 31 Jul 2023 11:13:13 +0200 Subject: [PATCH 4/7] fix: improve support of non-buildkit Docker build See comment here: https://github.com/overhangio/tutor/pull/868#issuecomment-1640429396 See also the conversation that spawned this PR: https://discuss.openedx.org/t/issue-in-tutor-palm-release-with-tuotr-dev-launch-while-installing/10629 --- ...731_110301_regis_fix_non_buildkit_build.md | 4 +++ tests/commands/test_images.py | 1 - tutor/commands/images.py | 28 ++++++++++++------- tutor/templates/build/openedx/Dockerfile | 7 +++++ 4 files changed, 29 insertions(+), 11 deletions(-) create mode 100644 changelog.d/20230731_110301_regis_fix_non_buildkit_build.md diff --git a/changelog.d/20230731_110301_regis_fix_non_buildkit_build.md b/changelog.d/20230731_110301_regis_fix_non_buildkit_build.md new file mode 100644 index 0000000..74c9b73 --- /dev/null +++ b/changelog.d/20230731_110301_regis_fix_non_buildkit_build.md @@ -0,0 +1,4 @@ +- [Improvement] Improve support of legacy non-BuildKit mode: (by @regisb) + - [Bugfix] Fix building of openedx Docker image. + - [Improvement] Remove `--cache-from` build option. + - [Improvement] Add a warning concerning the lack of support of the `--build-context` option. diff --git a/tests/commands/test_images.py b/tests/commands/test_images.py index f3e0132..cd0545a 100644 --- a/tests/commands/test_images.py +++ b/tests/commands/test_images.py @@ -147,7 +147,6 @@ class ImagesTests(PluginsTestCase, TestCommandMixin): "--target", "target", "docker_args", - "--cache-from=type=registry,ref=service1:1.0.0-cache", ], list(image_build.call_args[0][1:]), ) diff --git a/tutor/commands/images.py b/tutor/commands/images.py index 0108fed..d21e280 100644 --- a/tutor/commands/images.py +++ b/tutor/commands/images.py @@ -223,16 +223,27 @@ def build( image_build_args = [*command_args, *custom_args] # Registry cache - if not no_registry_cache: - image_build_args.append(f"--cache-from=type=registry,ref={tag}-cache") - if cache_to_registry: - image_build_args.append( - f"--cache-to=type=registry,mode=max,ref={tag}-cache" - ) + if utils.is_buildkit_enabled(): + if not no_registry_cache: + image_build_args.append( + f"--cache-from=type=registry,ref={tag}-cache" + ) + if cache_to_registry: + image_build_args.append( + f"--cache-to=type=registry,mode=max,ref={tag}-cache" + ) # Build contexts for host_path, stage_name in build_contexts.get(name, []): - image_build_args.append(f"--build-context={stage_name}={host_path}") + if utils.is_buildkit_enabled(): + fmt.echo_info( + f"Adding {host_path} to the build context '{stage_name}' of image '{image}'" + ) + image_build_args.append(f"--build-context={stage_name}={host_path}") + else: + fmt.echo_alert( + f"Unable to add {host_path} to the build context '{stage_name}' of image '{host_path}' because BuildKit is disabled." + ) # Build images.build( @@ -257,9 +268,6 @@ def get_image_build_contexts(config: Config) -> dict[str, list[tuple[str, str]]] for image_name, stage_name in hooks.Filters.IMAGES_BUILD_MOUNTS.iterate( user_mount ): - fmt.echo_info( - f"Adding {user_mount} to the build context '{stage_name}' of image '{image_name}'" - ) if image_name not in build_contexts: build_contexts[image_name] = [] build_contexts[image_name].append((user_mount, stage_name)) diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index abfe968..f515550 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -89,6 +89,9 @@ RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip, setuptools==67.6.1 pip==23.0.1. wheel==0.40.0 # Install base requirements +{% if not is_buildkit_enabled() %} +COPY --from=edx-platform /requirements/edx/base.txt /openedx/edx-platform/requirements/edx/base.txt +{% endif %} RUN {% if is_buildkit_enabled() %}--mount=type=bind,from=edx-platform,source=/requirements/edx/base.txt,target=/openedx/edx-platform/requirements/edx/base.txt \ --mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install -r /openedx/edx-platform/requirements/edx/base.txt @@ -123,6 +126,10 @@ RUN nodeenv /openedx/nodeenv --node=16.14.0 --prebuilt # Install nodejs requirements ARG NPM_REGISTRY={{ NPM_REGISTRY }} WORKDIR /openedx/edx-platform +{% if not is_buildkit_enabled() %} +COPY --from=edx-platform /package.json /openedx/edx-platform/package.json +COPY --from=edx-platform /package-lock.json /openedx/edx-platform/package-lock.json +{% endif %} RUN {% if is_buildkit_enabled() %}--mount=type=bind,from=edx-platform,source=/package.json,target=/openedx/edx-platform/package.json \ --mount=type=bind,from=edx-platform,source=/package-lock.json,target=/openedx/edx-platform/package-lock.json \ --mount=type=cache,target=/root/.npm,sharing=shared {% endif %}npm clean-install --no-audit --registry=$NPM_REGISTRY From b51fdb46f3f3f3090b5fc24cb6b02286dda9270c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Wed, 16 Aug 2023 18:46:28 +0200 Subject: [PATCH 5/7] fix: ask whether we run as prod in `local launch` User was no longer asked whether they wanted to run on prod or not. In other words, it was not convenient to run as local.overhang.io. --- .../20230816_184458_regis_fix_local_non_prod.md | 1 + tutor/commands/compose.py | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) create mode 100644 changelog.d/20230816_184458_regis_fix_local_non_prod.md diff --git a/changelog.d/20230816_184458_regis_fix_local_non_prod.md b/changelog.d/20230816_184458_regis_fix_local_non_prod.md new file mode 100644 index 0000000..7f80b45 --- /dev/null +++ b/changelog.d/20230816_184458_regis_fix_local_non_prod.md @@ -0,0 +1 @@ +- [Bugfix] Ask whether user wants to run locally during `tutor local launch`. (by @regisb) diff --git a/tutor/commands/compose.py b/tutor/commands/compose.py index 8e66c6d..baa1ca0 100644 --- a/tutor/commands/compose.py +++ b/tutor/commands/compose.py @@ -90,13 +90,13 @@ def launch( skip_build: bool, ) -> None: context_name = context.obj.NAME - run_for_prod = context_name != "dev" + run_for_prod = False if context_name == "dev" else None utils.warn_macos_docker_memory() # Upgrade has to run before configuration - interactive_upgrade(context, not non_interactive, run_for_prod) - interactive_configuration(context, not non_interactive, run_for_prod) + interactive_upgrade(context, not non_interactive, run_for_prod=run_for_prod) + interactive_configuration(context, not non_interactive, run_for_prod=run_for_prod) config = tutor_config.load(context.obj.root) @@ -136,7 +136,7 @@ def launch( def interactive_upgrade( - context: click.Context, interactive: bool, run_for_prod: bool + context: click.Context, interactive: bool, run_for_prod: t.Optional[bool] ) -> None: """ Piece of code that is only used in launch. @@ -187,7 +187,7 @@ Are you sure you want to continue?""" def interactive_configuration( - context: click.Context, interactive: bool, run_for_prod: bool + context: click.Context, interactive: bool, run_for_prod: t.Optional[bool] = None ) -> None: click.echo(fmt.title("Interactive platform configuration")) config = tutor_config.load_minimal(context.obj.root) From df07422adb079d4e7a74f80fb242319734a8658b Mon Sep 17 00:00:00 2001 From: 0x29a Date: Fri, 14 Jul 2023 15:02:26 +0200 Subject: [PATCH 6/7] fix: race condition could cause mkdirs() to fail with "dir exists" --- CHANGELOG.md | 1 + tutor/templates/apps/openedx/settings/partials/common_cms.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96d429c..837f0fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -253,6 +253,7 @@ pen-release/palm.master --repo-dir=test-course/course`. (by @regisb) - [Bugfix] Build openedx-dev Docker image even when the host user is root, for instance on Windows. (by @regisb) - [Bugfix] Patch nutmeg.1 release with [LTI 1.3 fix](https://github.com/openedx/edx-platform/pull/30716). (by @ormsbee) - [Improvement] Make it possible to override k8s resources in plugins using `k8s-override` patch. (by @foadlind) +- [Bugfix] Fix a race condition that could prevent a newly provisioned Studio container from starting due to a FileExistsError when creating logs directory. ## v14.0.2 (2022-06-27) diff --git a/tutor/templates/apps/openedx/settings/partials/common_cms.py b/tutor/templates/apps/openedx/settings/partials/common_cms.py index c513de5..567cb0c 100644 --- a/tutor/templates/apps/openedx/settings/partials/common_cms.py +++ b/tutor/templates/apps/openedx/settings/partials/common_cms.py @@ -23,7 +23,7 @@ FRONTEND_REGISTER_URL = LMS_ROOT_URL + '/register' # Create folders if necessary for folder in [LOG_DIR, MEDIA_ROOT, STATIC_ROOT_BASE]: if not os.path.exists(folder): - os.makedirs(folder) + os.makedirs(folder, exist_ok=True) {{ patch("openedx-cms-common-settings") }} From 2a47100d6aa300c3f754e354fca841f7bf98e8f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Tue, 15 Aug 2023 06:57:31 +0200 Subject: [PATCH 7/7] fix: broken mysql after palm upgrade This fix is for a rather serious issue that affects users who upgrade from Olive to Palm. The client mysql charset and collation was incorrectly set to utf8mb4, while the server stil runs utf8mb3. Only users who run the mysql container are affected. To resolve this issue, we explicitely configure the client to use the utf8mb3 charset/collation. Important note: users who have somehow managed to upgrade from olive to Palm before may find themselves in an undefined state. They might have to fix their mysql data manually. Same thing for users who launched Palm from scratch; although, according to my preliinary tests, they should be able to downgrade their connection from utf8mb4 to utf8mb3 without issue. In addition, we upgrade to mysql 8.1.0. Among many other fixes, this avoids a server restart after the upgrade: > An in-place upgrade from MySQL 5.7 to MySQL 8.0, without a server > restart, could result in unexpected errors when executing queries on > tables. This fix eliminates the need to restart the server between the > upgrade and queries. (Bug #35410528) https://dev.mysql.com/doc/relnotes/mysql/8.0/en/news-8-0-34.html See also the 8.1.0 release notes: https://dev.mysql.com/doc/relnotes/mysql/8.1/en/news-8-1-0.html Close #887. --- changelog.d/20230815_064652_regis_fix_mysql_upgrade_charset.md | 2 ++ docs/configuration.rst | 2 +- tests/commands/test_images.py | 2 +- tutor/commands/jobs.py | 2 +- tutor/templates/apps/openedx/config/partials/auth.yml | 3 +++ tutor/templates/config/defaults.yml | 2 +- tutor/templates/local/docker-compose.yml | 2 +- 7 files changed, 10 insertions(+), 5 deletions(-) create mode 100644 changelog.d/20230815_064652_regis_fix_mysql_upgrade_charset.md diff --git a/changelog.d/20230815_064652_regis_fix_mysql_upgrade_charset.md b/changelog.d/20230815_064652_regis_fix_mysql_upgrade_charset.md new file mode 100644 index 0000000..612e1e6 --- /dev/null +++ b/changelog.d/20230815_064652_regis_fix_mysql_upgrade_charset.md @@ -0,0 +1,2 @@ +- 💥[Bugfix] Fix mysql crash after upgrade to Palm. After an upgrade to Palm, the mysql client run by Django defaults to a utf8mb4 character set and collation, but the mysql server still runs with utf8mb3. This causes broken data during migration from Olive to Palm, and more generally when data is written to the database. To resolve this issue, we explicitely set the utf8mb3 charset and collation in the client. Users who were running Palm might have to fix their data manually. In the future we will upgrade the mysql server to utf8mb4. (by @regisb) +- [Improvement] We upgrade to MySQL 8.1.0 to avoid having to restart the server after the upgrade. diff --git a/docs/configuration.rst b/docs/configuration.rst index 15543af..a6b5e07 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -81,7 +81,7 @@ This configuration parameter defines which MongoDB Docker image to use. .. https://hub.docker.com/_/mysql/tags?page=1&name=8.0 -- ``DOCKER_IMAGE_MYSQL`` (default: ``"docker.io/mysql:8.0.33"``) +- ``DOCKER_IMAGE_MYSQL`` (default: ``"docker.io/mysql:8.1.0"``) This configuration parameter defines which MySQL Docker image to use. diff --git a/tests/commands/test_images.py b/tests/commands/test_images.py index cd0545a..7b09577 100644 --- a/tests/commands/test_images.py +++ b/tests/commands/test_images.py @@ -49,7 +49,7 @@ class ImagesTests(PluginsTestCase, TestCommandMixin): self.assertIsNone(result.exception) self.assertEqual(0, result.exit_code) # Note: we should update this tag whenever the mysql image is updated - image_pull.assert_called_once_with("docker.io/mysql:8.0.33") + image_pull.assert_called_once_with("docker.io/mysql:8.1.0") def test_images_printtag_image(self) -> None: result = self.invoke(["images", "printtag", "openedx"]) diff --git a/tutor/commands/jobs.py b/tutor/commands/jobs.py index eb35683..562e78d 100644 --- a/tutor/commands/jobs.py +++ b/tutor/commands/jobs.py @@ -241,7 +241,7 @@ def sqlshell(args: list[str]) -> t.Iterable[tuple[str, str]]: Extra arguments will be passed to the `mysql` command verbatim. For instance, to show tables from the "openedx" database, run `do sqlshell openedx -e 'show tables'`. """ - command = "mysql --user={{ MYSQL_ROOT_USERNAME }} --password={{ MYSQL_ROOT_PASSWORD }} --host={{ MYSQL_HOST }} --port={{ MYSQL_PORT }}" + command = "mysql --user={{ MYSQL_ROOT_USERNAME }} --password={{ MYSQL_ROOT_PASSWORD }} --host={{ MYSQL_HOST }} --port={{ MYSQL_PORT }} --default-character-set=utf8mb3" if args: command += " " + shlex.join(args) # pylint: disable=protected-access yield ("lms", command) diff --git a/tutor/templates/apps/openedx/config/partials/auth.yml b/tutor/templates/apps/openedx/config/partials/auth.yml index 3e72058..74d46d8 100644 --- a/tutor/templates/apps/openedx/config/partials/auth.yml +++ b/tutor/templates/apps/openedx/config/partials/auth.yml @@ -17,5 +17,8 @@ DATABASES: ATOMIC_REQUESTS: true OPTIONS: init_command: "SET sql_mode='STRICT_TRANS_TABLES'" + {%- if RUN_MYSQL %} + charset: "utf8mb3" + {%- endif %} EMAIL_HOST_USER: "{{ SMTP_USERNAME }}" EMAIL_HOST_PASSWORD: "{{ SMTP_PASSWORD }}" diff --git a/tutor/templates/config/defaults.yml b/tutor/templates/config/defaults.yml index 5a8399b..16b99ed 100644 --- a/tutor/templates/config/defaults.yml +++ b/tutor/templates/config/defaults.yml @@ -15,7 +15,7 @@ DOCKER_IMAGE_OPENEDX_DEV: "openedx-dev:{{ TUTOR_VERSION }}" DOCKER_IMAGE_CADDY: "docker.io/caddy:2.6.4" DOCKER_IMAGE_ELASTICSEARCH: "docker.io/elasticsearch:7.17.9" DOCKER_IMAGE_MONGODB: "docker.io/mongo:4.4.22" -DOCKER_IMAGE_MYSQL: "docker.io/mysql:8.0.33" +DOCKER_IMAGE_MYSQL: "docker.io/mysql:8.1.0" DOCKER_IMAGE_PERMISSIONS: "{{ DOCKER_REGISTRY }}overhangio/openedx-permissions:{{ TUTOR_VERSION }}" DOCKER_IMAGE_REDIS: "docker.io/redis:7.0.11" DOCKER_IMAGE_SMTP: "docker.io/devture/exim-relay:4.96-r1-0" diff --git a/tutor/templates/local/docker-compose.yml b/tutor/templates/local/docker-compose.yml index 2dc6b72..a2ace7c 100644 --- a/tutor/templates/local/docker-compose.yml +++ b/tutor/templates/local/docker-compose.yml @@ -40,7 +40,7 @@ services: {% if RUN_MYSQL -%} mysql: image: {{ DOCKER_IMAGE_MYSQL }} - command: mysqld --character-set-server=utf8 --collation-server=utf8_general_ci + command: mysqld --character-set-server=utf8mb3 --collation-server=utf8mb3_general_ci restart: unless-stopped user: "999:999" volumes: