From b3c3c4a2cc468e99da6660202af61c1dc44b3277 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Wed, 12 Apr 2023 10:35:00 +0200 Subject: [PATCH] feat: upgrade to Palm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Among other changes: ORA2 file uploads were stored in a folder named "SET-ME-PLEASE (ex. bucket-name)" (sigh). With this change, the folder should be automatically renamed to "openedxuploads". This issue has been occuring since June 2019... (sigh²) Close #707 --- changelog.d/20230412_100608_regis_palm.md | 2 ++ docs/configuration.rst | 28 ++++++++++++------- docs/dev.rst | 2 +- docs/faq.rst | 2 +- docs/quickstart.rst | 2 +- docs/reference/indexes.rst | 10 +++---- docs/tutorials/arm64.rst | 23 ++------------- requirements/plugins.txt | 22 +++++++-------- tests/commands/test_images.py | 2 +- tests/test_env.py | 2 +- tutor/__about__.py | 2 +- tutor/commands/dev.py | 12 ++------ tutor/commands/local.py | 15 ++-------- tutor/commands/upgrade/__init__.py | 12 ++++++-- tutor/commands/upgrade/common.py | 8 ++++++ tutor/commands/upgrade/k8s.py | 20 +++++++++++++ tutor/commands/upgrade/local.py | 15 ++++++++++ tutor/env.py | 8 ++++-- .../openedx/settings/partials/common_all.py | 13 ++++++--- .../openedx/settings/partials/common_lms.py | 5 ---- tutor/templates/build/openedx/Dockerfile | 16 +++++------ tutor/templates/build/openedx/revisions.yml | 2 +- tutor/templates/config/defaults.yml | 14 +++++----- tutor/utils.py | 13 +++++++++ 24 files changed, 146 insertions(+), 104 deletions(-) create mode 100644 changelog.d/20230412_100608_regis_palm.md diff --git a/changelog.d/20230412_100608_regis_palm.md b/changelog.d/20230412_100608_regis_palm.md new file mode 100644 index 0000000..381ba29 --- /dev/null +++ b/changelog.d/20230412_100608_regis_palm.md @@ -0,0 +1,2 @@ +- đŸ’¥[Feature] Upgrade to Palm. (by @regisb) + - [Bugfix] Rename ORA2 file upload folder from "SET-ME-PLEASE (ex. bucket-name)" to "openedxuploads". This has the effect of moving the corresponding folder from the `/data/lms/ora2` directory. MinIO users were not affected by this bug. diff --git a/docs/configuration.rst b/docs/configuration.rst index 254e751..b36f493 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -67,11 +67,13 @@ This configuration parameter defines the name of the Docker image to run for the This configuration paramater defines the name of the Docker image to run the development version of the lms and cms containers. By default, the Docker image tag matches the Tutor version it was built with. +.. https://hub.docker.com/r/devture/exim-relay/tags + - ``DOCKER_IMAGE_CADDY`` (default: ``"docker.io/caddy:2.6.2"``) This configuration paramater defines which Caddy Docker image to use. -- ``DOCKER_IMAGE_ELASTICSEARCH`` (default: ``"docker.io/elasticsearch:7.10.1"``) +- ``DOCKER_IMAGE_ELASTICSEARCH`` (default: ``"docker.io/elasticsearch:7.17.9"``) This configuration parameter defines which Elasticsearch Docker image to use. @@ -79,15 +81,21 @@ This configuration parameter defines which Elasticsearch Docker image to use. This configuration parameter defines which MongoDB Docker image to use. -- ``DOCKER_IMAGE_MYSQL`` (default: ``"docker.io/mysql:5.7.35"``) +.. https://hub.docker.com/_/mysql/tags?page=1&name=8.0 + +- ``DOCKER_IMAGE_MYSQL`` (default: ``"docker.io/mysql:8.0.33"``) This configuration parameter defines which MySQL Docker image to use. -- ``DOCKER_IMAGE_REDIS`` (default: ``"docker.io/redis:6.2.6"``) +.. https://hub.docker.com/_/redis/tags + +- ``DOCKER_IMAGE_REDIS`` (default: ``"docker.io/redis:7.0.11"``) This configuration parameter defines which Redis Docker image to use. -- ``DOCKER_IMAGE_SMTP`` (default: ``"docker.io/devture/exim-relay:4.95-r0-2``) +.. https://hub.docker.com/r/devture/exim-relay/tags + +- ``DOCKER_IMAGE_SMTP`` (default: ``"docker.io/devture/exim-relay:4.96-r1-0``) This configuration parameter defines which Simple Mail Transfer Protocol (SMTP) Docker image to use. @@ -130,7 +138,7 @@ Open edX customisation This defines the git repository from which you install Open edX platform code. If you run an Open edX fork with custom patches, set this to your own git repository. You may also override this configuration parameter at build time, by providing a ``--build-arg`` option. -- ``OPENEDX_COMMON_VERSION`` (default: ``"open-release/olive.4"``) +- ``OPENEDX_COMMON_VERSION`` (default: ``"open-release/palm.master"``) This defines the default version that will be pulled from all Open edX git repositories. @@ -150,7 +158,7 @@ These two configuration parameters define which Redis database to use for Open e .. _openedx_extra_pip_requirements: -- ``OPENEDX_EXTRA_PIP_REQUIREMENTS`` (default: ``["openedx-scorm-xblock>=15.0.0,<16.0.0"]``) +- ``OPENEDX_EXTRA_PIP_REQUIREMENTS`` (default: ``["openedx-scorm-xblock>=16.0.0,<17.0.0"]``) This defines extra pip packages that are going to be installed for Open edX. @@ -404,14 +412,14 @@ If you don't create your fork from this tag, you *will* have important compatibi - Do not try to run a fork from an older (pre-Olive) version of edx-platform: this will simply not work. - Do not try to run a fork from the edx-platform master branch: there is a 99% probability that it will fail. -- Do not try to run a fork from the open-release/olive.master branch: Tutor will attempt to apply security and bug fix patches that might already be included in the open-release/olive.master but which were not yet applied to the latest release tag. Patch application will thus fail if you base your fork from the open-release/olive.master branch. +- Do not try to run a fork from the open-release/palm.master branch: Tutor will attempt to apply security and bug fix patches that might already be included in the open-release/palm.master but which were not yet applied to the latest release tag. Patch application will thus fail if you base your fork from the open-release/palm.master branch. .. _i18n: Adding custom translations ~~~~~~~~~~~~~~~~~~~~~~~~~~ -If you are not running Open edX in English (``LANGUAGE_CODE`` default: ``"en"``), chances are that some strings will not be properly translated. In most cases, this is because not enough contributors have helped translate Open edX into your language. It happens! With Tutor, available translated languages include those that come bundled with `edx-platform `__ as well as those from `openedx-i18n `__. +If you are not running Open edX in English (``LANGUAGE_CODE`` default: ``"en"``), chances are that some strings will not be properly translated. In most cases, this is because not enough contributors have helped translate Open edX into your language. It happens! With Tutor, available translated languages include those that come bundled with `edx-platform `__ as well as those from `openedx-i18n `__. Tutor offers a relatively simple mechanism to add custom translations to the openedx Docker image. You should create a folder that corresponds to your language code in the "build/openedx/locale" folder of the Tutor environment. This folder should contain a "LC_MESSAGES" folder. For instance:: @@ -432,9 +440,9 @@ Then, add a "django.po" file there that will contain your custom translations:: .. warning:: Don't forget to specify the file ``Content-Type`` when adding message strings with non-ASCII characters; otherwise a ``UnicodeDecodeError`` will be raised during compilation. -The "String to translate" part should match *exactly* the string that you would like to translate. You cannot make it up! The best way to find this string is to copy-paste it from the `upstream django.po file for the English language `__. +The "String to translate" part should match *exactly* the string that you would like to translate. You cannot make it up! The best way to find this string is to copy-paste it from the `upstream django.po file for the English language `__. -If you cannot find the string to translate in this file, then it means that you are trying to translate a string that is used in some piece of javascript code. Those strings are stored in a different file named "djangojs.po". You can check it out `in the edx-platform repo as well `__. Your custom javascript strings should also be stored in a "djangojs.po" file that should be placed in the same directory. +If you cannot find the string to translate in this file, then it means that you are trying to translate a string that is used in some piece of javascript code. Those strings are stored in a different file named "djangojs.po". You can check it out `in the edx-platform repo as well `__. Your custom javascript strings should also be stored in a "djangojs.po" file that should be placed in the same directory. To recap, here is an example. To translate a few strings in French, both from django.po and djangojs.po, we would have the following file hierarchy:: diff --git a/docs/dev.rst b/docs/dev.rst index 13d75fa..5ee10c0 100644 --- a/docs/dev.rst +++ b/docs/dev.rst @@ -128,7 +128,7 @@ The ``openedx-dev`` Docker image is based on the same ``openedx`` image used by - Additional Python and system requirements are installed for convenient debugging: `ipython `__, `ipdb `__, vim, telnet. -- The edx-platform `development requirements `__ are installed. +- The edx-platform `development requirements `__ are installed. If you are using a custom ``openedx`` image, then you will need to rebuild ``openedx-dev`` every time you modify ``openedx``. To so, run:: diff --git a/docs/faq.rst b/docs/faq.rst index 045eaa5..a2fc8d1 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -38,7 +38,7 @@ The `devstack `_ is meant for development o Is Tutor officially supported by edX? ------------------------------------- -Yes: as of the Open edX Maple release (December 9th 2021), Tutor is the only officially supported installation method for Open edX: see the `official installation instructions `__. +Yes: as of the Open edX Maple release (December 9th 2021), Tutor is the only officially supported installation method for Open edX: see the `official installation instructions `__. What features are missing from Tutor? ------------------------------------- diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 5bb1a63..cad6dd2 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -22,7 +22,7 @@ Yes :) This is what happens when you run ``tutor local launch``: 2. Configuration files are generated from templates. 3. Docker images are downloaded. 4. Docker containers are provisioned. -5. A full, production-ready Open edX platform (`Olive `__ release) is run with docker-compose. +5. A full, production-ready Open edX platform (`Palm `__ release) is run with docker-compose. The whole procedure should require less than 10 minutes, on a server with good bandwidth. Note that your host environment will not be affected in any way, since everything runs inside docker containers. Root access is not even necessary. diff --git a/docs/reference/indexes.rst b/docs/reference/indexes.rst index a1eb7de..db06fcf 100644 --- a/docs/reference/indexes.rst +++ b/docs/reference/indexes.rst @@ -7,10 +7,10 @@ Plugin indexes are a great way to have your plugins discovered by other users. P Index file paths ================ -A plugin index is a yaml-formatted file. It can be stored on the web or on your computer. In both cases, the index file location must end with "/plugins.yml". For instance, the following are valid index locations if you run the Open edX "Olive" release: +A plugin index is a yaml-formatted file. It can be stored on the web or on your computer. In both cases, the index file location must end with "/plugins.yml". For instance, the following are valid index locations if you run the Open edX "Palm" release: -- https://overhang.io/tutor/main/olive/plugins.yml -- ``/path/to/your/local/index/olive/plugins.yml`` +- https://overhang.io/tutor/main/palm/plugins.yml +- ``/path/to/your/local/index/palm/plugins.yml`` To add either indexes, run the ``tutor plugins index add`` command without the suffix. For instance:: @@ -106,9 +106,9 @@ Manage plugins in development Plugin developers and maintainers often want to install local versions of their plugins. They usually achieve this with ``pip install -e /path/to/tutor-plugin``. We can improve that workflow by creating an index for local plugins:: # Create the plugin index directory - mkdir -p ~/localindex/olive/ + mkdir -p ~/localindex/palm/ # Edit the index - vim ~/localindex/olive/plugins.yml + vim ~/localindex/palm/plugins.yml Add the following to the index:: diff --git a/docs/tutorials/arm64.rst b/docs/tutorials/arm64.rst index a48c619..0ddc177 100644 --- a/docs/tutorials/arm64.rst +++ b/docs/tutorials/arm64.rst @@ -7,7 +7,6 @@ Tutor can be used on ARM64 systems, although no official ARM64 docker images are .. note:: There are generally two ways to run Tutor on an ARM system - using emulation (via qemu or Rosetta 2) to run x86_64 images or running native ARM images. Since emulation can be noticeably slower (typically 20-100% slower depending on the emulation method), this tutorial aims to use native images where possible. - Building the images ------------------- @@ -27,30 +26,14 @@ Then, build the "openedx" and "permissions" images:: tutor images build openedx permissions +.. TODO we don't want this instruction anymore If you want to use Tutor as an Open edX development environment, you should also build the development images:: - tutor dev dc build lms -Change the database server --------------------------- - -The version of MySQL that Open edX uses by default (5.7) does not support the ARM architecture. You need to tell Tutor to use MySQL 8.0, which does support the ARM architecture and which has been supported by Open edX since the "Nutmeg" release. - -Configure Tutor to use MySQL 8:: - - tutor config save --set DOCKER_IMAGE_MYSQL=docker.io/mysql:8.0 - -(If you need to run an older release of Open edX on ARM64, you can try using `mariadb:10.4` although it's not officially supported nor recommended for production.) - -Finish setup and start Tutor ----------------------------- - From this point on, use Tutor as normal. For example, start Open edX and run migrations with:: - tutor local start -d - tutor local do init + tutor local launch Or for a development environment:: - tutor dev start -d - tutor dev do init + tutor dev launch diff --git a/requirements/plugins.txt b/requirements/plugins.txt index 7958312..13e4d0c 100644 --- a/requirements/plugins.txt +++ b/requirements/plugins.txt @@ -1,11 +1,11 @@ -# change version ranges when upgrading from olive -tutor-android>=15.0.0,<16.0.0 -tutor-discovery>=15.0.0,<16.0.0 -tutor-ecommerce>=15.0.0,<16.0.0 -tutor-forum>=15.0.0,<16.0.0 -tutor-license>=15.0.0,<16.0.0 -tutor-mfe>=15.0.0,<16.0.0 -tutor-minio>=15.0.0,<16.0.0 -tutor-notes>=15.0.0,<16.0.0 -tutor-webui>=15.0.0,<16.0.0 -tutor-xqueue>=15.0.0,<16.0.0 +# change version ranges when upgrading from palm +tutor-android>=16.0.0,<17.0.0 +tutor-discovery>=16.0.0,<17.0.0 +tutor-ecommerce>=16.0.0,<17.0.0 +tutor-forum>=16.0.0,<17.0.0 +tutor-license>=16.0.0,<17.0.0 +tutor-mfe>=16.0.0,<17.0.0 +tutor-minio>=16.0.0,<17.0.0 +tutor-notes>=16.0.0,<17.0.0 +tutor-webui>=16.0.0,<17.0.0 +tutor-xqueue>=16.0.0,<17.0.0 diff --git a/tests/commands/test_images.py b/tests/commands/test_images.py index 99e74e9..20abdd2 100644 --- a/tests/commands/test_images.py +++ b/tests/commands/test_images.py @@ -49,7 +49,7 @@ class ImagesTests(PluginsTestCase, TestCommandMixin): self.assertIsNone(result.exception) self.assertEqual(0, result.exit_code) # Note: we should update this tag whenever the mysql image is updated - image_pull.assert_called_once_with("docker.io/mysql:5.7.35") + image_pull.assert_called_once_with("docker.io/mysql:8.0.33") def test_images_printtag_image(self) -> None: result = self.invoke(["images", "printtag", "openedx"]) diff --git a/tests/test_env.py b/tests/test_env.py index e79d891..9dd299c 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -259,7 +259,7 @@ class CurrentVersionTests(unittest.TestCase): ) as f: f.write(__version__) self.assertEqual(__version__, env.current_version(root)) - self.assertEqual("olive", env.get_env_release(root)) + self.assertEqual("palm", env.get_env_release(root)) self.assertIsNone(env.should_upgrade_from_release(root)) self.assertTrue(env.is_up_to_date(root)) diff --git a/tutor/__about__.py b/tutor/__about__.py index e05ea24..45ab496 100644 --- a/tutor/__about__.py +++ b/tutor/__about__.py @@ -2,7 +2,7 @@ import os # Increment this version number to trigger a new release. See # docs/tutor.html#versioning for information on the versioning scheme. -__version__ = "15.3.7" +__version__ = "16.0.0" # The version suffix will be appended to the actual version, separated by a # dash. Use this suffix to differentiate between the actual released version and diff --git a/tutor/commands/dev.py b/tutor/commands/dev.py index 0fcb58d..124c6a9 100644 --- a/tutor/commands/dev.py +++ b/tutor/commands/dev.py @@ -4,7 +4,7 @@ import click from tutor import config as tutor_config from tutor import env as tutor_env -from tutor import exceptions, fmt, hooks +from tutor import fmt, hooks from tutor import interactive as interactive_config from tutor import utils from tutor.commands import compose @@ -73,15 +73,7 @@ def launch( mounts: tuple[list[compose.MountParam.MountType]], ) -> None: compose.mount_tmp_volumes(mounts, context.obj) - try: - utils.check_macos_docker_memory() - except exceptions.TutorError as e: - fmt.echo_alert( - f"""Could not verify sufficient RAM allocation in Docker: - {e} -Tutor may not work if Docker is configured with < 4 GB RAM. Please follow instructions from: - https://docs.tutor.overhang.io/install.html""" - ) + utils.warn_macos_docker_memory() click.echo(fmt.title("Interactive platform configuration")) config = tutor_config.load_minimal(context.obj.root) diff --git a/tutor/commands/local.py b/tutor/commands/local.py index e391b9c..9a91392 100644 --- a/tutor/commands/local.py +++ b/tutor/commands/local.py @@ -6,7 +6,7 @@ import click from tutor import config as tutor_config from tutor import env as tutor_env -from tutor import exceptions, fmt, hooks +from tutor import fmt, hooks from tutor import interactive as interactive_config from tutor import utils from tutor.commands import compose @@ -77,18 +77,7 @@ def launch( pullimages: bool, ) -> None: compose.mount_tmp_volumes(mounts, context.obj) - try: - utils.check_macos_docker_memory() - except exceptions.TutorError as e: - fmt.echo_alert( - f"""Could not verify sufficient RAM allocation in Docker: - - {e} - -Tutor may not work if Docker is configured with < 4 GB RAM. Please follow instructions from: - - https://docs.tutor.overhang.io/install.html""" - ) + utils.warn_macos_docker_memory() run_upgrade_from_release = tutor_env.should_upgrade_from_release(context.obj.root) if run_upgrade_from_release is not None: diff --git a/tutor/commands/upgrade/__init__.py b/tutor/commands/upgrade/__init__.py index cf484d1..9c1d203 100644 --- a/tutor/commands/upgrade/__init__.py +++ b/tutor/commands/upgrade/__init__.py @@ -1,2 +1,10 @@ -# Note: don't forget to change this when we upgrade from olive -OPENEDX_RELEASE_NAMES = ["ironwood", "juniper", "koa", "lilac", "maple", "nutmeg"] +# Note: don't forget to change this when we upgrade from palm +OPENEDX_RELEASE_NAMES = [ + "ironwood", + "juniper", + "koa", + "lilac", + "maple", + "nutmeg", + "olive", +] diff --git a/tutor/commands/upgrade/common.py b/tutor/commands/upgrade/common.py index 85ef5cc..0b50f3e 100644 --- a/tutor/commands/upgrade/common.py +++ b/tutor/commands/upgrade/common.py @@ -39,3 +39,11 @@ def upgrade_from_nutmeg(context: click.Context, config: Config) -> None: context.obj.job_runner(config).run_task( "lms", "./manage.py lms compute_grades -v1 --all_courses" ) + + +PALM_RENAME_ORA2_FOLDER_COMMAND = """ +if stat '/openedx/data/ora2/SET-ME-PLEASE (ex. bucket-name)' 2> /dev/null; then + echo "Renaming ora2 folder..." + mv '/openedx/data/ora2/SET-ME-PLEASE (ex. bucket-name)' /openedx/data/ora2/openedxuploads +fi +""" diff --git a/tutor/commands/upgrade/k8s.py b/tutor/commands/upgrade/k8s.py index 7d9d94a..df83ead 100644 --- a/tutor/commands/upgrade/k8s.py +++ b/tutor/commands/upgrade/k8s.py @@ -38,6 +38,10 @@ def upgrade_from(context: click.Context, from_release: str) -> None: common_upgrade.upgrade_from_nutmeg(context, config) running_release = "olive" + if running_release == "olive": + upgrade_from_olive(context.obj, config) + running_release = "palm" + def upgrade_from_ironwood(config: Config) -> None: if not config["RUN_MONGODB"]: @@ -173,3 +177,19 @@ def upgrade_from_maple(context: Context, config: Config) -> None: k8s.kubectl_exec( config, "cms", ["sh", "-e", "-c", "./manage.py cms simulate_publish"] ) + + +def upgrade_from_olive(context: Context, config: Config) -> None: + # Note that we need to exec because the ora2 folder is not bind-mounted in the job + # services. + k8s.kubectl_apply( + context.root, + "--selector", + "app.kubernetes.io/name=lms", + ) + k8s.wait_for_deployment_ready(config, "lms") + k8s.kubectl_exec( + config, + "lms", + ["sh", "-e", "-c", common_upgrade.PALM_RENAME_ORA2_FOLDER_COMMAND], + ) diff --git a/tutor/commands/upgrade/local.py b/tutor/commands/upgrade/local.py index 7c7832f..80b0092 100644 --- a/tutor/commands/upgrade/local.py +++ b/tutor/commands/upgrade/local.py @@ -39,6 +39,10 @@ def upgrade_from(context: click.Context, from_release: str) -> None: common_upgrade.upgrade_from_nutmeg(context, config) running_release = "olive" + if running_release == "olive": + upgrade_from_olive(context) + running_release = "palm" + def upgrade_from_ironwood(context: click.Context, config: Config) -> None: click.echo(fmt.title("Upgrading from Ironwood")) @@ -146,6 +150,17 @@ def upgrade_from_maple(context: click.Context, config: Config) -> None: ) +def upgrade_from_olive(context: click.Context) -> None: + # Note that we need to exec because the ora2 folder is not bind-mounted in the job + # services. + context.invoke(compose.start, detach=True, services=["lms"]) + context.invoke( + compose.execute, + args=["lms", "sh", "-e", "-c", common_upgrade.PALM_RENAME_ORA2_FOLDER_COMMAND], + ) + context.invoke(compose.stop) + + def upgrade_mongodb( context: click.Context, config: Config, diff --git a/tutor/env.py b/tutor/env.py index f0375a4..e24e90e 100644 --- a/tutor/env.py +++ b/tutor/env.py @@ -161,7 +161,7 @@ class Renderer: try: patches.append(self.render_str(patch)) except exceptions.TutorError: - fmt.echo_error(f"Error rendering patch '{name}': {patch}") + fmt.echo_error(f"Error rendering patch '{name}':\n{patch}") raise rendered = separator.join(patches) if rendered: @@ -169,7 +169,10 @@ class Renderer: return rendered def render_str(self, text: str) -> str: - template = self.environment.from_string(text) + try: + template = self.environment.from_string(text) + except jinja2.exceptions.TemplateSyntaxError as e: + raise exceptions.TutorError(f"Template syntax error: {e.args[0]}") return self.__render(template) def render_template(self, template_name: str) -> t.Union[str, bytes]: @@ -450,6 +453,7 @@ def get_release(version: str) -> str: "13": "maple", "14": "nutmeg", "15": "olive", + "16": "palm", }[version.split(".", maxsplit=1)[0]] diff --git a/tutor/templates/apps/openedx/settings/partials/common_all.py b/tutor/templates/apps/openedx/settings/partials/common_all.py index c7a4669..46a563d 100644 --- a/tutor/templates/apps/openedx/settings/partials/common_all.py +++ b/tutor/templates/apps/openedx/settings/partials/common_all.py @@ -118,8 +118,10 @@ GRADES_DOWNLOAD = { }, } +# ORA2 ORA2_FILEUPLOAD_BACKEND = "filesystem" ORA2_FILEUPLOAD_ROOT = "/openedx/data/ora2" +FILE_UPLOAD_STORAGE_BUCKET_NAME = "openedxuploads" ORA2_FILEUPLOAD_CACHE_NAME = "ora2-storage" # Change syslog-based loggers which don't work inside docker containers @@ -135,18 +137,21 @@ LOGGING["handlers"]["tracking"] = { "formatter": "standard", } LOGGING["loggers"]["tracking"]["handlers"] = ["console", "local", "tracking"] -# Silence some loggers (note: we must attempt to get rid of these when upgrading from one release to the next) +# Silence some loggers (note: we must attempt to get rid of these when upgrading from one release to the next) +LOGGING["loggers"]["blockstore.apps.bundles.storage"] = {"handlers": ["console"], "level": "WARNING"} + +# These warnings are visible in simple commands and init tasks import warnings from django.utils.deprecation import RemovedInDjango40Warning, RemovedInDjango41Warning warnings.filterwarnings("ignore", category=RemovedInDjango40Warning) warnings.filterwarnings("ignore", category=RemovedInDjango41Warning) -warnings.filterwarnings("ignore", category=DeprecationWarning, module="lms.djangoapps.course_wiki.plugins.markdownedx.wiki_plugin") warnings.filterwarnings("ignore", category=DeprecationWarning, module="wiki.plugins.links.wiki_plugin") warnings.filterwarnings("ignore", category=DeprecationWarning, module="boto.plugin") warnings.filterwarnings("ignore", category=DeprecationWarning, module="botocore.vendored.requests.packages.urllib3._collections") -warnings.filterwarnings("ignore", category=DeprecationWarning, module="storages.backends.s3boto") -warnings.filterwarnings("ignore", category=DeprecationWarning, module="openedx.core.types.admin") +warnings.filterwarnings("ignore", category=DeprecationWarning, module="pkg_resources") +warnings.filterwarnings("ignore", category=DeprecationWarning, module="fs") +warnings.filterwarnings("ignore", category=DeprecationWarning, module="fs.opener") SILENCED_SYSTEM_CHECKS = ["2_0.W001", "fields.W903"] # Email diff --git a/tutor/templates/apps/openedx/settings/partials/common_lms.py b/tutor/templates/apps/openedx/settings/partials/common_lms.py index 5461d65..c1ec4af 100644 --- a/tutor/templates/apps/openedx/settings/partials/common_lms.py +++ b/tutor/templates/apps/openedx/settings/partials/common_lms.py @@ -36,11 +36,6 @@ CACHES["staticfiles"] = { "BACKEND": "django.core.cache.backends.locmem.LocMemCache", "LOCATION": "staticfiles_lms", } -CACHES["ora2-storage"] = { - "KEY_PREFIX": "ora2-storage", - "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}", -} # Create folders if necessary for folder in [DATA_DIR, LOG_DIR, MEDIA_ROOT, STATIC_ROOT_BASE, ORA2_FILEUPLOAD_ROOT]: diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index 698f631..3bbc842 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -15,9 +15,11 @@ RUN apt update && \ apt install -y libssl-dev zlib1g-dev libbz2-dev \ libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \ xz-utils tk-dev libffi-dev liblzma-dev python-openssl git -ARG PYTHON_VERSION=3.8.12 +# https://github.com/pyenv/pyenv/releases +# https://www.python.org/downloads/ +ARG PYTHON_VERSION=3.8.15 ENV PYENV_ROOT /opt/pyenv -RUN git clone https://github.com/pyenv/pyenv $PYENV_ROOT --branch v2.2.2 --depth 1 +RUN git clone https://github.com/pyenv/pyenv $PYENV_ROOT --branch v2.3.17 --depth 1 RUN $PYENV_ROOT/bin/pyenv install $PYTHON_VERSION RUN $PYENV_ROOT/versions/$PYTHON_VERSION/bin/python -m venv /openedx/venv @@ -38,12 +40,9 @@ RUN git config --global user.email "tutor@overhang.io" \ {{ patch("openedx-dockerfile-git-patches-default") }} {%- else %} # Patch edx-platform -# Fix broken Circuit Schematic Builder problem template -# https://github.com/openedx/edx-platform/pull/31365 -RUN curl -fsSL https://github.com/openedx/edx-platform/commit/20b93b8b01276edadddfbbb67f15714fddd81c31.patch | git am {%- endif %} -{# Example: RUN curl -fsSL https://github.com/openedx/edx-platform/commit/ | git am #} +{# Example: RUN curl -fsSL https://github.com/openedx/edx-platform/commit/.patch | git am #} {{ patch("openedx-dockerfile-post-git-checkout") }} ###### Download extra locales to /openedx/locale/contrib/locale @@ -67,7 +66,7 @@ RUN apt update && apt install -y software-properties-common libmysqlclient-dev l # https://pypi.org/project/setuptools/ # https://pypi.org/project/pip/ # https://pypi.org/project/wheel/ -RUN pip install setuptools==65.5.1 pip==22.3.1. wheel==0.38.4 +RUN pip install setuptools==67.6.1 pip==23.0.1. wheel==0.40.0 # Install base requirements COPY --from=code /openedx/edx-platform/requirements/edx/base.txt /tmp/base.txt @@ -98,6 +97,7 @@ ENV PATH /openedx/nodeenv/bin:/openedx/venv/bin:${PATH} # Install nodeenv with the version provided by edx-platform # https://github.com/openedx/edx-platform/blob/master/requirements/edx/base.txt +# https://github.com/pyenv/pyenv/releases RUN pip install nodeenv==1.7.0 RUN nodeenv /openedx/nodeenv --node=16.14.0 --prebuilt @@ -224,7 +224,7 @@ USER app RUN pip install -r requirements/edx/development.txt # https://pypi.org/project/ipdb/ # https://pypi.org/project/ipython -RUN pip install ipdb==0.13.9 ipython==8.7.0 +RUN pip install ipdb==0.13.13 ipython==8.12.0 # Add ipdb as default PYTHONBREAKPOINT ENV PYTHONBREAKPOINT=ipdb.set_trace diff --git a/tutor/templates/build/openedx/revisions.yml b/tutor/templates/build/openedx/revisions.yml index ecd9dce..34724a9 100644 --- a/tutor/templates/build/openedx/revisions.yml +++ b/tutor/templates/build/openedx/revisions.yml @@ -1 +1 @@ -EDX_PLATFORM_REVISION: olive +EDX_PLATFORM_REVISION: palm diff --git a/tutor/templates/config/defaults.yml b/tutor/templates/config/defaults.yml index 48de126..e00f3c2 100644 --- a/tutor/templates/config/defaults.yml +++ b/tutor/templates/config/defaults.yml @@ -12,13 +12,13 @@ DOCKER_COMPOSE_VERSION: "3.7" DOCKER_REGISTRY: "docker.io/" DOCKER_IMAGE_OPENEDX: "{{ DOCKER_REGISTRY }}overhangio/openedx:{{ TUTOR_VERSION }}" DOCKER_IMAGE_OPENEDX_DEV: "openedx-dev:{{ TUTOR_VERSION }}" -DOCKER_IMAGE_CADDY: "docker.io/caddy:2.6.3" -DOCKER_IMAGE_ELASTICSEARCH: "docker.io/elasticsearch:7.10.1" +DOCKER_IMAGE_CADDY: "docker.io/caddy:2.6.4" +DOCKER_IMAGE_ELASTICSEARCH: "docker.io/elasticsearch:7.17.9" DOCKER_IMAGE_MONGODB: "docker.io/mongo:4.2.24" -DOCKER_IMAGE_MYSQL: "docker.io/mysql:5.7.35" +DOCKER_IMAGE_MYSQL: "docker.io/mysql:8.0.33" DOCKER_IMAGE_PERMISSIONS: "{{ DOCKER_REGISTRY }}overhangio/openedx-permissions:{{ TUTOR_VERSION }}" -DOCKER_IMAGE_REDIS: "docker.io/redis:6.2.6" -DOCKER_IMAGE_SMTP: "docker.io/devture/exim-relay:4.95-r0-2" +DOCKER_IMAGE_REDIS: "docker.io/redis:7.0.11" +DOCKER_IMAGE_SMTP: "docker.io/devture/exim-relay:4.96-r1-0" EDX_PLATFORM_REPOSITORY: "https://github.com/openedx/edx-platform.git" EDX_PLATFORM_VERSION: "{{ OPENEDX_COMMON_VERSION }}" ELASTICSEARCH_HOST: "elasticsearch" @@ -51,9 +51,9 @@ OPENEDX_CMS_UWSGI_WORKERS: 2 OPENEDX_LMS_UWSGI_WORKERS: 2 OPENEDX_MYSQL_DATABASE: "openedx" OPENEDX_MYSQL_USERNAME: "openedx" -OPENEDX_COMMON_VERSION: "open-release/olive.4" +OPENEDX_COMMON_VERSION: "open-release/palm.master" OPENEDX_EXTRA_PIP_REQUIREMENTS: - - "openedx-scorm-xblock>=15.0.0,<16.0.0" + - "openedx-scorm-xblock>=16.0.0,<17.0.0" MYSQL_HOST: "mysql" MYSQL_PORT: 3306 MYSQL_ROOT_USERNAME: "root" diff --git a/tutor/utils.py b/tutor/utils.py index 15291cd..6492958 100644 --- a/tutor/utils.py +++ b/tutor/utils.py @@ -261,6 +261,19 @@ def check_output(*command: str) -> bytes: raise exceptions.TutorError(f"Command failed: {literal_command}") from e +def warn_macos_docker_memory() -> None: + try: + check_macos_docker_memory() + except exceptions.TutorError as e: + fmt.echo_alert( + f"""Could not verify sufficient RAM allocation in Docker: + + {e} + +Tutor may not work if Docker is configured with < 4 GB RAM. Please follow instructions from: + https://docs.tutor.overhang.io/install.html""" + ) + def check_macos_docker_memory() -> None: """ Try to check that the RAM allocated to the Docker VM on macOS is at least 4 GB.