From 53ebfd45be632a51cf49f3fa921253bbb44bb94a Mon Sep 17 00:00:00 2001 From: Omar Al-Ithawi Date: Fri, 15 Dec 2023 19:30:22 +0300 Subject: [PATCH] feat: add atlas pull with global settings --- changelog.d/20231219_214829_i_atlas.md | 1 + docs/configuration.rst | 68 +++---------------- docs/troubleshooting.rst | 4 +- .../openedx/settings/partials/common_all.py | 2 - tutor/templates/build/openedx/Dockerfile | 23 ++----- .../build/openedx/locale/customlocales.md | 13 ---- .../build/openedx/settings/cms/assets.py | 3 - .../build/openedx/settings/lms/assets.py | 3 - .../build/openedx/settings/partials/i18n.py | 2 - tutor/templates/config/defaults.yml | 3 + 10 files changed, 22 insertions(+), 100 deletions(-) create mode 100644 changelog.d/20231219_214829_i_atlas.md delete mode 100644 tutor/templates/build/openedx/locale/customlocales.md diff --git a/changelog.d/20231219_214829_i_atlas.md b/changelog.d/20231219_214829_i_atlas.md new file mode 100644 index 0000000..6c3ef1b --- /dev/null +++ b/changelog.d/20231219_214829_i_atlas.md @@ -0,0 +1 @@ +- 💥[Feature] Pull translations via `atlas` during Docker build. This breaks the `openedx-i18n` custom locale Tutor feature in favor of [OEP-58](https://docs.openedx.org/en/latest/developers/concepts/oep58.html) in favor of . (by @omarithawi) diff --git a/docs/configuration.rst b/docs/configuration.rst index 0f88d79..6260d29 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -387,68 +387,22 @@ If you don't create your fork from this tag, you *will* have important compatibi .. _i18n: -Adding custom translations -~~~~~~~~~~~~~~~~~~~~~~~~~~ +Getting and customizing Translations +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -If you are not running Open edX in English (``LANGUAGE_CODE`` default: ``"en"``), chances are that some strings will not be properly translated. In most cases, this is because not enough contributors have helped translate Open edX into your language. It happens! With Tutor, available translated languages include those that come bundled with `edx-platform `__ as well as those from `openedx-i18n `__. +Tutor builds images with the latest translations using the ```atlas pull`` command `__. -Tutor offers a relatively simple mechanism to add custom translations to the openedx Docker image. You should create a folder that corresponds to your language code in the "build/openedx/locale" folder of the Tutor environment. This folder should contain a "LC_MESSAGES" folder. For instance:: +By default the translations are pulled from the `openedx-translations repository `_ +from the ``ATLAS_REVISION`` branch. You can use custom translations on your fork of the openedx-translations repository by setting the following configuration parameters: - mkdir -p "$(tutor config printroot)/env/build/openedx/locale/fr/LC_MESSAGES" +- ``ATLAS_REVISION`` (default: ``"main"`` on nightly and ``"{{ OPENEDX_COMMON_VERSION }}"`` if a named release is used) +- ``ATLAS_REPOSITORY`` (default: ``"openedx/openedx-translations"``). There's a feature request to `support GitLab and other providers `_. +- ``ATLAS_OPTIONS`` (default: ``""``) Pass additional arguments to ``atlas pull``. Refer to the `atlas documentations `_ for more information. -The language code should be similar to those used in edx-platform or openedx-i18n (see links above). - -Then, add a "django.po" file there that will contain your custom translations:: - - msgid "" - msgstr "" - "Content-Type: text/plain; charset=UTF-8" - - msgid "String to translate" - msgstr "你翻译的东西 la traduction de votre bidule" - - -.. warning:: - Don't forget to specify the file ``Content-Type`` when adding message strings with non-ASCII characters; otherwise a ``UnicodeDecodeError`` will be raised during compilation. - -The "String to translate" part should match *exactly* the string that you would like to translate. You cannot make it up! The best way to find this string is to copy-paste it from the `upstream django.po file for the English language `__. - -If you cannot find the string to translate in this file, then it means that you are trying to translate a string that is used in some piece of javascript code. Those strings are stored in a different file named "djangojs.po". You can check it out `in the edx-platform repo as well `__. Your custom javascript strings should also be stored in a "djangojs.po" file that should be placed in the same directory. - -To recap, here is an example. To translate a few strings in French, both from django.po and djangojs.po, we would have the following file hierarchy:: - - $(tutor config printroot)/env/build/openedx/locale/ - fr/ - LC_MESSAGES/ - django.po - djangojs.po - -With django.po containing:: - - msgid "" - msgstr "" - "Content-Type: text/plain; charset=UTF-8" - - msgid "It works! Powered by Open edX{registered_trademark}" - msgstr "Ça marche ! Propulsé by Open edX{registered_trademark}" - -And djangojs.po:: - - msgid "" - msgstr "" - "Content-Type: text/plain; charset=UTF-8" - - msgid "%(num_points)s point possible (graded, results hidden)" - msgid_plural "%(num_points)s points possible (graded, results hidden)" - msgstr[0] "%(num_points)s point possible (noté, résultats cachés)" - msgstr[1] "%(num_points)s points possibles (notés, résultats cachés)" - -Then you will have to re-build the openedx Docker image:: - - tutor images build openedx - -Beware that this will take a long time! Unfortunately, it's difficult to accelerate this process, as translation files need to be compiled before collecting the assets. In development it's possible to accelerate the iteration loop -- but that exercise is left to the reader. +If you are not running Open edX in English (``LANGUAGE_CODE`` default: ``"en"``), chances are that some strings will not be properly translated. In most cases, this is because not enough contributors have helped translate Open edX into your language. It happens! +With ``atlas``, it's possible to add custom translations by either `contributing to the Translations project in Transifex `_ or forking the `openedx-translations repository `_ +and making custom changes as explained in `the repository docs `_. Running a different ``openedx`` Docker image ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/troubleshooting.rst b/docs/troubleshooting.rst index 8c2bc8a..7e78608 100644 --- a/docs/troubleshooting.rst +++ b/docs/troubleshooting.rst @@ -158,7 +158,9 @@ The error produced should help you better understand what is happening. The chosen default language does not display properly ----------------------------------------------------- -By default, Open edX comes with a `limited set ` of translation/localization files. To complement these languages, we add locales from the `openedx-i18n project `_. But not all supported locales are downloaded. In some cases, the chosen default language will not display properly because it was not packaged in either edx-platform or openedx-i18n. If you feel like your language should be packaged, please `open an issue on the openedx-i18n project `_. +By default, Open edX comes with a `limited set ` of translation/localization files. + +Refer to the :ref:`i18n` section for more information about using your own translations. When I make changes to a course in the CMS, they are not taken into account by the LMS -------------------------------------------------------------------------------------- diff --git a/tutor/templates/apps/openedx/settings/partials/common_all.py b/tutor/templates/apps/openedx/settings/partials/common_all.py index cf3bc44..3db8bf9 100644 --- a/tutor/templates/apps/openedx/settings/partials/common_all.py +++ b/tutor/templates/apps/openedx/settings/partials/common_all.py @@ -178,8 +178,6 @@ ACE_CHANNEL_TRANSACTIONAL_EMAIL = "django_email" EMAIL_FILE_PATH = "/tmp/openedx/emails" # Language/locales -LOCALE_PATHS.append("/openedx/locale/contrib/locale") -LOCALE_PATHS.append("/openedx/locale/user/locale") LANGUAGE_COOKIE_NAME = "openedx-language-preference" # Allow the platform to include itself in an iframe diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index 0aef8e8..9010de1 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -67,15 +67,6 @@ COPY --from=code /openedx/edx-platform / FROM scratch as mnt-{{ name }} {% endfor %} -###### Download extra locales to /openedx/locale/contrib/locale -FROM minimal as locales -ARG OPENEDX_I18N_REPOSITORY=https://github.com/openedx/openedx-i18n.git -ARG OPENEDX_I18N_VERSION=release/quince -ADD --keep-git-dir=true $OPENEDX_I18N_REPOSITORY#$OPENEDX_I18N_VERSION /tmp/openedx-i18n -RUN mkdir --parents /openedx/locale && \ - mv /tmp/openedx-i18n/edx-platform/locale /openedx/locale/contrib && \ - rm -rf /tmp/openedx-i18n/ - ###### Install python requirements in virtualenv FROM python as python-requirements ENV PATH /openedx/venv/bin:${PATH} @@ -156,7 +147,6 @@ USER ${APP_USER_ID} # https://hub.docker.com/r/powerman/dockerize/tags COPY --link --from=docker.io/powerman/dockerize:0.19.0 /usr/local/bin/dockerize /usr/local/bin/dockerize COPY --chown=app:app --from=edx-platform / /openedx/edx-platform -COPY --chown=app:app --from=locales /openedx/locale /openedx/locale COPY --chown=app:app --from=python /opt/pyenv /opt/pyenv COPY --chown=app:app --from=python-requirements /openedx/venv /openedx/venv COPY --chown=app:app --from=python-requirements /mnt /mnt @@ -190,15 +180,10 @@ ENV REVISION_CFG /openedx/config/revisions.yml COPY --chown=app:app settings/lms/*.py ./lms/envs/tutor/ COPY --chown=app:app settings/cms/*.py ./cms/envs/tutor/ -# Copy user-specific locales to /openedx/locale/user/locale and compile them -RUN mkdir /openedx/locale/user -COPY --chown=app:app ./locale/ /openedx/locale/user/locale/ -RUN cd /openedx/locale/user && \ - django-admin compilemessages -v1 - -# Compile i18n strings: in some cases, js locales are not properly compiled out of the box -# and we need to do a pass ourselves. Also, we need to compile the djangojs.js files for -# the downloaded locales. +# Pull latest translations via atlas +RUN atlas pull --repository='{{ ATLAS_REPOSITORY }}' --branch='{{ ATLAS_REVISION }}' {{ ATLAS_OPTIONS }} \ + translations/edx-platform/conf/locale:conf/locale +RUN ./manage.py lms --settings=tutor.i18n compilemessages -v1 RUN ./manage.py lms --settings=tutor.i18n compilejsi18n RUN ./manage.py cms --settings=tutor.i18n compilejsi18n diff --git a/tutor/templates/build/openedx/locale/customlocales.md b/tutor/templates/build/openedx/locale/customlocales.md deleted file mode 100644 index a098936..0000000 --- a/tutor/templates/build/openedx/locale/customlocales.md +++ /dev/null @@ -1,13 +0,0 @@ -Add your custom translations to this folder, with the following filesystem structure: - - languagecode/ - LC_MESSAGES/ - django.po - djangojs.po - -Where "languagecode" is one of "fr", "de_DE", "zh_CN", etc. - -The localized string in the *.po file should have the following format: - - msgid "String to translate" - msgstr "Your custom translation 你发音的东西 le bidule que vous voulez traduire" diff --git a/tutor/templates/build/openedx/settings/cms/assets.py b/tutor/templates/build/openedx/settings/cms/assets.py index 59b1dd0..914453d 100644 --- a/tutor/templates/build/openedx/settings/cms/assets.py +++ b/tutor/templates/build/openedx/settings/cms/assets.py @@ -4,6 +4,3 @@ STATIC_ROOT = path(STATIC_ROOT_BASE) / 'studio' WEBPACK_LOADER['DEFAULT']['STATS_FILE'] = STATIC_ROOT / "webpack-stats.json" derive_settings(__name__) - -LOCALE_PATHS.append("/openedx/locale/contrib/locale") -LOCALE_PATHS.append("/openedx/locale/user/locale") diff --git a/tutor/templates/build/openedx/settings/lms/assets.py b/tutor/templates/build/openedx/settings/lms/assets.py index 4cf476d..b3f85de 100644 --- a/tutor/templates/build/openedx/settings/lms/assets.py +++ b/tutor/templates/build/openedx/settings/lms/assets.py @@ -4,6 +4,3 @@ STATIC_ROOT = path(STATIC_ROOT_BASE) WEBPACK_LOADER['DEFAULT']['STATS_FILE'] = STATIC_ROOT / "webpack-stats.json" derive_settings(__name__) - -LOCALE_PATHS.append("/openedx/locale/contrib/locale") -LOCALE_PATHS.append("/openedx/locale/user/locale") diff --git a/tutor/templates/build/openedx/settings/partials/i18n.py b/tutor/templates/build/openedx/settings/partials/i18n.py index 708c250..3ca5259 100644 --- a/tutor/templates/build/openedx/settings/partials/i18n.py +++ b/tutor/templates/build/openedx/settings/partials/i18n.py @@ -14,7 +14,5 @@ DATABASES = { derive_settings(__name__) -LOCALE_PATHS.append("/openedx/locale/contrib/locale") -LOCALE_PATHS.append("/openedx/locale/user/locale") {{ patch("openedx-common-i18n-settings") }} diff --git a/tutor/templates/config/defaults.yml b/tutor/templates/config/defaults.yml index 9ab1967..d49ce62 100644 --- a/tutor/templates/config/defaults.yml +++ b/tutor/templates/config/defaults.yml @@ -2,6 +2,9 @@ # This file includes all Tutor setting defaults. Settings that do not have a # default value, such as passwords, should be stored in base.yml. # This must be defined early +ATLAS_REVISION: "{% if OPENEDX_COMMON_VERSION == 'master' %}main{% else %}{{ OPENEDX_COMMON_VERSION }}{% endif %}" +ATLAS_REPOSITORY: "openedx/openedx-translations" +ATLAS_OPTIONS: "" CADDY_HTTP_PORT: 80 CMS_HOST: "studio.{{ LMS_HOST }}" CMS_OAUTH2_KEY_SSO: "cms-sso"