From 028162fea4fee4126c024508cde61eda0ec8003e Mon Sep 17 00:00:00 2001 From: liupengcheng Date: Fri, 14 May 2021 10:33:41 +0800 Subject: [PATCH] feat: Make openedx redis db configurable. Redis db 0 is used for celery broker and db 1 is used for cache by default. --- CHANGELOG.md | 2 ++ docs/configuration.rst | 4 ++++ tutor/templates/apps/openedx/config/cms.env.json | 13 +++++++------ tutor/templates/apps/openedx/config/lms.env.json | 15 ++++++++------- tutor/templates/config.yml | 2 ++ 5 files changed, 23 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 98ea5d6..4124d08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Note: Breaking changes between versions are indicated by "💥". ## Unreleased +- [Feature] Add redis database configuration for both cache and celery. + ## v11.2.10 (2021-05-17) - [Security] Apply Django security patches by upgrading from 2.2.20 to 2.2.23. diff --git a/docs/configuration.rst b/docs/configuration.rst index 88e04c2..57763d8 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -89,6 +89,10 @@ This defines the default version that will be pulled from all Open edX git repos By default there are 2 `uwsgi worker processes `__ to serve requests for the LMS and the CMS. However, each workers requires upwards of 500 Mb of RAM. You should reduce this value to 1 if your computer/server does not have enough memory. +- ``OPENEDX_CELERY_REDIS_DB`` (default: ``0``) +- ``OPENEDX_CACHE_REDIS_DB`` (default: ``1``) + +These two configuration parameters define which redis database to use for Open edX cache and celery task. Vendor services ~~~~~~~~~~~~~~~ diff --git a/tutor/templates/apps/openedx/config/cms.env.json b/tutor/templates/apps/openedx/config/cms.env.json index 85543ee..06b4b8c 100644 --- a/tutor/templates/apps/openedx/config/cms.env.json +++ b/tutor/templates/apps/openedx/config/cms.env.json @@ -22,6 +22,7 @@ "CONTACT_EMAIL": "{{ CONTACT_EMAIL }}", "CELERY_BROKER_TRANSPORT": "redis", "CELERY_BROKER_HOSTNAME": "{{ REDIS_HOST }}:{{ REDIS_PORT }}", + "CELERY_BROKER_VHOST": "{{ OPENEDX_CELERY_REDIS_DB }}", "CELERY_BROKER_USER": "{{ REDIS_USERNAME }}", "CELERY_BROKER_PASSWORD": "{{ REDIS_PASSWORD }}", "ALTERNATE_WORKER_QUEUES": "lms", @@ -46,18 +47,18 @@ "KEY_PREFIX": "default", "VERSION": "1", "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/1" + "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}" }, "general": { "KEY_PREFIX": "general", "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/1" + "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}" }, "mongo_metadata_inheritance": { "KEY_PREFIX": "mongo_metadata_inheritance", "TIMEOUT": 300, "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/1" + "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}" }, "staticfiles": { "KEY_PREFIX": "staticfiles_cms", @@ -67,19 +68,19 @@ "configuration": { "KEY_PREFIX": "configuration", "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/1" + "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}" }, "celery": { "KEY_PREFIX": "celery", "TIMEOUT": "7200", "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/1" + "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}" }, "course_structure_cache": { "KEY_PREFIX": "course_structure", "TIMEOUT": "7200", "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/1" + "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}" } }, {% include "apps/openedx/config/partials/auth.json" %} diff --git a/tutor/templates/apps/openedx/config/lms.env.json b/tutor/templates/apps/openedx/config/lms.env.json index 0d3707b..3e5c208 100644 --- a/tutor/templates/apps/openedx/config/lms.env.json +++ b/tutor/templates/apps/openedx/config/lms.env.json @@ -29,6 +29,7 @@ "CONTACT_EMAIL": "{{ CONTACT_EMAIL }}", "CELERY_BROKER_TRANSPORT": "redis", "CELERY_BROKER_HOSTNAME": "{{ REDIS_HOST }}:{{ REDIS_PORT }}", + "CELERY_BROKER_VHOST": "{{ OPENEDX_CELERY_REDIS_DB }}", "CELERY_BROKER_USER": "{{ REDIS_USERNAME }}", "CELERY_BROKER_PASSWORD": "{{ REDIS_PASSWORD }}", "ALTERNATE_WORKER_QUEUES": "cms", @@ -55,18 +56,18 @@ "KEY_PREFIX": "default", "VERSION": "1", "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/1" + "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}" }, "general": { "KEY_PREFIX": "general", "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/1" + "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}" }, "mongo_metadata_inheritance": { "KEY_PREFIX": "mongo_metadata_inheritance", "TIMEOUT": 300, "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/1" + "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}" }, "staticfiles": { "KEY_PREFIX": "staticfiles_lms", @@ -76,24 +77,24 @@ "configuration": { "KEY_PREFIX": "configuration", "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/1" + "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}" }, "celery": { "KEY_PREFIX": "celery", "TIMEOUT": "7200", "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/1" + "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}" }, "course_structure_cache": { "KEY_PREFIX": "course_structure", "TIMEOUT": "7200", "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/1" + "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}" }, "ora2-storage": { "KEY_PREFIX": "ora2-storage", "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/1" + "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}" } }, {% include "apps/openedx/config/partials/auth.json" %} diff --git a/tutor/templates/config.yml b/tutor/templates/config.yml index 7068de2..4d3b1c3 100644 --- a/tutor/templates/config.yml +++ b/tutor/templates/config.yml @@ -63,6 +63,8 @@ OPENEDX_MYSQL_DATABASE: "openedx" OPENEDX_CSMH_MYSQL_DATABASE: "{{ OPENEDX_MYSQL_DATABASE }}_csmh" OPENEDX_MYSQL_USERNAME: "openedx" OPENEDX_COMMON_VERSION: "open-release/koa.3" +OPENEDX_CELERY_REDIS_DB: 0 +OPENEDX_CACHE_REDIS_DB: 1 MYSQL_HOST: "mysql" MYSQL_PORT: 3306 MYSQL_ROOT_USERNAME: "root"