From 59b1987ff1d0269b81fb4e2264a2b4a7781c1abf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Thu, 8 Feb 2024 11:20:10 +0100 Subject: [PATCH] fix: infinite cache growth See the discussion here: https://github.com/overhangio/tutor/pull/984 And the upstream PR here: https://github.com/openedx/edx-platform/pull/34210 The tl;dr is that the Redis course structure cache was growing without bounds. While the upstream fix should resolve that issue, we decided that Tutor should have a maxmemory limit and an eviction policy set for operational safety. Thus, Redis now has a 4gb maxmemory. If you need more memory on your instance, you should implement the "redis-conf" patch. To manually expire existing keys, run: tutor local run cms ./manage.py cms shell -c "from django.core.cache import caches; c = caches['course_structure_cache']; [c.expire(key, 604800) for key in c.keys('*')]" --- changelog.d/20240208_111236_regis_infinite_course_cache.md | 4 ++++ docs/reference/patches.rst | 7 +++++++ .../templates/apps/openedx/settings/partials/common_all.py | 2 +- tutor/templates/apps/redis/redis.conf | 7 +++++++ tutor/templates/build/openedx/Dockerfile | 3 +++ 5 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 changelog.d/20240208_111236_regis_infinite_course_cache.md diff --git a/changelog.d/20240208_111236_regis_infinite_course_cache.md b/changelog.d/20240208_111236_regis_infinite_course_cache.md new file mode 100644 index 0000000..82126b1 --- /dev/null +++ b/changelog.d/20240208_111236_regis_infinite_course_cache.md @@ -0,0 +1,4 @@ +- 💥[Bugfix] Prevent infinite growth of course structure cache in Redis. (by @regisb) + - Redis is now configured with a maximum memory size of 4GB. If this is too low for your platform, you should increase this value using the new "redis-conf" patch. + - Make sure that course structure cache keys have an actual timeout. +- [Feature] Introduce the "redis-conf" patch. (by @regisb) diff --git a/docs/reference/patches.rst b/docs/reference/patches.rst index e581786..47ae131 100644 --- a/docs/reference/patches.rst +++ b/docs/reference/patches.rst @@ -376,6 +376,13 @@ File: ``apps/openedx/settings/lms/production.py`` Python-formatted LMS settings in production. Values defined here override the values from :patch:`openedx-lms-common-settings`. +``redis-conf`` +============== + +File: ``apps/redis/redis.conf`` + +Implement this patch to override hard-coded Redis configuration values. See the `Redis configuration reference `__`. + ``uwsgi-config`` ================ diff --git a/tutor/templates/apps/openedx/settings/partials/common_all.py b/tutor/templates/apps/openedx/settings/partials/common_all.py index c088b97..fb963be 100644 --- a/tutor/templates/apps/openedx/settings/partials/common_all.py +++ b/tutor/templates/apps/openedx/settings/partials/common_all.py @@ -73,7 +73,7 @@ CACHES = { }, "course_structure_cache": { "KEY_PREFIX": "course_structure", - "TIMEOUT": 7200, + "TIMEOUT": 604800, # 1 week "BACKEND": "django_redis.cache.RedisCache", "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}", }, diff --git a/tutor/templates/apps/redis/redis.conf b/tutor/templates/apps/redis/redis.conf index cfdf709..1fdd69b 100644 --- a/tutor/templates/apps/redis/redis.conf +++ b/tutor/templates/apps/redis/redis.conf @@ -39,3 +39,10 @@ auto-aof-rewrite-percentage 100 auto-aof-rewrite-min-size 64mb aof-load-truncated yes aof-use-rdb-preamble yes + +############################## MEMORY MANAGEMENT ################################ + +maxmemory 4gb +maxmemory-policy allkeys-lru + +{{ patch("redis-conf") }} diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index 0aef8e8..1cbad72 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -51,6 +51,9 @@ RUN git config --global user.email "tutor@overhang.io" \ {{ patch("openedx-dockerfile-git-patches-default") }} {%- else %} # Patch edx-platform +# Prevent course structure cache infinite growth +# https://github.com/openedx/edx-platform/pull/34210 +RUN curl -fsSL https://github.com/openedx/edx-platform/commit/ad201cd664b6c722cbefcbda23ae390c06daf621.patch | git am {%- endif %} {# Example: RUN curl -fsSL https://github.com/openedx/edx-platform/commit/.patch | git am #}