Fix missing video transcripts in LMS

Video transcripts uploaded in the CMS were not visible in the LMS. This
was a symptom caused by the fact that the LMS and the CMS do not share
the same MEDIA_ROOT. We initially thought that data uploaded in the CMS
(such as transcripts) was stored in a shared data service, such as
mongodb. It is, in fact, not. This makes it even more important to run
an object storage service like minio for distributed services.

Close #229

CHANGELOG.md

@@ -4,6 +4,7 @@ Note: Breaking changes between versions are indicated by "💥".
 
 ## Latest
 
+- [Bugfix] Fix missing video transcripts in LMS (#229)
 - [Improvement] Make it possible to enable/disable multiple plugins at once
 - [Improvement] Add 'local-docker-compose-nginx-volumes' patch
 

tutor/templates/apps/nginx/cms.conf

@@ -40,7 +40,7 @@ server {
   }
 
   location ~ ^/media/(?P<file>.*) {
-    root /openedx/data/cms/uploads;
+    root /var/www/openedx-media;
     try_files /$file =404;
     expires 31536000s;
   }

tutor/templates/apps/nginx/lms.conf

@@ -56,7 +56,7 @@ server {
   }
 
   location ~ ^/media/(?P<file>.*) {
-    root /openedx/data/lms/uploads;
+    root /var/www/openedx-media;
     try_files /$file =404;
     expires 31536000s;
   }

tutor/templates/apps/openedx/config/cms.env.json

@@ -22,7 +22,6 @@
   "CELERY_BROKER_USER": "{{ RABBITMQ_USERNAME }}",
   "CELERY_BROKER_PASSWORD": "{{ RABBITMQ_PASSWORD }}",
   "COMPREHENSIVE_THEME_DIRS": ["/openedx/themes"],
-  "MEDIA_ROOT": "/openedx/data/uploads/",
   "STATIC_ROOT_BASE": "/openedx/staticfiles",
   "ELASTIC_SEARCH_CONFIG": [{
     "host": "{{ ELASTICSEARCH_HOST }}",

tutor/templates/apps/openedx/config/lms.env.json

@@ -28,7 +28,6 @@
   "COMMENTS_SERVICE_URL": "http://{{ FORUM_HOST }}:4567",
   "COMMENTS_SERVICE_KEY": "forumapikey",
   "COMPREHENSIVE_THEME_DIRS": ["/openedx/themes"],
-  "MEDIA_ROOT": "/openedx/data/uploads/",
   "STATIC_ROOT_BASE": "/openedx/staticfiles",
   "ELASTIC_SEARCH_CONFIG": [{
     "host": "{{ ELASTICSEARCH_HOST }}",

tutor/templates/apps/openedx/settings/cms/common.py

@@ -2,7 +2,7 @@
 update_module_store_settings(MODULESTORE, doc_store_settings=DOC_STORE_CONFIG)
 
 # Set uploaded media file path
-MEDIA_ROOT = "/openedx/data/uploads/"
+MEDIA_ROOT = "/openedx/media/"
 
 # Video settings
 VIDEO_IMAGE_SETTINGS["STORAGE_KWARGS"]["location"] = MEDIA_ROOT

tutor/templates/apps/openedx/settings/lms/common.py

@@ -4,7 +4,7 @@
 update_module_store_settings(MODULESTORE, doc_store_settings=DOC_STORE_CONFIG)
 
 # Set uploaded media file path
-MEDIA_ROOT = "/openedx/data/uploads/"
+MEDIA_ROOT = "/openedx/media/"
 
 # Video settings
 VIDEO_IMAGE_SETTINGS["STORAGE_KWARGS"]["location"] = MEDIA_ROOT
@@ -31,7 +31,6 @@ PROFILE_IMAGE_BACKEND["options"]["location"] = os.path.join(
     MEDIA_ROOT, "profile-images/"
 )
 
-
 ORA2_FILEUPLOAD_BACKEND = "filesystem"
 ORA2_FILEUPLOAD_ROOT = "/openedx/data/ora2"
 ORA2_FILEUPLOAD_CACHE_NAME = "ora2-storage"
@@ -40,7 +39,7 @@ GRADES_DOWNLOAD = {
     "STORAGE_TYPE": "",
     "STORAGE_KWARGS": {
         "base_url": "/media/grades/",
-        "location": os.path.join(MEDIA_ROOT, "grades"),
+        "location": "/openedx/media/grades",
     },
 }
 

tutor/templates/hooks/cms/init

@@ -1 +1,10 @@
 dockerize -wait tcp://{{ MYSQL_HOST }}:{{ MYSQL_PORT }} -timeout 20s && ./manage.py cms migrate
+
+# Fix incorrect uploaded file path
+if [ -d /openedx/data/uploads/ ]; then
+  if [ -n "$(ls -A /openedx/data/uploads/)" ]; then
+    echo "Migrating CMS uploaded files to shared directory"
+    mv /openedx/data/uploads/* /openedx/media/
+    rm -rf /openedx/data/uploads/
+  fi
+fi

tutor/templates/hooks/lms/init

@@ -4,4 +4,13 @@ dockerize -wait tcp://{{ MYSQL_HOST }}:{{ MYSQL_PORT }} -timeout 20s
 ./manage.py lms --settings=tutor.production create_oauth2_client \
     "http://androidapp.com" "http://androidapp.com/redirect" public \
     --client_id android --client_secret {{ ANDROID_OAUTH2_SECRET }} \
-    --trusted
+    --trusted
+    
+# Fix incorrect uploaded file path
+if [ -d /openedx/data/uploads/ ]; then
+  if [ -n "$(ls -A /openedx/data/uploads/)" ]; then
+    echo "Migrating LMS uploaded files to shared directory"
+    mv /openedx/data/uploads/* /openedx/media/
+    rm -rf /openedx/data/uploads/
+  fi
+fi

tutor/templates/local/docker-compose.yml

@@ -69,8 +69,7 @@ services:
     volumes:
       - ../apps/nginx:/etc/nginx/conf.d/:ro
       - ../../data/openedx:/var/www/openedx:ro
-      {% if ACTIVATE_CMS %}- ../../data/cms:/openedx/data/cms/:ro{% endif %}
-      {% if ACTIVATE_LMS %}- ../../data/lms:/openedx/data/lms/:ro{% endif %}
+      - ../../data/openedx-media:/var/www/openedx-media:ro
       {% if ACTIVATE_HTTPS %}- ../../data/letsencrypt:/etc/letsencrypt/:ro{% endif %}
       {{ patch("local-docker-compose-nginx-volumes")|indent(6) }}
     depends_on:
@@ -121,6 +120,7 @@ services:
       - ../apps/openedx/settings/cms/:/openedx/edx-platform/cms/envs/tutor/
       - ../apps/openedx/config/:/openedx/config/
       - ../../data/lms:/openedx/data
+      - ../../data/openedx-media:/openedx/media
     depends_on:
       {% if ACTIVATE_ELASTICSEARCH %}- elasticsearch{% endif %}
       {% if ACTIVATE_FORUM %}- forum{% endif %}
@@ -143,6 +143,7 @@ services:
       - ../apps/openedx/settings/cms/:/openedx/edx-platform/cms/envs/tutor/
       - ../apps/openedx/config/:/openedx/config/
       - ../../data/cms:/openedx/data
+      - ../../data/openedx-media:/openedx/media
     depends_on:
       {% if ACTIVATE_ELASTICSEARCH %}- elasticsearch{% endif %}
       {% if ACTIVATE_MEMCACHED %}- memcached{% endif %}
@@ -168,6 +169,7 @@ services:
       - ../apps/openedx/settings/cms/:/openedx/edx-platform/cms/envs/tutor/
       - ../apps/openedx/config/:/openedx/config/
       - ../../data/lms:/openedx/data
+      - ../../data/openedx-media:/openedx/media
     depends_on:
       - lms
   {% endif %}
@@ -186,6 +188,7 @@ services:
       - ../apps/openedx/settings/cms/:/openedx/edx-platform/cms/envs/tutor/
       - ../apps/openedx/config/:/openedx/config/
       - ../../data/cms:/openedx/data
+      - ../../data/openedx-media:/openedx/media
     depends_on:
       - cms
   {% endif %}