Refactoring: merge ops and scripts module

The "scripts" template folder is not rendered to the environment, as we
want to keep the possibility to make radical changes to this folder.

tutor/config.py

@@ -165,7 +165,7 @@ def load_defaults(config):
             config[k] = v
 
 def ask(question, key, config):
-    default = env.render_str(config[key], config)
+    default = env.render_str(config, config[key])
     config[key] = click.prompt(
         fmt.question(question),
         prompt_suffix=" ", default=default, show_default=True,

tutor/env.py

@@ -29,14 +29,17 @@ def render_target(root, config, target):
     """
     for src, dst in walk_templates(root, target):
         if is_part_of_env(src):
-            with codecs.open(src, encoding='utf-8') as fi:
-                try:
-                    substituted = render_str(fi.read(), config)
-                except jinja2.exceptions.TemplateError:
-                    print("Error rendering template", src)
-                    raise
+            rendered = render_file(config, src)
             with open(dst, "w") as of:
-                of.write(substituted)
+                of.write(rendered)
+
+def render_file(config, path):
+    with codecs.open(path, encoding='utf-8') as fi:
+        try:
+            return render_str(config, fi.read())
+        except jinja2.exceptions.TemplateError:
+            print("Error rendering template", path)
+            raise
 
 def render_dict(config):
     """
@@ -49,14 +52,14 @@ def render_dict(config):
     rendered = {}
     for key, value in config.items():
         if isinstance(value, str):
-            rendered[key] = render_str(value, config)
+            rendered[key] = render_str(config, value)
         else:
             rendered[key] = value
     for k, v in rendered.items():
         config[k] = v
     pass
 
-def render_str(text, config):
+def render_str(config, text):
     """
     Args:
         text (str)
@@ -103,7 +106,7 @@ def read(*path):
     """
     Read template content located at `path`.
     """
-    src = os.path.join(TEMPLATES_ROOT, *path)
+    src = template_path(*path)
     with codecs.open(src, encoding='utf-8') as fi:
         return fi.read()
 
@@ -115,8 +118,8 @@ def walk_templates(root, target):
         src: template path
         dst: destination path inside root
     """
-    target_root = os.path.join(TEMPLATES_ROOT, target)
-    for dirpath, _, filenames in os.walk(os.path.join(TEMPLATES_ROOT, target)):
+    target_root = template_path(target)
+    for dirpath, _, filenames in os.walk(target_root):
         dst_dir = pathjoin(
             root, target,
             os.path.relpath(dirpath, target_root)
@@ -128,6 +131,9 @@ def walk_templates(root, target):
             dst = os.path.join(dst_dir, filename)
             yield src, dst
 
+def template_path(*path):
+    return os.path.join(TEMPLATES_ROOT, *path)
+
 def data_path(root, *path):
     return os.path.join(os.path.abspath(root), "data", *path)
 

tutor/k8s.py

@@ -6,7 +6,7 @@ from . import env as tutor_env
 from . import exceptions
 from . import fmt
 from . import opts
-from . import ops
+from . import scripts
 from . import utils
 
 
@@ -62,7 +62,7 @@ def delete(yes):
 )
 @opts.root
 def databases(root):
-    ops.migrate(root, run_bash)
+    scripts.migrate(root, run_bash)
 
 @click.command(help="Create an Open edX user and interactively set their password")
 @opts.root
@@ -71,13 +71,13 @@ def databases(root):
 @click.argument("name")
 @click.argument("email")
 def createuser(root, superuser, staff, name, email):
-    ops.create_user(root, run_bash, superuser, staff, name, email)
+    scripts.create_user(root, run_bash, superuser, staff, name, email)
 
 @click.command(help="Import the demo course")
 @opts.root
 def importdemocourse(root):
     click.echo(fmt.info("Importing demo course"))
-    ops.import_demo_course(root, run_bash)
+    scripts.import_demo_course(root, run_bash)
     click.echo(fmt.info("Re-indexing courses"))
     indexcourses.callback(root)
 
@@ -86,7 +86,7 @@ def importdemocourse(root):
 def indexcourses(root):
     # Note: this is currently broken with "pymongo.errors.ConnectionFailure: [Errno 111] Connection refused"
     # I'm not quite sure the settings are correctly picked up. Which is weird because migrations work very well.
-    ops.index_courses(root, run_bash)
+    scripts.index_courses(root, run_bash)
 
 @click.command(
     help="Launch a shell in LMS or CMS",

tutor/local.py

@@ -6,13 +6,11 @@ from time import sleep
 import click
 
 from . import config as tutor_config
-from . import exceptions
 from . import fmt
 from . import opts
 from . import scripts
 from . import utils
 from . import env as tutor_env
-from . import ops
 
 
 @click.group(
@@ -122,7 +120,7 @@ def run(root, service, command, args):
 @opts.root
 def databases(root):
     init_mysql(root)
-    ops.migrate(root, run_bash)
+    scripts.migrate(root, run_bash)
 
 def init_mysql(root):
     config = tutor_config.load(root)
@@ -166,11 +164,13 @@ def https_create(root):
         click.echo(fmt.info("HTTPS is not activated: certificate generation skipped"))
         return
 
-    script = tutor_env.render_str(scripts.https_certificates_create, config)
+    script = scripts.render_template(config, 'https_create.sh')
 
     if config['WEB_PROXY']:
         click.echo(fmt.info(
-            """You are running Tutor behind a web proxy (WEB_PROXY=true): SSL/TLS certificates must be generated on the host. For instance, to generate certificates with Let's Encrypt, run:
+"""You are running Tutor behind a web proxy (WEB_PROXY=true): SSL/TLS
+certificates must be generated on the host. For instance, to generate
+certificates with Let's Encrypt, run:
 
 {}
 
@@ -194,11 +194,13 @@ def https_renew(root):
         return
     if config['WEB_PROXY']:
         click.echo(fmt.info(
-            """You are running Tutor behind a web proxy (WEB_PROXY=true): SSL/TLS certificates must be renewed on the host. For instance, to renew Let's Encrypt certificates, run:
+"""You are running Tutor behind a web proxy (WEB_PROXY=true): SSL/TLS
+certificates must be renewed on the host. For instance, to renew Let's Encrypt
+certificates, run:
 
     certbot renew
 
-See the official certbot documentation: for your platform https://certbot.eff.org/"""))
+See the official certbot documentation for your platform: https://certbot.eff.org/"""))
         return
     docker_run = [
         "--volume", "{}:/etc/letsencrypt/".format(tutor_env.data_path(root, "letsencrypt")),
@@ -229,20 +231,20 @@ def logs(root, follow, tail, service):
 @click.argument("name")
 @click.argument("email")
 def createuser(root, superuser, staff, name, email):
-    ops.create_user(root, run_bash, superuser, staff, name, email)
+    scripts.create_user(root, run_bash, superuser, staff, name, email)
 
 @click.command(help="Import the demo course")
 @opts.root
 def importdemocourse(root):
     click.echo(fmt.info("Importing demo course"))
-    ops.import_demo_course(root, run_bash)
+    scripts.import_demo_course(root, run_bash)
     click.echo(fmt.info("Re-indexing courses"))
     indexcourses.callback(root)
 
 @click.command(help="Re-index courses for better searching")
 @opts.root
 def indexcourses(root):
-    ops.index_courses(root, run_bash)
+    scripts.index_courses(root, run_bash)
 
 @click.command(
     help="Run Portainer (https://portainer.io), a UI for container supervision",

tutor/ops.py

@@ -1,51 +0,0 @@
-import click
-
-from . import config as tutor_config
-from . import env
-from . import fmt
-from . import scripts
-
-# TODO merge this with scripts.py
-
-def migrate(root, run_func):
-    config = tutor_config.load(root)
-    click.echo(fmt.info("Creating lms/cms databases..."))
-    run_template(config, root, "lms", scripts.create_databases, run_func)
-    click.echo(fmt.info("Running lms migrations..."))
-    run_template(config, root, "lms", scripts.migrate_lms, run_func)
-    click.echo(fmt.info("Running cms migrations..."))
-    run_template(config, root, "cms", scripts.migrate_cms, run_func)
-    click.echo(fmt.info("Running forum migrations..."))
-    run_template(config, root, "forum", scripts.migrate_forum, run_func)
-    if config["ACTIVATE_NOTES"]:
-        click.echo(fmt.info("Running notes migrations..."))
-        run_template(config, root, "notes", scripts.migrate_notes, run_func)
-    if config["ACTIVATE_XQUEUE"]:
-        click.echo(fmt.info("Running xqueue migrations..."))
-        run_template(config, root, "xqueue", scripts.migrate_xqueue, run_func)
-    click.echo(fmt.info("Creating oauth2 users..."))
-    run_template(config, root, "lms", scripts.oauth2, run_func)
-    click.echo(fmt.info("Databases ready."))
-
-def create_user(root, run_func, superuser, staff, name, email):
-    config = {
-        "OPTS": "",
-        "USERNAME": name,
-        "EMAIL": email,
-    }
-    if superuser:
-        config["OPTS"] += " --superuser"
-    if staff:
-        config["OPTS"] += " --staff"
-    run_template(config, root, "lms", scripts.create_user, run_func)
-
-def import_demo_course(root, run_func):
-    run_template({}, root, "cms", scripts.import_demo_course, run_func)
-
-def index_courses(root, run_func):
-    run_template({}, root, "cms", scripts.index_courses, run_func)
-
-def run_template(config, root, service, template, run_func):
-    command = env.render_str(template, config).strip()
-    if command:
-        run_func(root, service, command)

tutor/scripts.py

@@ -1,43 +1,52 @@
-create_databases = """dockerize -wait tcp://{{ MYSQL_HOST }}:{{ MYSQL_PORT }} -timeout 20s
-mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" -e 'CREATE DATABASE IF NOT EXISTS {{ OPENEDX_MYSQL_DATABASE }};'
-mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" -e 'GRANT ALL ON {{ OPENEDX_MYSQL_DATABASE }}.* TO "{{ OPENEDX_MYSQL_USERNAME }}"@"%" IDENTIFIED BY "{{ OPENEDX_MYSQL_PASSWORD }}";'
+import click
 
-{% if ACTIVATE_NOTES %}
-mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" -e 'CREATE DATABASE IF NOT EXISTS {{ NOTES_MYSQL_DATABASE }};'
-mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" -e 'GRANT ALL ON {{ NOTES_MYSQL_DATABASE }}.* TO "{{ NOTES_MYSQL_USERNAME }}"@"%" IDENTIFIED BY "{{ NOTES_MYSQL_PASSWORD }}";'
-{% endif %}
+from . import config as tutor_config
+from . import env
+from . import fmt
 
-{% if ACTIVATE_XQUEUE %}
-mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" -e 'CREATE DATABASE IF NOT EXISTS {{ XQUEUE_MYSQL_DATABASE }};'
-mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" -e 'GRANT ALL ON {{ XQUEUE_MYSQL_DATABASE }}.* TO "{{ XQUEUE_MYSQL_USERNAME }}"@"%" IDENTIFIED BY "{{ XQUEUE_MYSQL_PASSWORD }}";'
-{% endif %}
-"""
+def migrate(root, run_func):
+    config = tutor_config.load(root)
+    click.echo(fmt.info("Creating lms/cms databases..."))
+    run_template(root, config, "lms", "create_databases.sh", run_func)
+    click.echo(fmt.info("Running lms migrations..."))
+    run_template(root, config, "lms", "migrate_lms.sh", run_func)
+    click.echo(fmt.info("Running cms migrations..."))
+    run_template(root, config, "cms", "migrate_cms.sh", run_func)
+    click.echo(fmt.info("Running forum migrations..."))
+    run_template(root, config, "forum", "migrate_forum.sh", run_func)
+    if config["ACTIVATE_NOTES"]:
+        click.echo(fmt.info("Running notes migrations..."))
+        run_template(root, config, "notes", "migrate_django.sh", run_func)
+    if config["ACTIVATE_XQUEUE"]:
+        click.echo(fmt.info("Running xqueue migrations..."))
+        run_template(root, config, "xqueue", "migrate_django.sh", run_func)
+    click.echo(fmt.info("Creating oauth2 users..."))
+    run_template(root, config, "lms", "oauth2.sh", run_func)
+    click.echo(fmt.info("Databases ready."))
 
-migrate_lms = "dockerize -wait tcp://{{ MYSQL_HOST }}:{{ MYSQL_PORT }} -timeout 20s && ./manage.py lms migrate"
-migrate_cms = "dockerize -wait tcp://{{ MYSQL_HOST }}:{{ MYSQL_PORT }} -timeout 20s && ./manage.py cms migrate"
-migrate_forum = "bundle exec rake search:initialize && bundle exec rake search:rebuild_index"
-migrate_notes = "./manage.py migrate"
-migrate_xqueue = "./manage.py migrate"
+def create_user(root, run_func, superuser, staff, name, email):
+    config = {
+        "OPTS": "",
+        "USERNAME": name,
+        "EMAIL": email,
+    }
+    if superuser:
+        config["OPTS"] += " --superuser"
+    if staff:
+        config["OPTS"] += " --staff"
+    run_template(root, config, "lms", "create_user.sh", run_func)
 
-oauth2 = """
-./manage.py lms create_oauth2_client \
-    "http://androidapp.com" "http://androidapp.com/redirect" public \
-    --client_id android --client_secret {{ ANDROID_OAUTH2_SECRET }} \
-    --trusted
+def import_demo_course(root, run_func):
+    run_template(root, {}, "cms", "import_demo_course.sh", run_func)
 
-{% if ACTIVATE_NOTES %}
-./manage.py lms manage_user notes notes@{{ LMS_HOST }} --staff --superuser
-./manage.py lms create_oauth2_client \
-    "http://notes.openedx:8000" "http://notes.openedx:8000/complete/edx-oidc/" confidential \
-    --client_name edx-notes --client_id notes --client_secret {{ NOTES_OAUTH2_SECRET }} \
-    --trusted --logout_uri "http://notes.openedx:8000/logout/" --username notes
-{% endif %}"""
+def index_courses(root, run_func):
+    run_template(root, {}, "cms", "index_courses.sh", run_func)
 
-https_certificates_create = """certbot certonly --standalone -n --agree-tos -m admin@{{ LMS_HOST }} -d {{ LMS_HOST }} -d {{ CMS_HOST }} -d preview.{{ LMS_HOST }}
-{% if ACTIVATE_NOTES %}certbot certonly --standalone -n --agree-tos -m admin@{{ LMS_HOST }} -d notes.{{ LMS_HOST }}{% endif %}"""
+def run_template(root, config, service, template, run_func):
+    command = render_template(config, template)
+    if command:
+        run_func(root, service, command)
 
-create_user = """./manage.py lms --settings=tutor.production manage_user {{ OPTS }} {{ USERNAME }} {{ EMAIL }}
-./manage.py lms --settings=tutor.production changepassword {{ USERNAME }}"""
-import_demo_course = """git clone https://github.com/edx/edx-demo-course --branch open-release/hawthorn.2 --depth 1 ../edx-demo-course
-python ./manage.py cms --settings=tutor.production import ../data ../edx-demo-course"""
-index_courses = "./manage.py cms --settings=tutor.production reindex_course --all --setup"
+def render_template(config, template):
+    path = env.template_path("scripts", template)
+    return env.render_file(config, path).strip()

tutor/templates/scripts/create_databases.sh

@@ -0,0 +1,13 @@
+dockerize -wait tcp://{{ MYSQL_HOST }}:{{ MYSQL_PORT }} -timeout 20s
+mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" -e 'CREATE DATABASE IF NOT EXISTS {{ OPENEDX_MYSQL_DATABASE }};'
+mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" -e 'GRANT ALL ON {{ OPENEDX_MYSQL_DATABASE }}.* TO "{{ OPENEDX_MYSQL_USERNAME }}"@"%" IDENTIFIED BY "{{ OPENEDX_MYSQL_PASSWORD }}";'
+
+{% if ACTIVATE_NOTES %}
+mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" -e 'CREATE DATABASE IF NOT EXISTS {{ NOTES_MYSQL_DATABASE }};'
+mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" -e 'GRANT ALL ON {{ NOTES_MYSQL_DATABASE }}.* TO "{{ NOTES_MYSQL_USERNAME }}"@"%" IDENTIFIED BY "{{ NOTES_MYSQL_PASSWORD }}";'
+{% endif %}
+
+{% if ACTIVATE_XQUEUE %}
+mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" -e 'CREATE DATABASE IF NOT EXISTS {{ XQUEUE_MYSQL_DATABASE }};'
+mysql -u root --password="{{ MYSQL_ROOT_PASSWORD }}" --host "{{ MYSQL_HOST }}" -e 'GRANT ALL ON {{ XQUEUE_MYSQL_DATABASE }}.* TO "{{ XQUEUE_MYSQL_USERNAME }}"@"%" IDENTIFIED BY "{{ XQUEUE_MYSQL_PASSWORD }}";'
+{% endif %}

tutor/templates/scripts/create_user.sh

@@ -0,0 +1,2 @@
+./manage.py lms --settings=tutor.production manage_user {{ OPTS }} {{ USERNAME }} {{ EMAIL }}
+./manage.py lms --settings=tutor.production changepassword {{ USERNAME }}

tutor/templates/scripts/https_create.sh

@@ -0,0 +1,2 @@
+certbot certonly --standalone -n --agree-tos -m admin@{{ LMS_HOST }} -d {{ LMS_HOST }} -d {{ CMS_HOST }} -d preview.{{ LMS_HOST }}
+{% if ACTIVATE_NOTES %}certbot certonly --standalone -n --agree-tos -m admin@{{ LMS_HOST }} -d notes.{{ LMS_HOST }}{% endif %}

tutor/templates/scripts/import_demo_course.sh

@@ -0,0 +1,2 @@
+git clone https://github.com/edx/edx-demo-course --branch open-release/hawthorn.2 --depth 1 ../edx-demo-course
+python ./manage.py cms --settings=tutor.production import ../data ../edx-demo-course

tutor/templates/scripts/index_courses.sh

@@ -0,0 +1 @@
+./manage.py cms --settings=tutor.production reindex_course --all --setup

tutor/templates/scripts/migrate_cms.sh

@@ -0,0 +1 @@
+dockerize -wait tcp://{{ MYSQL_HOST }}:{{ MYSQL_PORT }} -timeout 20s && ./manage.py cms migrate

tutor/templates/scripts/migrate_django.sh

@@ -0,0 +1 @@
+./manage.py migrate

tutor/templates/scripts/migrate_forum.sh

@@ -0,0 +1 @@
+bundle exec rake search:initialize && bundle exec rake search:rebuild_index

tutor/templates/scripts/migrate_lms.sh

@@ -0,0 +1 @@
+dockerize -wait tcp://{{ MYSQL_HOST }}:{{ MYSQL_PORT }} -timeout 20s && ./manage.py lms migrate

tutor/templates/scripts/oauth2.sh

@@ -0,0 +1,12 @@
+./manage.py lms create_oauth2_client \
+    "http://androidapp.com" "http://androidapp.com/redirect" public \
+    --client_id android --client_secret {{ ANDROID_OAUTH2_SECRET }} \
+    --trusted
+
+{% if ACTIVATE_NOTES %}
+./manage.py lms manage_user notes notes@{{ LMS_HOST }} --staff --superuser
+./manage.py lms create_oauth2_client \
+    "http://notes.openedx:8000" "http://notes.openedx:8000/complete/edx-oidc/" confidential \
+    --client_name edx-notes --client_id notes --client_secret {{ NOTES_OAUTH2_SECRET }} \
+    --trusted --logout_uri "http://notes.openedx:8000/logout/" --username notes
+{% endif %}