christianlight
/
tutor
mirror of https://github.com/ChristianLight/tutor.git
7
0
Fork 0
Code Issues Releases Activity

refactor: add type annotations 

Annotations were generated with pyannotate:
https://github.com/dropbox/pyannotate

We are running in strict mode, which is awesome!

This affects a large part of the code base, which might be an issue for
people running a fork of Tutor. Nonetheless, the behavior should not be
affected. If anything, this process has helped find and resolve a few
type-related bugs. Thus, this is not considered as a breaking change.
This commit is contained in:
Régis Behmo 2021-02-25 09:09:14 +01:00
parent 1d4ab79863
commit 0a670d7ead
35 changed files with 919 additions and 750 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -4,6 +4,7 @@ Note: Breaking changes between versions are indicated by "💥".
## Unreleased
- [Improvement] Annotate types all over the Tutor code base.
- [Bugfix] Fix parsing of YAML CLI arguments that include equal "=" signs.
- [Bugfix] Fix minor edge case in `long_to_base64` utility function.
- [Improvement] Add openedx patches to add settings during build process.

5
Makefile
View File

@ -24,7 +24,7 @@ build-pythonpackage: ## Build a python package ready to upload to pypi
push-pythonpackage: ## Push python packages to pypi
twine upload --skip-existing dist/tutor-*.tar.gz
test: test-lint test-unit test-format test-pythonpackage ## Run all tests by decreasing order or priority
test: test-lint test-unit test-types test-format test-pythonpackage ## Run all tests by decreasing order or priority
test-format: ## Run code formatting tests
black --check --diff $(BLACK_OPTS)
@ -35,6 +35,9 @@ test-lint: ## Run code linting tests
test-unit: ## Run unit tests
python -m unittest discover tests
test-types: ## Check type definitions
mypy --exclude=templates --ignore-missing-imports --strict tutor/ tests/
test-pythonpackage: build-pythonpackage ## Test that package can be uploaded to pypi
twine check dist/tutor-openedx-$(shell make version).tar.gz

1
requirements/base.in
View File

@ -1,6 +1,7 @@
appdirs
click>=7.0
click_repl
mypy
pycryptodome
jinja2>=2.9
kubernetes

8
requirements/base.txt
View File

@ -30,6 +30,10 @@ kubernetes==12.0.1
# via -r requirements/base.in
markupsafe==1.1.1
# via jinja2
mypy-extensions==0.4.3
# via mypy
mypy==0.812
# via -r requirements/base.in
oauthlib==3.1.0
# via requests-oauthlib
prompt-toolkit==3.0.14
@ -63,6 +67,10 @@ six==1.15.0
# kubernetes
# python-dateutil
# websocket-client
typed-ast==1.4.2
# via mypy
typing-extensions==3.7.4.3
# via mypy
urllib3==1.25.11
# via
# -r requirements/base.in

26
requirements/dev.txt
View File

@ -53,6 +53,11 @@ idna==2.10
# via
# -r requirements/base.txt
# requests
importlib-metadata==3.7.0
# via
# keyring
# pyinstaller
# twine
isort==5.7.0
# via pylint
jeepney==0.6.0
@ -74,7 +79,12 @@ markupsafe==1.1.1
mccabe==0.6.1
# via pylint
mypy-extensions==0.4.3
# via black
# via
# -r requirements/base.txt
# black
# mypy
mypy==0.812
# via -r requirements/base.txt
oauthlib==3.1.0
# via
# -r requirements/base.txt
@ -167,9 +177,17 @@ tqdm==4.56.1
twine==3.3.0
# via -r requirements/dev.in
typed-ast==1.4.2
# via black
# via
# -r requirements/base.txt
# astroid
# black
# mypy
typing-extensions==3.7.4.3
# via black
# via
# -r requirements/base.txt
# black
# importlib-metadata
# mypy
urllib3==1.25.11
# via
# -r requirements/base.txt
@ -187,6 +205,8 @@ websocket-client==0.57.0
# kubernetes
wrapt==1.12.1
# via astroid
zipp==3.4.0
# via importlib-metadata
# The following packages are considered to be unsafe in a requirements file:
# pip

14
requirements/docs.txt
View File

@ -51,6 +51,12 @@ markupsafe==1.1.1
# via
# -r requirements/base.txt
# jinja2
mypy-extensions==0.4.3
# via
# -r requirements/base.txt
# mypy
mypy==0.812
# via -r requirements/base.txt
oauthlib==3.1.0
# via
# -r requirements/base.txt
@ -128,6 +134,14 @@ sphinxcontrib-qthelp==1.0.3
# via sphinx
sphinxcontrib-serializinghtml==1.1.4
# via sphinx
typed-ast==1.4.2
# via
# -r requirements/base.txt
# mypy
typing-extensions==3.7.4.3
# via
# -r requirements/base.txt
# mypy
urllib3==1.25.11
# via
# -r requirements/base.txt

6
tests/test_bindmounts.py
View File

@ -5,17 +5,17 @@ from tutor.exceptions import TutorError
class BindMountsTests(unittest.TestCase):
def test_get_name(self):
def test_get_name(self) -> None:
self.assertEqual("venv", bindmounts.get_name("/openedx/venv"))
self.assertEqual("venv", bindmounts.get_name("/openedx/venv/"))
def test_get_name_root_folder(self):
def test_get_name_root_folder(self) -> None:
with self.assertRaises(TutorError):
bindmounts.get_name("/")
with self.assertRaises(TutorError):
bindmounts.get_name("")
def test_parse_volumes(self):
def test_parse_volumes(self) -> None:
volume_args, non_volume_args = bindmounts.parse_volumes(
[
"run",

38
tests/test_config.py
View File

@ -1,39 +1,33 @@
from typing import Any, Dict
import unittest
import unittest.mock
from unittest.mock import Mock, patch
import tempfile
from tutor import config as tutor_config
from tutor import env
from tutor import interactive
class ConfigTests(unittest.TestCase):
def setUp(self):
# This is necessary to avoid cached mocks
env.Renderer.reset()
def test_version(self):
def test_version(self) -> None:
defaults = tutor_config.load_defaults()
self.assertNotIn("TUTOR_VERSION", defaults)
def test_merge(self):
def test_merge(self) -> None:
config1 = {"x": "y"}
config2 = {"x": "z"}
tutor_config.merge(config1, config2)
self.assertEqual({"x": "y"}, config1)
def test_merge_render(self):
config = {}
def test_merge_render(self) -> None:
config: Dict[str, Any] = {}
defaults = tutor_config.load_defaults()
with unittest.mock.patch.object(
tutor_config.utils, "random_string", return_value="abcd"
):
with patch.object(tutor_config.utils, "random_string", return_value="abcd"):
tutor_config.merge(config, defaults)
self.assertEqual("abcd", config["MYSQL_ROOT_PASSWORD"])
@unittest.mock.patch.object(tutor_config.fmt, "echo")
def test_update_twice(self, _):
@patch.object(tutor_config.fmt, "echo")
def test_update_twice(self, _: Mock) -> None:
with tempfile.TemporaryDirectory() as root:
tutor_config.update(root)
config1 = tutor_config.load_user(root)
@ -43,27 +37,27 @@ class ConfigTests(unittest.TestCase):
self.assertEqual(config1, config2)
@unittest.mock.patch.object(tutor_config.fmt, "echo")
def test_removed_entry_is_added_on_save(self, _):
@patch.object(tutor_config.fmt, "echo")
def test_removed_entry_is_added_on_save(self, _: Mock) -> None:
with tempfile.TemporaryDirectory() as root:
with unittest.mock.patch.object(
with patch.object(
tutor_config.utils, "random_string"
) as mock_random_string:
mock_random_string.return_value = "abcd"
config1, _ = tutor_config.load_all(root)
config1, _defaults1 = tutor_config.load_all(root)
password1 = config1["MYSQL_ROOT_PASSWORD"]
config1.pop("MYSQL_ROOT_PASSWORD")
tutor_config.save_config_file(root, config1)
mock_random_string.return_value = "efgh"
config2, _ = tutor_config.load_all(root)
config2, _defaults2 = tutor_config.load_all(root)
password2 = config2["MYSQL_ROOT_PASSWORD"]
self.assertEqual("abcd", password1)
self.assertEqual("efgh", password2)
def test_interactive_load_all(self):
def test_interactive_load_all(self) -> None:
with tempfile.TemporaryDirectory() as rootdir:
config, defaults = interactive.load_all(rootdir, interactive=False)
@ -73,7 +67,7 @@ class ConfigTests(unittest.TestCase):
self.assertEqual("www.myopenedx.com", defaults["LMS_HOST"])
self.assertEqual("studio.{{ LMS_HOST }}", defaults["CMS_HOST"])
def test_is_service_activated(self):
def test_is_service_activated(self) -> None:
config = {"RUN_SERVICE1": True, "RUN_SERVICE2": False}
self.assertTrue(tutor_config.is_service_activated(config, "service1"))

64
tests/test_env.py
View File

@ -1,7 +1,8 @@
import os
import tempfile
from typing import Any, Dict
import unittest
import unittest.mock
from unittest.mock import patch, Mock
from tutor import config as tutor_config
from tutor import env
@ -10,41 +11,38 @@ from tutor import exceptions
class EnvTests(unittest.TestCase):
def setUp(self):
env.Renderer.reset()
def test_walk_templates(self):
def test_walk_templates(self) -> None:
renderer = env.Renderer({}, [env.TEMPLATES_ROOT])
templates = list(renderer.walk_templates("local"))
self.assertIn("local/docker-compose.yml", templates)
def test_walk_templates_partials_are_ignored(self):
def test_walk_templates_partials_are_ignored(self) -> None:
template_name = "apps/openedx/settings/partials/common_all.py"
renderer = env.Renderer({}, [env.TEMPLATES_ROOT], ignore_folders=["partials"])
templates = list(renderer.walk_templates("apps"))
self.assertIn(template_name, renderer.environment.loader.list_templates())
self.assertIn(template_name, renderer.environment.loader.list_templates()) # type: ignore
self.assertNotIn(template_name, templates)
def test_is_binary_file(self):
def test_is_binary_file(self) -> None:
self.assertTrue(env.is_binary_file("/home/somefile.ico"))
def test_find_os_path(self):
def test_find_os_path(self) -> None:
renderer = env.Renderer({}, [env.TEMPLATES_ROOT])
path = renderer.find_os_path("local/docker-compose.yml")
self.assertTrue(os.path.exists(path))
def test_pathjoin(self):
def test_pathjoin(self) -> None:
self.assertEqual(
"/tmp/env/target/dummy", env.pathjoin("/tmp", "target", "dummy")
)
self.assertEqual("/tmp/env/dummy", env.pathjoin("/tmp", "dummy"))
def test_render_str(self):
def test_render_str(self) -> None:
self.assertEqual(
"hello world", env.render_str({"name": "world"}, "hello {{ name }}")
)
def test_common_domain(self):
def test_common_domain(self) -> None:
self.assertEqual(
"mydomain.com",
env.render_str(
@ -53,64 +51,62 @@ class EnvTests(unittest.TestCase):
),
)
def test_render_str_missing_configuration(self):
def test_render_str_missing_configuration(self) -> None:
self.assertRaises(exceptions.TutorError, env.render_str, {}, "hello {{ name }}")
def test_render_file(self):
config = {}
def test_render_file(self) -> None:
config: Dict[str, Any] = {}
tutor_config.merge(config, tutor_config.load_defaults())
config["MYSQL_ROOT_PASSWORD"] = "testpassword"
rendered = env.render_file(config, "hooks", "mysql", "init")
self.assertIn("testpassword", rendered)
@unittest.mock.patch.object(tutor_config.fmt, "echo")
def test_render_file_missing_configuration(self, _):
@patch.object(tutor_config.fmt, "echo")
def test_render_file_missing_configuration(self, _: Mock) -> None:
self.assertRaises(
exceptions.TutorError, env.render_file, {}, "local", "docker-compose.yml"
)
def test_save_full(self):
def test_save_full(self) -> None:
defaults = tutor_config.load_defaults()
with tempfile.TemporaryDirectory() as root:
config = tutor_config.load_current(root, defaults)
tutor_config.merge(config, defaults)
with unittest.mock.patch.object(fmt, "STDOUT"):
with patch.object(fmt, "STDOUT"):
env.save(root, config)
self.assertTrue(
os.path.exists(os.path.join(root, "env", "local", "docker-compose.yml"))
)
def test_save_full_with_https(self):
def test_save_full_with_https(self) -> None:
defaults = tutor_config.load_defaults()
with tempfile.TemporaryDirectory() as root:
config = tutor_config.load_current(root, defaults)
tutor_config.merge(config, defaults)
config["ENABLE_HTTPS"] = True
with unittest.mock.patch.object(fmt, "STDOUT"):
with patch.object(fmt, "STDOUT"):
env.save(root, config)
with open(os.path.join(root, "env", "apps", "caddy", "Caddyfile")) as f:
self.assertIn("www.myopenedx.com {", f.read())
def test_patch(self):
def test_patch(self) -> None:
patches = {"plugin1": "abcd", "plugin2": "efgh"}
with unittest.mock.patch.object(
with patch.object(
env.plugins, "iter_patches", return_value=patches.items()
) as mock_iter_patches:
rendered = env.render_str({}, '{{ patch("location") }}')
mock_iter_patches.assert_called_once_with({}, "location")
self.assertEqual("abcd\nefgh", rendered)
def test_patch_separator_suffix(self):
def test_patch_separator_suffix(self) -> None:
patches = {"plugin1": "abcd", "plugin2": "efgh"}
with unittest.mock.patch.object(
env.plugins, "iter_patches", return_value=patches.items()
):
with patch.object(env.plugins, "iter_patches", return_value=patches.items()):
rendered = env.render_str(
{}, '{{ patch("location", separator=",\n", suffix=",") }}'
)
self.assertEqual("abcd,\nefgh,", rendered)
def test_plugin_templates(self):
def test_plugin_templates(self) -> None:
with tempfile.TemporaryDirectory() as plugin_templates:
# Create plugin
plugin1 = env.plugins.DictPlugin(
@ -132,7 +128,7 @@ class EnvTests(unittest.TestCase):
config = {"ID": "abcd"}
# Render templates
with unittest.mock.patch.object(
with patch.object(
env.plugins,
"iter_enabled",
return_value=[plugin1],
@ -153,7 +149,7 @@ class EnvTests(unittest.TestCase):
with open(dst_rendered) as f:
self.assertEqual("Hello my ID is abcd", f.read())
def test_renderer_is_reset_on_config_change(self):
def test_renderer_is_reset_on_config_change(self) -> None:
with tempfile.TemporaryDirectory() as plugin_templates:
plugin1 = env.plugins.DictPlugin(
{"name": "plugin1", "version": "0", "templates": plugin_templates}
@ -166,10 +162,10 @@ class EnvTests(unittest.TestCase):
f.write("some content")
# Load env once
config = {"PLUGINS": []}
config: Dict[str, Any] = {"PLUGINS": []}
env1 = env.Renderer.instance(config).environment
with unittest.mock.patch.object(
with patch.object(
env.plugins,
"iter_enabled",
return_value=[plugin1],
@ -178,5 +174,5 @@ class EnvTests(unittest.TestCase):
config["PLUGINS"].append("myplugin")
env2 = env.Renderer.instance(config).environment
self.assertNotIn("plugin1/myplugin.txt", env1.loader.list_templates())
self.assertIn("plugin1/myplugin.txt", env2.loader.list_templates())
self.assertNotIn("plugin1/myplugin.txt", env1.loader.list_templates()) # type: ignore
self.assertIn("plugin1/myplugin.txt", env2.loader.list_templates()) # type: ignore

2
tests/test_images.py
View File

@ -3,7 +3,7 @@ from tutor import images
class ImagesTests(unittest.TestCase):
def test_get_tag(self):
def test_get_tag(self) -> None:
config = {
"DOCKER_IMAGE_OPENEDX": "registry/openedx",
"DOCKER_IMAGE_OPENEDX_DEV": "registry/openedxdev",

65
tests/test_plugins.py
View File

@ -1,5 +1,6 @@
from typing import Any, Dict
import unittest
from unittest.mock import patch
from unittest.mock import Mock, patch
from tutor import config as tutor_config
from tutor import exceptions
@ -8,22 +9,22 @@ from tutor import plugins
class PluginsTests(unittest.TestCase):
def setUp(self):
def setUp(self) -> None:
plugins.Plugins.clear()
@patch.object(plugins.DictPlugin, "iter_installed", return_value=[])
def test_iter_installed(self, _dict_plugin_iter_installed):
with patch.object(plugins.pkg_resources, "iter_entry_points", return_value=[]):
def test_iter_installed(self, _dict_plugin_iter_installed: Mock) -> None:
with patch.object(plugins.pkg_resources, "iter_entry_points", return_value=[]): # type: ignore
self.assertEqual([], list(plugins.iter_installed()))
def test_is_installed(self):
def test_is_installed(self) -> None:
self.assertFalse(plugins.is_installed("dummy"))
@patch.object(plugins.DictPlugin, "iter_installed", return_value=[])
def test_official_plugins(self, _dict_plugin_iter_installed):
with patch.object(plugins.importlib, "import_module", return_value=42):
def test_official_plugins(self, _dict_plugin_iter_installed: Mock) -> None:
with patch.object(plugins.importlib, "import_module", return_value=42): # type: ignore
plugin1 = plugins.OfficialPlugin.load("plugin1")
with patch.object(plugins.importlib, "import_module", return_value=43):
with patch.object(plugins.importlib, "import_module", return_value=43): # type: ignore
plugin2 = plugins.OfficialPlugin.load("plugin2")
with patch.object(
plugins.EntrypointPlugin,
@ -35,32 +36,32 @@ class PluginsTests(unittest.TestCase):
list(plugins.iter_installed()),
)
def test_enable(self):
config = {plugins.CONFIG_KEY: []}
def test_enable(self) -> None:
config: Dict[str, Any] = {plugins.CONFIG_KEY: []}
with patch.object(plugins, "is_installed", return_value=True):
plugins.enable(config, "plugin2")
plugins.enable(config, "plugin1")
self.assertEqual(["plugin1", "plugin2"], config[plugins.CONFIG_KEY])
def test_enable_twice(self):
config = {plugins.CONFIG_KEY: []}
def test_enable_twice(self) -> None:
config: Dict[str, Any] = {plugins.CONFIG_KEY: []}
with patch.object(plugins, "is_installed", return_value=True):
plugins.enable(config, "plugin1")
plugins.enable(config, "plugin1")
self.assertEqual(["plugin1"], config[plugins.CONFIG_KEY])
def test_enable_not_installed_plugin(self):
config = {"PLUGINS": []}
def test_enable_not_installed_plugin(self) -> None:
config: Dict[str, Any] = {"PLUGINS": []}
with patch.object(plugins, "is_installed", return_value=False):
self.assertRaises(exceptions.TutorError, plugins.enable, config, "plugin1")
def test_disable(self):
config = {"PLUGINS": ["plugin1", "plugin2"]}
def test_disable(self) -> None:
config: Dict[str, Any] = {"PLUGINS": ["plugin1", "plugin2"]}
with patch.object(fmt, "STDOUT"):
plugins.disable(config, "plugin1")
self.assertEqual(["plugin2"], config["PLUGINS"])
def test_disable_removes_set_config(self):
def test_disable_removes_set_config(self) -> None:
with patch.object(
plugins.Plugins,
"iter_enabled",
@ -80,7 +81,7 @@ class PluginsTests(unittest.TestCase):
self.assertEqual([], config["PLUGINS"])
self.assertNotIn("KEY", config)
def test_patches(self):
def test_patches(self) -> None:
class plugin1:
patches = {"patch1": "Hello {{ ID }}"}
@ -92,7 +93,7 @@ class PluginsTests(unittest.TestCase):
patches = list(plugins.iter_patches({}, "patch1"))
self.assertEqual([("plugin1", "Hello {{ ID }}")], patches)
def test_plugin_without_patches(self):
def test_plugin_without_patches(self) -> None:
with patch.object(
plugins.Plugins,
"iter_enabled",
@ -101,9 +102,9 @@ class PluginsTests(unittest.TestCase):
patches = list(plugins.iter_patches({}, "patch1"))
self.assertEqual([], patches)
def test_configure(self):
def test_configure(self) -> None:
config = {"ID": "id"}
defaults = {}
defaults: Dict[str, Any] = {}
class plugin1:
config = {
@ -130,7 +131,7 @@ class PluginsTests(unittest.TestCase):
)
self.assertEqual({"PLUGIN1_PARAM4": "value4"}, defaults)
def test_configure_set_does_not_override(self):
def test_configure_set_does_not_override(self) -> None:
config = {"ID": "oldid"}
class plugin1:
@ -145,8 +146,8 @@ class PluginsTests(unittest.TestCase):
self.assertEqual({"ID": "oldid"}, config)
def test_configure_set_random_string(self):
config = {}
def test_configure_set_random_string(self) -> None:
config: Dict[str, Any] = {}
class plugin1:
config = {"set": {"PARAM1": "{{ 128|random_string }}"}}
@ -159,8 +160,8 @@ class PluginsTests(unittest.TestCase):
tutor_config.load_plugins(config, {})
self.assertEqual(128, len(config["PARAM1"]))
def test_configure_default_value_with_previous_definition(self):
config = {}
def test_configure_default_value_with_previous_definition(self) -> None:
config: Dict[str, Any] = {}
defaults = {"PARAM1": "value"}
class plugin1:
@ -174,8 +175,8 @@ class PluginsTests(unittest.TestCase):
tutor_config.load_plugins(config, defaults)
self.assertEqual("{{ PARAM1 }}", defaults["PLUGIN1_PARAM2"])
def test_configure_add_twice(self):
config = {}
def test_configure_add_twice(self) -> None:
config: Dict[str, Any] = {}
class plugin1:
config = {"add": {"PARAM1": "{{ 10|random_string }}"}}
@ -199,7 +200,7 @@ class PluginsTests(unittest.TestCase):
self.assertEqual(10, len(value2))
self.assertEqual(value1, value2)
def test_hooks(self):
def test_hooks(self) -> None:
class plugin1:
hooks = {"init": ["myclient"]}
@ -212,8 +213,8 @@ class PluginsTests(unittest.TestCase):
[("plugin1", ["myclient"])], list(plugins.iter_hooks({}, "init"))
)
def test_plugins_are_updated_on_config_change(self):
config = {"PLUGINS": []}
def test_plugins_are_updated_on_config_change(self) -> None:
config: Dict[str, Any] = {"PLUGINS": []}
plugins1 = plugins.Plugins(config)
self.assertEqual(0, len(list(plugins1.iter_enabled())))
config["PLUGINS"].append("plugin1")
@ -225,7 +226,7 @@ class PluginsTests(unittest.TestCase):
plugins2 = plugins.Plugins(config)
self.assertEqual(1, len(list(plugins2.iter_enabled())))
def test_dict_plugin(self):
def test_dict_plugin(self) -> None:
plugin = plugins.DictPlugin(
{"name": "myplugin", "config": {"set": {"KEY": "value"}}, "version": "0.1"}
)

18
tests/test_serialize.py
View File

@ -6,32 +6,32 @@ from tutor import serialize
class SerializeTests(unittest.TestCase):
def test_parse_str(self):
def test_parse_str(self) -> None:
self.assertEqual("abcd", serialize.parse("abcd"))
def test_parse_int(self):
def test_parse_int(self) -> None:
self.assertEqual(1, serialize.parse("1"))
def test_parse_bool(self):
def test_parse_bool(self) -> None:
self.assertEqual(True, serialize.parse("true"))
self.assertEqual(False, serialize.parse("false"))
def test_parse_null(self):
def test_parse_null(self) -> None:
self.assertIsNone(serialize.parse("null"))
def test_parse_invalid_format(self):
def test_parse_invalid_format(self) -> None:
self.assertEqual('["abcd"', serialize.parse('["abcd"'))
def test_parse_list(self):
def test_parse_list(self) -> None:
self.assertEqual(["abcd"], serialize.parse('["abcd"]'))
def test_parse_weird_chars(self):
def test_parse_weird_chars(self) -> None:
self.assertEqual("*@google.com", serialize.parse("*@google.com"))
def test_parse_empty_string(self):
def test_parse_empty_string(self) -> None:
self.assertEqual("", serialize.parse("''"))
def test_yaml_param_type(self):
def test_yaml_param_type(self) -> None:
param = serialize.YamlParamType()
self.assertEqual(("name", True), param.convert("name=true", "param", {}))
self.assertEqual(("name", "abcd"), param.convert("name=abcd", "param", {}))

10
tests/test_utils.py
View File

@ -5,7 +5,7 @@ from tutor import utils
class UtilsTests(unittest.TestCase):
def test_common_domain(self):
def test_common_domain(self) -> None:
self.assertEqual(
"domain.com", utils.common_domain("sub1.domain.com", "sub2.domain.com")
)
@ -18,13 +18,13 @@ class UtilsTests(unittest.TestCase):
"domain.com", utils.common_domain("sub.domain.com", "ub.domain.com")
)
def test_reverse_host(self):
def test_reverse_host(self) -> None:
self.assertEqual("com.google.www", utils.reverse_host("www.google.com"))
def test_list_if(self):
def test_list_if(self) -> None:
self.assertEqual('["cms"]', utils.list_if([("lms", False), ("cms", True)]))
def test_encrypt_decrypt(self):
def test_encrypt_decrypt(self) -> None:
password = "passw0rd"
encrypted1 = utils.encrypt(password)
encrypted2 = utils.encrypt(password)
@ -32,7 +32,7 @@ class UtilsTests(unittest.TestCase):
self.assertTrue(utils.verify_encrypted(encrypted1, password))
self.assertTrue(utils.verify_encrypted(encrypted2, password))
def test_long_to_base64(self):
def test_long_to_base64(self) -> None:
self.assertEqual(
b"\x00", base64.urlsafe_b64decode(utils.long_to_base64(0) + "==")
)

22
tutor/bindmounts.py
View File

@ -1,12 +1,20 @@
import os
from typing import Any, Callable, Dict, List, Tuple
import click
from mypy_extensions import VarArg
from .exceptions import TutorError
from .utils import get_user_id
def create(root, config, docker_compose_func, service, path):
def create(
root: str,
config: Dict[str, Any],
docker_compose_func: Callable[[str, Dict[str, Any], VarArg(str)], int],
service: str,
path: str,
) -> str:
volumes_root_path = get_root_path(root)
volume_name = get_name(path)
container_volumes_root_path = "/tmp/volumes"
@ -41,12 +49,12 @@ chown -R {user_id} {volumes_path}/{volume_name}""".format(
return os.path.join(volumes_root_path, volume_name)
def get_path(root, container_bind_path):
def get_path(root: str, container_bind_path: str) -> str:
bind_basename = get_name(container_bind_path)
return os.path.join(get_root_path(root), bind_basename)
def get_name(container_bind_path):
def get_name(container_bind_path: str) -> str:
# We rstrip slashes, otherwise os.path.basename returns an empty string
# We don't use basename here as it will not work on Windows
name = container_bind_path.rstrip("/").split("/")[-1]
@ -55,11 +63,11 @@ def get_name(container_bind_path):
return name
def get_root_path(root):
def get_root_path(root: str) -> str:
return os.path.join(root, "volumes")
def parse_volumes(docker_compose_args):
def parse_volumes(docker_compose_args: List[str]) -> Tuple[List[str], List[str]]:
"""
Parse `-v/--volume` options from an arbitrary list of arguments.
"""
@ -67,7 +75,9 @@ def parse_volumes(docker_compose_args):
@click.command(context_settings={"ignore_unknown_options": True})
@click.option("-v", "--volume", "volumes", multiple=True)
@click.argument("args", nargs=-1, required=True)
def custom_docker_compose(volumes, args): # pylint: disable=unused-argument
def custom_docker_compose(
volumes: List[str], args: List[str]
) -> None: # pylint: disable=unused-argument
pass
if isinstance(docker_compose_args, tuple):

11
tutor/commands/android.py
View File

@ -1,3 +1,5 @@
from typing import Dict
import click
from .compose import ComposeJobRunner
@ -5,17 +7,18 @@ from .local import docker_compose as local_docker_compose
from .. import config as tutor_config
from .. import env as tutor_env
from .. import fmt
from .context import Context
@click.group(help="Build an Android app for your Open edX platform [BETA FEATURE]")
def android():
def android() -> None:
pass
@click.command(help="Build the application")
@click.argument("mode", type=click.Choice(["debug", "release"]))
@click.pass_obj
def build(context, mode):
def build(context: Context, mode: str) -> None:
config = tutor_config.load(context.root)
docker_run(context.root, build_command(config, mode))
fmt.echo_info(
@ -25,7 +28,7 @@ def build(context, mode):
)
def build_command(config, target):
def build_command(config: Dict[str, str], target: str) -> str:
gradle_target = {
"debug": "assembleProdDebuggable",
"release": "assembleProdRelease",
@ -41,7 +44,7 @@ cp OpenEdXMobile/build/outputs/apk/prod/{apk_folder}/*.apk /openedx/data/"""
return command
def docker_run(root, command):
def docker_run(root: str, command: str) -> None:
config = tutor_config.load(root)
runner = ComposeJobRunner(root, config, local_docker_compose)
runner.run_job("android", command)

10
tutor/commands/cli.py
View File

@ -21,7 +21,7 @@ from .. import fmt
from .. import utils
def main():
def main() -> None:
try:
click_repl.register_repl(cli, name="ui")
cli.add_command(images_command)
@ -55,7 +55,7 @@ def main():
help="Root project directory (environment variable: TUTOR_ROOT)",
)
@click.pass_context
def cli(context, root):
def cli(context: click.Context, root: str) -> None:
if utils.is_root():
fmt.echo_alert(
"You are running Tutor as root. This is strongly not recommended. If you are doing this in order to access"
@ -66,9 +66,9 @@ def cli(context, root):
@click.command(help="Print this help", name="help")
def print_help():
with click.Context(cli) as context:
click.echo(cli.get_help(context))
def print_help() -> None:
context = click.Context(cli)
click.echo(cli.get_help(context))
if __name__ == "__main__":

117
tutor/commands/compose.py
View File

@ -1,6 +1,8 @@
import os
from typing import Any, Callable, Dict, List
import click
from mypy_extensions import VarArg
from .. import bindmounts
from .. import config as tutor_config
@ -10,14 +12,20 @@ from .. import fmt
from .. import jobs
from .. import serialize
from .. import utils
from .context import Context
class ComposeJobRunner(jobs.BaseJobRunner):
def __init__(self, root, config, docker_compose_func):
def __init__(
self,
root: str,
config: Dict[str, Any],
docker_compose_func: Callable[[str, Dict[str, Any], VarArg(str)], int],
):
super().__init__(root, config)
self.docker_compose_func = docker_compose_func
def run_job(self, service, command):
def run_job(self, service: str, command: str) -> int:
"""
Run the "{{ service }}-job" service from local/docker-compose.jobs.yml with the
specified command. For backward-compatibility reasons, if the corresponding
@ -28,7 +36,7 @@ class ComposeJobRunner(jobs.BaseJobRunner):
job_service_name = "{}-job".format(service)
opts = [] if utils.is_a_tty() else ["-T"]
if job_service_name in serialize.load(open(jobs_path).read())["services"]:
self.docker_compose_func(
return self.docker_compose_func(
self.root,
self.config,
"-f",
@ -42,44 +50,43 @@ class ComposeJobRunner(jobs.BaseJobRunner):
"-c",
command,
)
else:
fmt.echo_alert(
(
"The '{job_service_name}' service does not exist in {jobs_path}. "
"This might be caused by an older plugin. Tutor switched to a job "
"runner model for running one-time commands, such as database"
" initialisation. For the record, this is the command that we are "
"running:\n"
"\n"
" {command}\n"
"\n"
"Old-style job running will be deprecated soon. Please inform "
"your plugin maintainer!"
).format(
job_service_name=job_service_name,
jobs_path=jobs_path,
command=command.replace("\n", "\n "),
)
)
self.docker_compose_func(
self.root,
self.config,
"run",
*opts,
"--rm",
service,
"sh",
"-e",
"-c",
command,
fmt.echo_alert(
(
"The '{job_service_name}' service does not exist in {jobs_path}. "
"This might be caused by an older plugin. Tutor switched to a job "
"runner model for running one-time commands, such as database"
" initialisation. For the record, this is the command that we are "
"running:\n"
"\n"
" {command}\n"
"\n"
"Old-style job running will be deprecated soon. Please inform "
"your plugin maintainer!"
).format(
job_service_name=job_service_name,
jobs_path=jobs_path,
command=command.replace("\n", "\n "),
)
)
return self.docker_compose_func(
self.root,
self.config,
"run",
*opts,
"--rm",
service,
"sh",
"-e",
"-c",
command,
)
@click.command(help="Run all or a selection of configured Open edX services")
@click.option("-d", "--detach", is_flag=True, help="Start in daemon mode")
@click.argument("services", metavar="service", nargs=-1)
@click.pass_obj
def start(context, detach, services):
def start(context: Context, detach: bool, services: List[str]) -> None:
command = ["up", "--remove-orphans"]
if detach:
command.append("-d")
@ -91,7 +98,7 @@ def start(context, detach, services):
@click.command(help="Stop a running platform")
@click.argument("services", metavar="service", nargs=-1)
@click.pass_obj
def stop(context, services):
def stop(context: Context, services: List[str]) -> None:
config = tutor_config.load(context.root)
context.docker_compose(context.root, config, "stop", *services)
@ -102,9 +109,10 @@ def stop(context, services):
)
@click.option("-d", "--detach", is_flag=True, help="Start in daemon mode")
@click.argument("services", metavar="service", nargs=-1)
def reboot(detach, services):
stop.callback(services)
start.callback(detach, services)
@click.pass_context
def reboot(context: click.Context, detach: bool, services: List[str]) -> None:
context.invoke(stop, services=services)
context.invoke(start, detach=detach, services=services)
@click.command(
@ -116,7 +124,7 @@ fully stop the platform, use the 'reboot' command.""",
)
@click.argument("services", metavar="service", nargs=-1)
@click.pass_obj
def restart(context, services):
def restart(context: Context, services: List[str]) -> None:
config = tutor_config.load(context.root)
command = ["restart"]
if "all" in services:
@ -136,7 +144,7 @@ def restart(context, services):
@click.command(help="Initialise all applications")
@click.option("-l", "--limit", help="Limit initialisation to this service or plugin")
@click.pass_obj
def init(context, limit):
def init(context: Context, limit: str) -> None:
config = tutor_config.load(context.root)
runner = ComposeJobRunner(context.root, config, context.docker_compose)
jobs.initialise(runner, limit_to=limit)
@ -153,7 +161,9 @@ def init(context, limit):
@click.argument("name")
@click.argument("email")
@click.pass_obj
def createuser(context, superuser, staff, password, name, email):
def createuser(
context: Context, superuser: str, staff: bool, password: str, name: str, email: str
) -> None:
config = tutor_config.load(context.root)
runner = ComposeJobRunner(context.root, config, context.docker_compose)
command = jobs.create_user_command(superuser, staff, name, email, password=password)
@ -166,7 +176,7 @@ def createuser(context, superuser, staff, password, name, email):
@click.argument("theme_name")
@click.argument("domain_names", metavar="domain_name", nargs=-1)
@click.pass_obj
def settheme(context, theme_name, domain_names):
def settheme(context: Context, theme_name: str, domain_names: List[str]) -> None:
config = tutor_config.load(context.root)
runner = ComposeJobRunner(context.root, config, context.docker_compose)
for domain_name in domain_names:
@ -175,7 +185,7 @@ def settheme(context, theme_name, domain_names):
@click.command(help="Import the demo course")
@click.pass_obj
def importdemocourse(context):
def importdemocourse(context: Context) -> None:
config = tutor_config.load(context.root)
runner = ComposeJobRunner(context.root, config, context.docker_compose)
fmt.echo_info("Importing demo course")
@ -192,11 +202,12 @@ def importdemocourse(context):
context_settings={"ignore_unknown_options": True},
)
@click.argument("args", nargs=-1, required=True)
def run(args):
@click.pass_context
def run(context: click.Context, args: List[str]) -> None:
extra_args = ["--rm"]
if not utils.is_a_tty():
extra_args.append("-T")
dc_command.callback("run", [*extra_args, *args])
context.invoke(dc_command, command="run", args=[*extra_args, *args])
@click.command(
@ -208,7 +219,7 @@ def run(args):
)
@click.argument("path")
@click.pass_obj
def bindmount_command(context, service, path):
def bindmount_command(context: Context, service: str, path: str) -> None:
config = tutor_config.load(context.root)
host_path = bindmounts.create(
context.root, config, context.docker_compose, service, path
@ -231,8 +242,9 @@ def bindmount_command(context, service, path):
name="exec",
)
@click.argument("args", nargs=-1, required=True)
def execute(args):
dc_command.callback("exec", args)
@click.pass_context
def execute(context: click.Context, args: List[str]) -> None:
context.invoke(dc_command, command="exec", args=args)
@click.command(
@ -242,14 +254,15 @@ def execute(args):
@click.option("-f", "--follow", is_flag=True, help="Follow log output")
@click.option("--tail", type=int, help="Number of lines to show from each container")
@click.argument("service", nargs=-1)
def logs(follow, tail, service):
@click.pass_context
def logs(context: click.Context, follow: bool, tail: bool, service: str) -> None:
args = []
if follow:
args.append("--follow")
if tail is not None:
args += ["--tail", str(tail)]
args += service
dc_command.callback("logs", args)
context.invoke(dc_command, command="logs", args=args)
@click.command(
@ -264,7 +277,7 @@ def logs(follow, tail, service):
@click.argument("command")
@click.argument("args", nargs=-1, required=True)
@click.pass_obj
def dc_command(context, command, args):
def dc_command(context: Context, command: str, args: List[str]) -> None:
config = tutor_config.load(context.root)
volumes, non_volume_args = bindmounts.parse_volumes(args)
volume_args = []
@ -286,7 +299,7 @@ def dc_command(context, command, args):
)
def add_commands(command_group):
def add_commands(command_group: click.Group) -> None:
command_group.add_command(start)
command_group.add_command(stop)
command_group.add_command(restart)

15
tutor/commands/config.py
View File

@ -1,3 +1,5 @@
from typing import Any, Dict, List
import click
from .. import config as tutor_config
@ -6,6 +8,7 @@ from .. import exceptions
from .. import fmt
from .. import interactive as interactive_config
from .. import serialize
from .context import Context
@click.group(
@ -13,7 +16,7 @@ from .. import serialize
short_help="Configure Open edX",
help="""Configure Open edX and store configuration values in $TUTOR_ROOT/config.yml""",
)
def config_command():
def config_command() -> None:
pass
@ -36,7 +39,9 @@ def config_command():
help="Remove a configuration value (can be used multiple times)",
)
@click.pass_obj
def save(context, interactive, set_vars, unset_vars):
def save(
context: Context, interactive: bool, set_vars: Dict[str, Any], unset_vars: List[str]
) -> None:
config, defaults = interactive_config.load_all(
context.root, interactive=interactive
)
@ -61,7 +66,7 @@ def save(context, interactive, set_vars, unset_vars):
@click.argument("src", type=click.Path(exists=True, resolve_path=True))
@click.argument("dst")
@click.pass_obj
def render(context, extra_configs, src, dst):
def render(context: Context, extra_configs: List[str], src: str, dst: str) -> None:
config = tutor_config.load(context.root)
for extra_config in extra_configs:
tutor_config.merge(
@ -75,14 +80,14 @@ def render(context, extra_configs, src, dst):
@click.command(help="Print the project root")
@click.pass_obj
def printroot(context):
def printroot(context: Context) -> None:
click.echo(context.root)
@click.command(help="Print a configuration value")
@click.argument("key")
@click.pass_obj
def printvalue(context, key):
def printvalue(context: Context, key: str) -> None:
config = tutor_config.load(context.root)
try:
# Note that this will incorrectly print None values

17
tutor/commands/context.py
View File

@ -1,8 +1,17 @@
from typing import Any, Dict
def unimplemented_docker_compose(
root: str, config: Dict[str, Any], *command: str
) -> int:
raise NotImplementedError
# pylint: disable=too-few-public-methods
class Context:
def __init__(self, root):
def __init__(self, root: str) -> None:
self.root = root
self.docker_compose_func = unimplemented_docker_compose
@staticmethod
def docker_compose(root, config, *command):
raise NotImplementedError
def docker_compose(self, root: str, config: Dict[str, Any], *command: str) -> int:
return self.docker_compose_func(root, config, *command)

20
tutor/commands/dev.py
View File

@ -1,15 +1,17 @@
import os
from typing import Any, Dict, List
import click
from . import compose
from .. import config as tutor_config
from .. import env as tutor_env
from .. import fmt
from .. import utils
from . import compose
from .context import Context
def docker_compose(root, config, *command):
def docker_compose(root: str, config: Dict[str, Any], *command: str) -> int:
"""
Run docker-compose with dev arguments.
"""
@ -27,15 +29,15 @@ def docker_compose(root, config, *command):
return utils.docker_compose(
*args,
"--project-name",
config["DEV_PROJECT_NAME"],
str(config["DEV_PROJECT_NAME"]),
*command,
)
@click.group(help="Run Open edX locally with development settings")
@click.pass_obj
def dev(context):
context.docker_compose = docker_compose
def dev(context: Context) -> None:
context.docker_compose_func = docker_compose
@click.command(
@ -44,9 +46,9 @@ def dev(context):
)
@click.argument("options", nargs=-1, required=False)
@click.argument("service")
@click.pass_obj
def runserver(context, options, service):
config = tutor_config.load(context.root)
@click.pass_context
def runserver(context: click.Context, options: List[str], service: str) -> None:
config = tutor_config.load(context.obj.root)
if service in ["lms", "cms"]:
port = 8000 if service == "lms" else 8001
host = config["LMS_HOST"] if service == "lms" else config["CMS_HOST"]
@ -56,7 +58,7 @@ def runserver(context, options, service):
)
)
args = ["--service-ports", *options, service]
compose.run.callback(args)
context.invoke(compose.run, args=args)
dev.add_command(runserver)

41
tutor/commands/images.py
View File

@ -1,3 +1,5 @@
from typing import cast, Any, Dict, Iterator, List, Tuple
import click
from .. import config as tutor_config
@ -5,6 +7,7 @@ from .. import env as tutor_env
from .. import images
from .. import plugins
from .. import utils
from .context import Context
BASE_IMAGE_NAMES = ["openedx", "forum", "android"]
DEV_IMAGE_NAMES = ["openedx-dev"]
@ -20,7 +23,7 @@ VENDOR_IMAGES = [
@click.group(name="images", short_help="Manage docker images")
def images_command():
def images_command() -> None:
pass
@ -50,7 +53,14 @@ def images_command():
help="Set the target build stage to build.",
)
@click.pass_obj
def build(context, image_names, no_cache, build_args, add_hosts, target):
def build(
context: Context,
image_names: List[str],
no_cache: bool,
build_args: List[str],
add_hosts: List[str],
target: str,
) -> None:
config = tutor_config.load(context.root)
command_args = []
if no_cache:
@ -68,7 +78,7 @@ def build(context, image_names, no_cache, build_args, add_hosts, target):
@click.command(short_help="Pull images from the Docker registry")
@click.argument("image_names", metavar="image", nargs=-1)
@click.pass_obj
def pull(context, image_names):
def pull(context: Context, image_names: List[str]) -> None:
config = tutor_config.load(context.root)
for image in image_names:
pull_image(config, image)
@ -77,7 +87,7 @@ def pull(context, image_names):
@click.command(short_help="Push images to the Docker registry")
@click.argument("image_names", metavar="image", nargs=-1)
@click.pass_obj
def push(context, image_names):
def push(context: Context, image_names: List[str]) -> None:
config = tutor_config.load(context.root)
for image in image_names:
push_image(config, image)
@ -86,16 +96,16 @@ def push(context, image_names):
@click.command(short_help="Print tag associated to a Docker image")
@click.argument("image_names", metavar="image", nargs=-1)
@click.pass_obj
def printtag(context, image_names):
def printtag(context: Context, image_names: List[str]) -> None:
config = tutor_config.load(context.root)
for image in image_names:
for _img, tag in iter_images(config, image, BASE_IMAGE_NAMES):
print(tag)
for _img, tag in iter_plugin_images(config, image, "build-image"):
for _plugin, _img, tag in iter_plugin_images(config, image, "build-image"):
print(tag)
def build_image(root, config, image, *args):
def build_image(root: str, config: Dict[str, Any], image: str, *args: str) -> None:
# Build base images
for img, tag in iter_images(config, image, BASE_IMAGE_NAMES):
images.build(tutor_env.pathjoin(root, "build", img), tag, *args)
@ -112,40 +122,45 @@ def build_image(root, config, image, *args):
images.build(tutor_env.pathjoin(root, "build", img), tag, *dev_build_arg, *args)
def pull_image(config, image):
def pull_image(config: Dict[str, Any], image: str) -> None:
for _img, tag in iter_images(config, image, all_image_names(config)):
images.pull(tag)
for _plugin, _img, tag in iter_plugin_images(config, image, "remote-image"):
images.pull(tag)
def push_image(config, image):
def push_image(config: Dict[str, Any], image: str) -> None:
for _img, tag in iter_images(config, image, BASE_IMAGE_NAMES):
images.push(tag)
for _plugin, _img, tag in iter_plugin_images(config, image, "remote-image"):
images.push(tag)
def iter_images(config, image, image_list):
def iter_images(
config: Dict[str, Any], image: str, image_list: List[str]
) -> Iterator[Tuple[str, str]]:
for img in image_list:
if image in [img, "all"]:
tag = images.get_tag(config, img)
yield img, tag
def iter_plugin_images(config, image, hook_name):
def iter_plugin_images(
config: Dict[str, Any], image: str, hook_name: str
) -> Iterator[Tuple[str, str, str]]:
for plugin, hook in plugins.iter_hooks(config, hook_name):
hook = cast(Dict[str, str], hook)
for img, tag in hook.items():
if image in [img, "all"]:
tag = tutor_env.render_str(config, tag)
yield plugin, img, tag
def all_image_names(config):
def all_image_names(config: Dict[str, Any]) -> List[str]:
return BASE_IMAGE_NAMES + vendor_image_names(config)
def vendor_image_names(config):
def vendor_image_names(config: Dict[str, Any]) -> List[str]:
vendor_images = VENDOR_IMAGES[:]
for image in VENDOR_IMAGES:
if not config.get("RUN_" + image.upper(), True):

595
tutor/commands/k8s.py
View File

@ -1,5 +1,6 @@
from datetime import datetime
from time import sleep
from typing import cast, Any, Dict, List, Optional, Type
import click
@ -11,286 +12,13 @@ from .. import interactive as interactive_config
from .. import jobs
from .. import serialize
from .. import utils
@click.group(help="Run Open edX on Kubernetes")
def k8s():
pass
@click.command(help="Configure and run Open edX from scratch")
@click.option("-I", "--non-interactive", is_flag=True, help="Run non-interactively")
@click.pass_obj
def quickstart(context, non_interactive):
click.echo(fmt.title("Interactive platform configuration"))
config = interactive_config.update(context.root, interactive=(not non_interactive))
if not config["RUN_CADDY"]:
fmt.echo_alert(
"Potentially invalid configuration: RUN_CADDY=false\n"
"This setting might have been defined because you previously set WEB_PROXY=true. This is no longer"
" necessary in order to get Tutor to work on Kubernetes. In Tutor v11+ a Caddy-based load balancer is"
" provided out of the box to handle SSL/TLS certificate generation at runtime. If you disable this"
" service, you will have to configure an Ingress resource and a certificate manager yourself to redirect"
" traffic to the nginx service. See the Kubernetes section in the Tutor documentation for more"
" information."
)
click.echo(fmt.title("Updating the current environment"))
tutor_env.save(context.root, config)
click.echo(fmt.title("Starting the platform"))
start.callback()
click.echo(fmt.title("Database creation and migrations"))
init.callback(limit=None)
fmt.echo_info(
"""Your Open edX platform is ready and can be accessed at the following urls:
{http}://{lms_host}
{http}://{cms_host}
""".format(
http="https" if config["ENABLE_HTTPS"] else "http",
lms_host=config["LMS_HOST"],
cms_host=config["CMS_HOST"],
)
)
@click.command(help="Run all configured Open edX services")
@click.pass_obj
def start(context):
# Create namespace
utils.kubectl(
"apply",
"--kustomize",
tutor_env.pathjoin(context.root),
"--wait",
"--selector",
"app.kubernetes.io/component=namespace",
)
# Create volumes
utils.kubectl(
"apply",
"--kustomize",
tutor_env.pathjoin(context.root),
"--wait",
"--selector",
"app.kubernetes.io/component=volume",
)
# Create everything else except jobs
utils.kubectl(
"apply",
"--kustomize",
tutor_env.pathjoin(context.root),
"--selector",
# Here use `notin (job, xxx)` when there are other components to ignore
"app.kubernetes.io/component!=job",
)
@click.command(help="Stop a running platform")
@click.pass_obj
def stop(context):
config = tutor_config.load(context.root)
utils.kubectl(
"delete",
*resource_selector(config),
"deployments,services,configmaps,jobs",
)
@click.command(help="Reboot an existing platform")
def reboot():
stop.callback()
start.callback()
def resource_selector(config, *selectors):
"""
Convenient utility for filtering only the resources that belong to this project.
"""
selector = ",".join(
["app.kubernetes.io/instance=openedx-" + config["ID"]] + list(selectors)
)
return ["--namespace", config["K8S_NAMESPACE"], "--selector=" + selector]
@click.command(help="Completely delete an existing platform")
@click.option("-y", "--yes", is_flag=True, help="Do not ask for confirmation")
@click.pass_obj
def delete(context, yes):
if not yes:
click.confirm(
"Are you sure you want to delete the platform? All data will be removed.",
abort=True,
)
utils.kubectl(
"delete",
"-k",
tutor_env.pathjoin(context.root),
"--ignore-not-found=true",
"--wait",
)
@click.command(help="Initialise all applications")
@click.option("-l", "--limit", help="Limit initialisation to this service or plugin")
@click.pass_obj
def init(context, limit):
config = tutor_config.load(context.root)
runner = K8sJobRunner(context.root, config)
for service in ["mysql", "elasticsearch", "mongodb"]:
if tutor_config.is_service_activated(config, service):
wait_for_pod_ready(config, service)
jobs.initialise(runner, limit_to=limit)
@click.command(help="Create an Open edX user and interactively set their password")
@click.option("--superuser", is_flag=True, help="Make superuser")
@click.option("--staff", is_flag=True, help="Make staff user")
@click.option(
"-p",
"--password",
help="Specify password from the command line. If undefined, you will be prompted to input a password",
)
@click.argument("name")
@click.argument("email")
@click.pass_obj
def createuser(context, superuser, staff, password, name, email):
config = tutor_config.load(context.root)
command = jobs.create_user_command(superuser, staff, name, email, password=password)
# This needs to be interactive in case the user needs to type a password
kubectl_exec(config, "lms", command, attach=True)
@click.command(help="Import the demo course")
@click.pass_obj
def importdemocourse(context):
fmt.echo_info("Importing demo course")
config = tutor_config.load(context.root)
runner = K8sJobRunner(context.root, config)
jobs.import_demo_course(runner)
@click.command(
help="Set a theme for a given domain name. To reset to the default theme , use 'default' as the theme name."
)
@click.argument("theme_name")
@click.argument("domain_names", metavar="domain_name", nargs=-1)
@click.pass_obj
def settheme(context, theme_name, domain_names):
config = tutor_config.load(context.root)
runner = K8sJobRunner(context.root, config)
for domain_name in domain_names:
jobs.set_theme(theme_name, domain_name, runner)
@click.command(name="exec", help="Execute a command in a pod of the given application")
@click.argument("service")
@click.argument("command")
@click.pass_obj
def exec_command(context, service, command):
config = tutor_config.load(context.root)
kubectl_exec(config, service, command, attach=True)
@click.command(help="View output from containers")
@click.option("-c", "--container", help="Print the logs of this specific container")
@click.option("-f", "--follow", is_flag=True, help="Follow log output")
@click.option("--tail", type=int, help="Number of lines to show from each container")
@click.argument("service")
@click.pass_obj
def logs(context, container, follow, tail, service):
config = tutor_config.load(context.root)
command = ["logs"]
selectors = ["app.kubernetes.io/name=" + service] if service else []
command += resource_selector(config, *selectors)
if container:
command += ["-c", container]
if follow:
command += ["--follow"]
if tail is not None:
command += ["--tail", str(tail)]
utils.kubectl(*command)
@click.command(help="Wait for a pod to become ready")
@click.argument("name")
@click.pass_obj
def wait(context, name):
config = tutor_config.load(context.root)
wait_for_pod_ready(config, name)
@click.command(help="Upgrade from a previous Open edX named release")
@click.option(
"--from", "from_version", default="ironwood", type=click.Choice(["ironwood"])
)
@click.pass_obj
def upgrade(context, from_version):
config = tutor_config.load(context.root)
running_version = from_version
if running_version == "ironwood":
upgrade_from_ironwood(config)
running_version = "juniper"
if running_version == "juniper":
running_version = "koa"
def upgrade_from_ironwood(config):
if not config["RUN_MONGODB"]:
fmt.echo_info(
"You are not running MongDB (RUN_MONGODB=false). It is your "
"responsibility to upgrade your MongoDb instance to v3.6. There is "
"nothing left to do to upgrade from Ironwood."
)
return
message = """Automatic release upgrade is unsupported in Kubernetes. To upgrade from Ironwood, you should upgrade
your MongoDb cluster from v3.2 to v3.6. You should run something similar to:
# Upgrade from v3.2 to v3.4
tutor k8s stop
tutor config save --set DOCKER_IMAGE_MONGODB=mongo:3.4.24
tutor k8s start
tutor k8s exec mongodb mongo --eval 'db.adminCommand({ setFeatureCompatibilityVersion: "3.4" })'
# Upgrade from v3.4 to v3.6
tutor k8s stop
tutor config save --set DOCKER_IMAGE_MONGODB=mongo:3.6.18
tutor k8s start
tutor k8s exec mongodb mongo --eval 'db.adminCommand({ setFeatureCompatibilityVersion: "3.6" })'
tutor config save --unset DOCKER_IMAGE_MONGODB"""
fmt.echo_info(message)
def upgrade_from_juniper(config):
if not config["RUN_MYSQL"]:
fmt.echo_info(
"You are not running MySQL (RUN_MYSQL=false). It is your "
"responsibility to upgrade your MySQL instance to v5.7. There is "
"nothing left to do to upgrade from Juniper."
)
return
message = """Automatic release upgrade is unsupported in Kubernetes. To upgrade from Juniper, you should upgrade
your MySQL database from v5.6 to v5.7. You should run something similar to:
tutor k8s start
tutor k8s exec mysql bash -e -c "mysql_upgrade \
-u $(tutor config printvalue MYSQL_ROOT_USERNAME) \
--password='$(tutor config printvalue MYSQL_ROOT_PASSWORD)'
"""
fmt.echo_info(message)
from .context import Context
class K8sClients:
_instance = None
def __init__(self):
def __init__(self) -> None:
# Loading the kubernetes module here to avoid import overhead
from kubernetes import client, config # pylint: disable=import-outside-toplevel
@ -300,33 +28,34 @@ class K8sClients:
self._client = client
@classmethod
def instance(cls):
def instance(cls: Type["K8sClients"]) -> "K8sClients":
if cls._instance is None:
cls._instance = cls()
return cls._instance
@property
def batch_api(self):
def batch_api(self): # type: ignore
if self._batch_api is None:
self._batch_api = self._client.BatchV1Api()
return self._batch_api
@property
def core_api(self):
def core_api(self): # type: ignore
if self._core_api is None:
self._core_api = self._client.CoreV1Api()
return self._core_api
class K8sJobRunner(jobs.BaseJobRunner):
def load_job(self, name):
jobs = self.render("k8s", "jobs.yml")
for job in serialize.load_all(jobs):
if job["metadata"]["name"] == name:
def load_job(self, name: str) -> Any:
all_jobs = self.render("k8s", "jobs.yml")
for job in serialize.load_all(all_jobs):
job_name = cast(str, job["metadata"]["name"])
if job_name == name:
return job
raise ValueError("Could not find job '{}'".format(name))
def active_job_names(self):
def active_job_names(self) -> List[str]:
"""
Return a list of active job names
Docs:
@ -339,7 +68,7 @@ class K8sJobRunner(jobs.BaseJobRunner):
if job.status.active
]
def run_job(self, service, command):
def run_job(self, service: str, command: str) -> int:
job_name = "{}-job".format(service)
try:
job = self.load_job(job_name)
@ -361,8 +90,7 @@ class K8sJobRunner(jobs.BaseJobRunner):
)
fmt.echo_alert(message)
wait_for_pod_ready(self.config, service)
kubectl_exec(self.config, service, command)
return
return kubectl_exec(self.config, service, command)
# Create a unique job name to make it deduplicate jobs and make it easier to
# find later. Logs of older jobs will remain available for some time.
job_name += "-" + datetime.now().strftime("%Y%m%d%H%M%S")
@ -417,12 +145,12 @@ class K8sJobRunner(jobs.BaseJobRunner):
# Wait for completion
field_selector = "metadata.name={}".format(job_name)
while True:
jobs = K8sClients.instance().batch_api.list_namespaced_job(
namespaced_jobs = K8sClients.instance().batch_api.list_namespaced_job(
self.config["K8S_NAMESPACE"], field_selector=field_selector
)
if not jobs.items:
if not namespaced_jobs.items:
continue
job = jobs.items[0]
job = namespaced_jobs.items[0]
if not job.status.active:
if job.status.succeeded:
fmt.echo_info("Job {} successful.".format(job_name))
@ -434,9 +162,292 @@ class K8sJobRunner(jobs.BaseJobRunner):
)
)
sleep(5)
return 0
def kubectl_exec(config, service, command, attach=False):
@click.group(help="Run Open edX on Kubernetes")
def k8s() -> None:
pass
@click.command(help="Configure and run Open edX from scratch")
@click.option("-I", "--non-interactive", is_flag=True, help="Run non-interactively")
@click.pass_context
def quickstart(context: click.Context, non_interactive: bool) -> None:
click.echo(fmt.title("Interactive platform configuration"))
config = interactive_config.update(
context.obj.root, interactive=(not non_interactive)
)
if not config["RUN_CADDY"]:
fmt.echo_alert(
"Potentially invalid configuration: RUN_CADDY=false\n"
"This setting might have been defined because you previously set WEB_PROXY=true. This is no longer"
" necessary in order to get Tutor to work on Kubernetes. In Tutor v11+ a Caddy-based load balancer is"
" provided out of the box to handle SSL/TLS certificate generation at runtime. If you disable this"
" service, you will have to configure an Ingress resource and a certificate manager yourself to redirect"
" traffic to the nginx service. See the Kubernetes section in the Tutor documentation for more"
" information."
)
click.echo(fmt.title("Updating the current environment"))
tutor_env.save(context.obj.root, config)
click.echo(fmt.title("Starting the platform"))
context.invoke(start)
click.echo(fmt.title("Database creation and migrations"))
context.invoke(init, limit=None)
fmt.echo_info(
"""Your Open edX platform is ready and can be accessed at the following urls:
{http}://{lms_host}
{http}://{cms_host}
""".format(
http="https" if config["ENABLE_HTTPS"] else "http",
lms_host=config["LMS_HOST"],
cms_host=config["CMS_HOST"],
)
)
@click.command(help="Run all configured Open edX services")
@click.pass_obj
def start(context: Context) -> None:
# Create namespace
utils.kubectl(
"apply",
"--kustomize",
tutor_env.pathjoin(context.root),
"--wait",
"--selector",
"app.kubernetes.io/component=namespace",
)
# Create volumes
utils.kubectl(
"apply",
"--kustomize",
tutor_env.pathjoin(context.root),
"--wait",
"--selector",
"app.kubernetes.io/component=volume",
)
# Create everything else except jobs
utils.kubectl(
"apply",
"--kustomize",
tutor_env.pathjoin(context.root),
"--selector",
# Here use `notin (job, xxx)` when there are other components to ignore
"app.kubernetes.io/component!=job",
)
@click.command(help="Stop a running platform")
@click.pass_obj
def stop(context: Context) -> None:
config = tutor_config.load(context.root)
utils.kubectl(
"delete",
*resource_selector(config),
"deployments,services,configmaps,jobs",
)
@click.command(help="Reboot an existing platform")
@click.pass_context
def reboot(context: click.Context) -> None:
context.invoke(stop)
context.invoke(start)
def resource_selector(config: Dict[str, str], *selectors: str) -> List[str]:
"""
Convenient utility for filtering only the resources that belong to this project.
"""
selector = ",".join(
["app.kubernetes.io/instance=openedx-" + config["ID"]] + list(selectors)
)
return ["--namespace", config["K8S_NAMESPACE"], "--selector=" + selector]
@click.command(help="Completely delete an existing platform")
@click.option("-y", "--yes", is_flag=True, help="Do not ask for confirmation")
@click.pass_obj
def delete(context: Context, yes: bool) -> None:
if not yes:
click.confirm(
"Are you sure you want to delete the platform? All data will be removed.",
abort=True,
)
utils.kubectl(
"delete",
"-k",
tutor_env.pathjoin(context.root),
"--ignore-not-found=true",
"--wait",
)
@click.command(help="Initialise all applications")
@click.option("-l", "--limit", help="Limit initialisation to this service or plugin")
@click.pass_obj
def init(context: Context, limit: Optional[str]) -> None:
config = tutor_config.load(context.root)
runner = K8sJobRunner(context.root, config)
for service in ["mysql", "elasticsearch", "mongodb"]:
if tutor_config.is_service_activated(config, service):
wait_for_pod_ready(config, service)
jobs.initialise(runner, limit_to=limit)
@click.command(help="Create an Open edX user and interactively set their password")
@click.option("--superuser", is_flag=True, help="Make superuser")
@click.option("--staff", is_flag=True, help="Make staff user")
@click.option(
"-p",
"--password",
help="Specify password from the command line. If undefined, you will be prompted to input a password",
)
@click.argument("name")
@click.argument("email")
@click.pass_obj
def createuser(
context: Context, superuser: str, staff: bool, password: str, name: str, email: str
) -> None:
config = tutor_config.load(context.root)
command = jobs.create_user_command(superuser, staff, name, email, password=password)
# This needs to be interactive in case the user needs to type a password
kubectl_exec(config, "lms", command, attach=True)
@click.command(help="Import the demo course")
@click.pass_obj
def importdemocourse(context: Context) -> None:
fmt.echo_info("Importing demo course")
config = tutor_config.load(context.root)
runner = K8sJobRunner(context.root, config)
jobs.import_demo_course(runner)
@click.command(
help="Set a theme for a given domain name. To reset to the default theme , use 'default' as the theme name."
)
@click.argument("theme_name")
@click.argument("domain_names", metavar="domain_name", nargs=-1)
@click.pass_obj
def settheme(context: Context, theme_name: str, domain_names: List[str]) -> None:
config = tutor_config.load(context.root)
runner = K8sJobRunner(context.root, config)
for domain_name in domain_names:
jobs.set_theme(theme_name, domain_name, runner)
@click.command(name="exec", help="Execute a command in a pod of the given application")
@click.argument("service")
@click.argument("command")
@click.pass_obj
def exec_command(context: Context, service: str, command: str) -> None:
config = tutor_config.load(context.root)
kubectl_exec(config, service, command, attach=True)
@click.command(help="View output from containers")
@click.option("-c", "--container", help="Print the logs of this specific container")
@click.option("-f", "--follow", is_flag=True, help="Follow log output")
@click.option("--tail", type=int, help="Number of lines to show from each container")
@click.argument("service")
@click.pass_obj
def logs(
context: Context, container: str, follow: bool, tail: bool, service: str
) -> None:
config = tutor_config.load(context.root)
command = ["logs"]
selectors = ["app.kubernetes.io/name=" + service] if service else []
command += resource_selector(config, *selectors)
if container:
command += ["-c", container]
if follow:
command += ["--follow"]
if tail is not None:
command += ["--tail", str(tail)]
utils.kubectl(*command)
@click.command(help="Wait for a pod to become ready")
@click.argument("name")
@click.pass_obj
def wait(context: Context, name: str) -> None:
config = tutor_config.load(context.root)
wait_for_pod_ready(config, name)
@click.command(help="Upgrade from a previous Open edX named release")
@click.option(
"--from", "from_version", default="ironwood", type=click.Choice(["ironwood"])
)
@click.pass_obj
def upgrade(context: Context, from_version: str) -> None:
config = tutor_config.load(context.root)
running_version = from_version
if running_version == "ironwood":
upgrade_from_ironwood(config)
running_version = "juniper"
if running_version == "juniper":
running_version = "koa"
def upgrade_from_ironwood(config: Dict[str, Any]) -> None:
if not config["RUN_MONGODB"]:
fmt.echo_info(
"You are not running MongDB (RUN_MONGODB=false). It is your "
"responsibility to upgrade your MongoDb instance to v3.6. There is "
"nothing left to do to upgrade from Ironwood."
)
return
message = """Automatic release upgrade is unsupported in Kubernetes. To upgrade from Ironwood, you should upgrade
your MongoDb cluster from v3.2 to v3.6. You should run something similar to:
# Upgrade from v3.2 to v3.4
tutor k8s stop
tutor config save --set DOCKER_IMAGE_MONGODB=mongo:3.4.24
tutor k8s start
tutor k8s exec mongodb mongo --eval 'db.adminCommand({ setFeatureCompatibilityVersion: "3.4" })'
# Upgrade from v3.4 to v3.6
tutor k8s stop
tutor config save --set DOCKER_IMAGE_MONGODB=mongo:3.6.18
tutor k8s start
tutor k8s exec mongodb mongo --eval 'db.adminCommand({ setFeatureCompatibilityVersion: "3.6" })'
tutor config save --unset DOCKER_IMAGE_MONGODB"""
fmt.echo_info(message)
def upgrade_from_juniper(config: Dict[str, Any]) -> None:
if not config["RUN_MYSQL"]:
fmt.echo_info(
"You are not running MySQL (RUN_MYSQL=false). It is your "
"responsibility to upgrade your MySQL instance to v5.7. There is "
"nothing left to do to upgrade from Juniper."
)
return
message = """Automatic release upgrade is unsupported in Kubernetes. To upgrade from Juniper, you should upgrade
your MySQL database from v5.6 to v5.7. You should run something similar to:
tutor k8s start
tutor k8s exec mysql bash -e -c "mysql_upgrade \
-u $(tutor config printvalue MYSQL_ROOT_USERNAME) \
--password='$(tutor config printvalue MYSQL_ROOT_PASSWORD)'
"""
fmt.echo_info(message)
def kubectl_exec(
config: Dict[str, Any], service: str, command: str, attach: bool = False
) -> int:
selector = "app.kubernetes.io/name={}".format(service)
pods = K8sClients.instance().core_api.list_namespaced_pod(
namespace=config["K8S_NAMESPACE"], label_selector=selector
@ -449,7 +460,7 @@ def kubectl_exec(config, service, command, attach=False):
# Run command
attach_opts = ["-i", "-t"] if attach else []
utils.kubectl(
return utils.kubectl(
"exec",
*attach_opts,
"--namespace",
@ -463,7 +474,7 @@ def kubectl_exec(config, service, command, attach=False):
)
def wait_for_pod_ready(config, service):
def wait_for_pod_ready(config: Dict[str, str], service: str) -> None:
fmt.echo_info("Waiting for a {} pod to be ready...".format(service))
utils.kubectl(
"wait",

100
tutor/commands/local.py
View File

@ -1,15 +1,18 @@
import os
from typing import Dict, Any
import click
from .. import config as tutor_config
from .. import env as tutor_env
from .. import fmt, utils
from .. import fmt
from .. import utils
from . import compose
from .config import save as config_save_command
from .context import Context
def docker_compose(root, config, *command):
def docker_compose(root: str, config: Dict[str, Any], *command: str) -> int:
"""
Run docker-compose with local and production yml files.
"""
@ -31,39 +34,41 @@ def docker_compose(root, config, *command):
@click.group(help="Run Open edX locally with docker-compose")
@click.pass_obj
def local(context):
context.docker_compose = docker_compose
def local(context: Context) -> None:
context.docker_compose_func = docker_compose
@click.command(help="Configure and run Open edX from scratch")
@click.option("-I", "--non-interactive", is_flag=True, help="Run non-interactively")
@click.option(
"-p", "--pullimages", "pullimages_", is_flag=True, help="Update docker images"
)
@click.pass_obj
def quickstart(context, non_interactive, pullimages_):
if tutor_env.needs_major_upgrade(context.root):
@click.option("-p", "--pullimages", is_flag=True, help="Update docker images")
@click.pass_context
def quickstart(context: click.Context, non_interactive: bool, pullimages: bool) -> None:
if tutor_env.needs_major_upgrade(context.obj.root):
click.echo(fmt.title("Upgrading from an older release"))
upgrade.callback(
from_version=tutor_env.current_release(context.root),
context.invoke(
upgrade,
from_version=tutor_env.current_release(context.obj.root),
non_interactive=non_interactive,
)
click.echo(fmt.title("Interactive platform configuration"))
config_save_command.callback(
interactive=(not non_interactive), set_vars=[], unset_vars=[]
context.invoke(
config_save_command,
interactive=(not non_interactive),
set_vars=[],
unset_vars=[],
)
click.echo(fmt.title("Stopping any existing platform"))
compose.stop.callback([])
if pullimages_:
context.invoke(compose.stop)
if pullimages:
click.echo(fmt.title("Docker image updates"))
compose.dc_command.callback(["pull"])
context.invoke(compose.dc_command, command="pull")
click.echo(fmt.title("Starting the platform in detached mode"))
compose.start.callback(True, [])
context.invoke(compose.start, detach=True)
click.echo(fmt.title("Database creation and migrations"))
compose.init.callback(limit=None)
context.invoke(compose.init)
config = tutor_config.load(context.root)
config = tutor_config.load(context.obj.root)
fmt.echo_info(
"""The Open edX platform is now running in detached mode
Your Open edX platform is ready and can be accessed at the following urls:
@ -86,9 +91,9 @@ Your Open edX platform is ready and can be accessed at the following urls:
type=click.Choice(["ironwood", "juniper"]),
)
@click.option("-I", "--non-interactive", is_flag=True, help="Run non-interactively")
@click.pass_obj
def upgrade(context, from_version, non_interactive):
config = tutor_config.load_no_check(context.root)
@click.pass_context
def upgrade(context: click.Context, from_version: str, non_interactive: bool) -> None:
config = tutor_config.load_no_check(context.obj.root)
if not non_interactive:
question = """You are about to upgrade your Open edX platform. It is strongly recommended to make a backup before upgrading. To do so, run:
@ -113,12 +118,12 @@ Are you sure you want to continue?"""
running_version = "koa"
def upgrade_from_ironwood(context, config):
def upgrade_from_ironwood(context: click.Context, config: Dict[str, Any]) -> None:
click.echo(fmt.title("Upgrading from Ironwood"))
tutor_env.save(context.root, config)
tutor_env.save(context.obj.root, config)
click.echo(fmt.title("Stopping any existing platform"))
compose.stop.callback([])
context.invoke(compose.stop)
if not config["RUN_MONGODB"]:
fmt.echo_info(
@ -132,39 +137,41 @@ def upgrade_from_ironwood(context, config):
# environment, not the configuration.
click.echo(fmt.title("Upgrading MongoDb from v3.2 to v3.4"))
config["DOCKER_IMAGE_MONGODB"] = "mongo:3.4.24"
tutor_env.save(context.root, config)
compose.start.callback(detach=True, services=["mongodb"])
compose.execute.callback(
[
tutor_env.save(context.obj.root, config)
context.invoke(compose.start, detach=True, services=["mongodb"])
context.invoke(
compose.execute,
args=[
"mongodb",
"mongo",
"--eval",
'db.adminCommand({ setFeatureCompatibilityVersion: "3.4" })',
]
],
)
compose.stop.callback([])
context.invoke(compose.stop)
click.echo(fmt.title("Upgrading MongoDb from v3.4 to v3.6"))
config["DOCKER_IMAGE_MONGODB"] = "mongo:3.6.18"
tutor_env.save(context.root, config)
compose.start.callback(detach=True, services=["mongodb"])
compose.execute.callback(
[
tutor_env.save(context.obj.root, config)
context.invoke(compose.start, detach=True, services=["mongodb"])
context.invoke(
compose.execute,
args=[
"mongodb",
"mongo",
"--eval",
'db.adminCommand({ setFeatureCompatibilityVersion: "3.6" })',
]
],
)
compose.stop.callback([])
context.invoke(compose.stop)
def upgrade_from_juniper(context, config):
def upgrade_from_juniper(context: click.Context, config: Dict[str, Any]) -> None:
click.echo(fmt.title("Upgrading from Juniper"))
tutor_env.save(context.root, config)
tutor_env.save(context.obj.root, config)
click.echo(fmt.title("Stopping any existing platform"))
compose.stop.callback([])
context.invoke(compose.stop)
if not config["RUN_MYSQL"]:
fmt.echo_info(
@ -175,9 +182,10 @@ def upgrade_from_juniper(context, config):
return
click.echo(fmt.title("Upgrading MySQL from v5.6 to v5.7"))
compose.start.callback(detach=True, services=["mysql"])
compose.execute.callback(
[
context.invoke(compose.start, detach=True, services=["mysql"])
context.invoke(
compose.execute,
args=[
"mysql",
"bash",
"-e",
@ -185,9 +193,9 @@ def upgrade_from_juniper(context, config):
"mysql_upgrade -u {} --password='{}'".format(
config["MYSQL_ROOT_USERNAME"], config["MYSQL_ROOT_PASSWORD"]
),
]
],
)
compose.stop.callback([])
context.invoke(compose.stop)
local.add_command(quickstart)

18
tutor/commands/plugins.py
View File

@ -1,5 +1,6 @@
import os
import shutil
from typing import List
import urllib.request
import click
@ -9,6 +10,7 @@ from .. import env as tutor_env
from .. import exceptions
from .. import fmt
from .. import plugins
from .context import Context
@click.group(
@ -16,7 +18,7 @@ from .. import plugins
short_help="Manage Tutor plugins",
help="Manage Tutor plugins to add new features and customize your Open edX platform",
)
def plugins_command():
def plugins_command() -> None:
"""
All plugin commands should work even if there is no existing config file. This is
because users might enable plugins prior to configuration or environment generation.
@ -25,7 +27,7 @@ def plugins_command():
@click.command(name="list", help="List installed plugins")
@click.pass_obj
def list_command(context):
def list_command(context: Context) -> None:
config = tutor_config.load_user(context.root)
for plugin in plugins.iter_installed():
status = "" if plugins.is_enabled(config, plugin.name) else " (disabled)"
@ -39,7 +41,7 @@ def list_command(context):
@click.command(help="Enable a plugin")
@click.argument("plugin_names", metavar="plugin", nargs=-1)
@click.pass_obj
def enable(context, plugin_names):
def enable(context: Context, plugin_names: List[str]) -> None:
config = tutor_config.load_user(context.root)
for plugin in plugin_names:
plugins.enable(config, plugin)
@ -56,7 +58,7 @@ def enable(context, plugin_names):
)
@click.argument("plugin_names", metavar="plugin", nargs=-1)
@click.pass_obj
def disable(context, plugin_names):
def disable(context: Context, plugin_names: List[str]) -> None:
config = tutor_config.load_user(context.root)
if "all" in plugin_names:
plugin_names = [plugin.name for plugin in plugins.iter_enabled(config)]
@ -70,7 +72,7 @@ def disable(context, plugin_names):
)
def delete_plugin(root, name):
def delete_plugin(root: str, name: str) -> None:
plugin_dir = tutor_env.pathjoin(root, "plugins", name)
if os.path.exists(plugin_dir):
try:
@ -90,7 +92,7 @@ defined by setting the {} environment variable""".format(
plugins.DictPlugin.ROOT_ENV_VAR_NAME
),
)
def printroot():
def printroot() -> None:
fmt.echo(plugins.DictPlugin.ROOT)
@ -102,7 +104,7 @@ location. The plugin will be installed to {}.""".format(
),
)
@click.argument("location")
def install(location):
def install(location: str) -> None:
basename = os.path.basename(location)
if not basename.endswith(".yml"):
basename += ".yml"
@ -127,7 +129,7 @@ def install(location):
fmt.echo_info("Plugin installed at {}".format(plugin_path))
def add_plugin_commands(command_group):
def add_plugin_commands(command_group: click.Group) -> None:
"""
Add commands provided by all plugins to the given command group. Each command is
added with a name that is equal to the plugin name.

2
tutor/commands/ui.py
View File

@ -6,7 +6,7 @@ import click_repl
short_help="Interactive shell",
help="Launch an interactive shell for launching Tutor commands",
)
def ui():
def ui() -> None:
click.echo(
"""Welcome to the Tutor interactive shell UI!
Type "help" to view all available commands.

20
tutor/commands/webui.py
View File

@ -4,6 +4,7 @@ import platform
import subprocess
import sys
import tarfile
from typing import Any, Dict
from urllib.request import urlopen
import click
@ -13,12 +14,13 @@ import click
from .. import fmt
from .. import env as tutor_env
from .. import serialize
from .context import Context
@click.group(
short_help="Web user interface", help="""Run Tutor commands from a web terminal"""
)
def webui():
def webui() -> None:
pass
@ -35,7 +37,7 @@ def webui():
"-h", "--host", default="0.0.0.0", show_default=True, help="Host address to listen"
)
@click.pass_obj
def start(context, port, host):
def start(context: Context, port: int, host: str) -> None:
check_gotty_binary(context.root)
fmt.echo_info("Access the Tutor web UI at http://{}:{}".format(host, port))
while True:
@ -86,7 +88,7 @@ def start(context, port, host):
help="Authentication password",
)
@click.pass_obj
def configure(context, user, password):
def configure(context: Context, user: str, password: str) -> None:
save_webui_config_file(context.root, {"user": user, "password": password})
fmt.echo_info(
"The web UI configuration has been updated. "
@ -95,7 +97,7 @@ def configure(context, user, password):
)
def check_gotty_binary(root):
def check_gotty_binary(root: str) -> None:
path = gotty_path(root)
if os.path.exists(path):
return
@ -119,7 +121,7 @@ def check_gotty_binary(root):
compressed.extract("./gotty", dirname)
def load_config(root):
def load_config(root: str) -> Dict[str, Any]:
path = config_path(root)
if not os.path.exists(path):
save_webui_config_file(root, {"user": None, "password": None})
@ -127,7 +129,7 @@ def load_config(root):
return serialize.load(f)
def save_webui_config_file(root, config):
def save_webui_config_file(root: str, config: Dict[str, Any]) -> None:
path = config_path(root)
directory = os.path.dirname(path)
if not os.path.exists(directory):
@ -136,15 +138,15 @@ def save_webui_config_file(root, config):
serialize.dump(config, of)
def gotty_path(root):
def gotty_path(root: str) -> str:
return get_path(root, "gotty")
def config_path(root):
def config_path(root: str) -> str:
return get_path(root, "config.yml")
def get_path(root, filename):
def get_path(root: str, filename: str) -> str:
return tutor_env.pathjoin(root, "webui", filename)

41
tutor/config.py
View File

@ -1,4 +1,5 @@
import os
from typing import Dict, Any, Tuple
from . import exceptions
from . import env
@ -8,7 +9,7 @@ from . import serialize
from . import utils
def update(root):
def update(root: str) -> Dict[str, Any]:
"""
Load and save the configuration.
"""
@ -18,7 +19,7 @@ def update(root):
return config
def load(root):
def load(root: str) -> Dict[str, Any]:
"""
Load full configuration. This will raise an exception if there is no current
configuration in the project root.
@ -27,13 +28,13 @@ def load(root):
return load_no_check(root)
def load_no_check(root):
def load_no_check(root: str) -> Dict[str, Any]:
config, defaults = load_all(root)
merge(config, defaults)
return config
def load_all(root):
def load_all(root: str) -> Tuple[Dict[str, Any], Dict[str, Any]]:
"""
Return:
current (dict): params currently saved in config.yml
@ -45,7 +46,9 @@ def load_all(root):
return current, defaults
def merge(config, defaults, force=False):
def merge(
config: Dict[str, str], defaults: Dict[str, str], force: bool = False
) -> None:
"""
Merge default values with user configuration and perform rendering of "{{...}}"
values.
@ -55,16 +58,16 @@ def merge(config, defaults, force=False):
config[key] = env.render_unknown(config, value)
def load_defaults():
def load_defaults() -> Dict[str, Any]:
return serialize.load(env.read_template_file("config.yml"))
def load_config_file(path):
def load_config_file(path: str) -> Dict[str, Any]:
with open(path) as f:
return serialize.load(f.read())
def load_current(root, defaults):
def load_current(root: str, defaults: Dict[str, str]) -> Dict[str, Any]:
"""
Load the configuration currently stored on disk.
Note: this modifies the defaults with the plugin default values.
@ -77,7 +80,7 @@ def load_current(root, defaults):
return config
def load_user(root):
def load_user(root: str) -> Dict[str, Any]:
path = config_path(root)
if not os.path.exists(path):
return {}
@ -87,14 +90,14 @@ def load_user(root):
return config
def load_env(config, defaults):
def load_env(config: Dict[str, str], defaults: Dict[str, str]) -> None:
for k in defaults.keys():
env_var = "TUTOR_" + k
if env_var in os.environ:
config[k] = serialize.parse(os.environ[env_var])
def load_required(config, defaults):
def load_required(config: Dict[str, str], defaults: Dict[str, str]) -> None:
"""
All these keys must be present in the user's config.yml. This includes all values
that are generated once and must be kept after that, such as passwords.
@ -111,7 +114,7 @@ def load_required(config, defaults):
config[key] = env.render_unknown(config, defaults[key])
def load_plugins(config, defaults):
def load_plugins(config: Dict[str, str], defaults: Dict[str, str]) -> None:
"""
Add, override and set new defaults from plugins.
"""
@ -133,11 +136,11 @@ def load_plugins(config, defaults):
config[key] = env.render_unknown(config, value)
def is_service_activated(config, service):
return config["RUN_" + service.upper()]
def is_service_activated(config: Dict[str, Any], service: str) -> bool:
return config["RUN_" + service.upper()] is not False
def upgrade_obsolete(config):
def upgrade_obsolete(config: Dict[str, Any]) -> None:
# Openedx-specific mysql passwords
if "MYSQL_PASSWORD" in config:
config["MYSQL_ROOT_PASSWORD"] = config["MYSQL_PASSWORD"]
@ -178,7 +181,7 @@ def upgrade_obsolete(config):
config[name.replace("ACTIVATE_", "RUN_")] = config.pop(name)
def convert_json2yml(root):
def convert_json2yml(root: str) -> None:
"""
Older versions of tutor used to have json config files.
"""
@ -199,7 +202,7 @@ def convert_json2yml(root):
)
def save_config_file(root, config):
def save_config_file(root: str, config: Dict[str, str]) -> None:
path = config_path(root)
utils.ensure_file_directory_exists(path)
with open(path, "w") as of:
@ -207,7 +210,7 @@ def save_config_file(root, config):
fmt.echo_info("Configuration saved to {}".format(path))
def check_existing_config(root):
def check_existing_config(root: str) -> None:
"""
Check there is a configuration on disk and the current environment is up-to-date.
"""
@ -220,5 +223,5 @@ def check_existing_config(root):
env.check_is_up_to_date(root)
def config_path(root):
def config_path(root: str) -> str:
return os.path.join(root, "config.yml")

97
tutor/env.py
View File

@ -1,6 +1,7 @@
import codecs
from copy import deepcopy
import os
from typing import Dict, Any, Iterable, List, Optional, Type, Union
import jinja2
import pkg_resources
@ -19,7 +20,7 @@ BIN_FILE_EXTENSIONS = [".ico", ".jpg", ".png", ".ttf", ".woff", ".woff2"]
class Renderer:
@classmethod
def instance(cls, config):
def instance(cls: Type["Renderer"], config: Dict[str, Any]) -> "Renderer":
# Load template roots: these are required to be able to use
# {% include .. %} directives
template_roots = [TEMPLATES_ROOT]
@ -29,11 +30,12 @@ class Renderer:
return cls(config, template_roots, ignore_folders=["partials"])
@classmethod
def reset(cls):
cls.INSTANCE = None
def __init__(self, config, template_roots, ignore_folders=None):
def __init__(
self,
config: Dict[str, Any],
template_roots: List[str],
ignore_folders: Optional[List[str]] = None,
):
self.config = deepcopy(config)
self.template_roots = template_roots
self.ignore_folders = ignore_folders or []
@ -57,16 +59,16 @@ class Renderer:
environment.globals["TUTOR_VERSION"] = __version__
self.environment = environment
def iter_templates_in(self, *prefix):
def iter_templates_in(self, *prefix: str) -> Iterable[str]:
"""
The elements of `prefix` must contain only "/", and not os.sep.
"""
prefix = "/".join(prefix)
for template in self.environment.loader.list_templates():
if template.startswith(prefix) and self.is_part_of_env(template):
full_prefix = "/".join(prefix)
for template in self.environment.loader.list_templates(): # type: ignore
if template.startswith(full_prefix) and self.is_part_of_env(template):
yield template
def walk_templates(self, subdir):
def walk_templates(self, subdir: str) -> Iterable[str]:
"""
Iterate on the template files from `templates/<subdir>`.
@ -75,7 +77,7 @@ class Renderer:
"""
yield from self.iter_templates_in(subdir + "/")
def is_part_of_env(self, path):
def is_part_of_env(self, path: str) -> bool:
"""
Determines whether a template should be rendered or not. Note that here we don't
rely on the OS separator, as we are handling templates
@ -91,7 +93,7 @@ class Renderer:
is_excluded = is_excluded or ignore_folder in parts
return not is_excluded
def find_os_path(self, template_name):
def find_os_path(self, template_name: str) -> str:
path = template_name.replace("/", os.sep)
for templates_root in self.template_roots:
full_path = os.path.join(templates_root, path)
@ -99,7 +101,7 @@ class Renderer:
return full_path
raise ValueError("Template path does not exist")
def patch(self, name, separator="\n", suffix=""):
def patch(self, name: str, separator: str = "\n", suffix: str = "") -> str:
"""
Render calls to {{ patch("...") }} in environment templates from plugin patches.
"""
@ -119,11 +121,11 @@ class Renderer:
rendered += suffix
return rendered
def render_str(self, text):
def render_str(self, text: str) -> str:
template = self.environment.from_string(text)
return self.__render(template)
def render_template(self, template_name):
def render_template(self, template_name: str) -> Union[str, bytes]:
"""
Render a template file. Return the corresponding string. If it's a binary file
(as indicated by its path), return bytes.
@ -151,7 +153,7 @@ class Renderer:
fmt.echo_error("Unknown error rendering template " + template_name)
raise
def render_all_to(self, root, *prefix):
def render_all_to(self, root: str, *prefix: str) -> None:
"""
`prefix` can be used to limit the templates to render.
"""
@ -160,7 +162,7 @@ class Renderer:
dst = os.path.join(root, template_name.replace("/", os.sep))
write_to(rendered, dst)
def __render(self, template):
def __render(self, template: jinja2.Template) -> str:
try:
return template.render(**self.config)
except jinja2.exceptions.UndefinedError as e:
@ -169,7 +171,7 @@ class Renderer:
)
def save(root, config):
def save(root: str, config: Dict[str, Any]) -> None:
"""
Save the full environment, including version information.
"""
@ -195,7 +197,7 @@ def save(root, config):
fmt.echo_info("Environment generated in {}".format(base_dir(root)))
def upgrade_obsolete(root):
def upgrade_obsolete(root: str) -> None:
# tutor.conf was renamed to _tutor.conf in order to be the first config file loaded
# by nginx
nginx_tutor_conf = pathjoin(root, "apps", "nginx", "tutor.conf")
@ -203,7 +205,9 @@ def upgrade_obsolete(root):
os.remove(nginx_tutor_conf)
def save_plugin_templates(plugin, root, config):
def save_plugin_templates(
plugin: plugins.BasePlugin, root: str, config: Dict[str, Any]
) -> None:
"""
Save plugin templates to plugins/<plugin name>/*.
Only the "apps" and "build" subfolders are rendered.
@ -214,7 +218,7 @@ def save_plugin_templates(plugin, root, config):
save_all_from(subdir_path, plugins_root, config)
def save_all_from(prefix, root, config):
def save_all_from(prefix: str, root: str, config: Dict[str, Any]) -> None:
"""
Render the templates that start with `prefix` and store them with the same
hierarchy at `root`. Here, `prefix` can be the result of os.path.join(...).
@ -223,23 +227,20 @@ def save_all_from(prefix, root, config):
renderer.render_all_to(root, prefix.replace(os.sep, "/"))
def write_to(content, path):
def write_to(content: Union[str, bytes], path: str) -> None:
"""
Write some content to a path. Content can be either str or bytes.
"""
open_kwargs = {"mode": "w"}
if isinstance(content, bytes):
open_kwargs["mode"] += "b"
else:
# Make files readable by Docker on Windows
open_kwargs["encoding"] = "utf8"
open_kwargs["newline"] = "\n"
utils.ensure_file_directory_exists(path)
with open(path, **open_kwargs) as of:
of.write(content)
if isinstance(content, bytes):
with open(path, mode="wb") as of_binary:
of_binary.write(content)
else:
with open(path, mode="w", encoding="utf8", newline="\n") as of_text:
of_text.write(content)
def render_file(config, *path):
def render_file(config: Dict[str, Any], *path: str) -> Union[str, bytes]:
"""
Return the rendered contents of a template.
"""
@ -248,7 +249,7 @@ def render_file(config, *path):
return renderer.render_template(template_name)
def render_dict(config):
def render_dict(config: Dict[str, Any]) -> None:
"""
Render the values from the dict. This is useful for rendering the default
values from config.yml.
@ -266,13 +267,13 @@ def render_dict(config):
config[k] = v
def render_unknown(config, value):
def render_unknown(config: Dict[str, Any], value: Any) -> Any:
if isinstance(value, str):
return render_str(config, value)
return value
def render_str(config, text):
def render_str(config: Dict[str, Any], text: str) -> str:
"""
Args:
text (str)
@ -284,7 +285,7 @@ def render_str(config, text):
return Renderer.instance(config).render_str(text)
def check_is_up_to_date(root):
def check_is_up_to_date(root: str) -> None:
if not is_up_to_date(root):
message = (
"The current environment stored at {} is not up-to-date: it is at "
@ -298,14 +299,14 @@ def check_is_up_to_date(root):
)
def is_up_to_date(root):
def is_up_to_date(root: str) -> bool:
"""
Check if the currently rendered version is equal to the current tutor version.
"""
return current_version(root) == __version__
def needs_major_upgrade(root):
def needs_major_upgrade(root: str) -> bool:
"""
Return the current version as a tuple of int. E.g: (1, 0, 2).
"""
@ -314,7 +315,7 @@ def needs_major_upgrade(root):
return 0 < current < required
def current_release(root):
def current_release(root: str) -> str:
"""
Return the name of the current Open edX release.
"""
@ -323,7 +324,7 @@ def current_release(root):
]
def current_version(root):
def current_version(root: str) -> str:
"""
Return the current environment version. If the current environment has no version,
return "0.0.0".
@ -334,7 +335,7 @@ def current_version(root):
return open(path).read().strip()
def read_template_file(*path):
def read_template_file(*path: str) -> str:
"""
Read raw content of template located at `path`.
"""
@ -343,40 +344,40 @@ def read_template_file(*path):
return fi.read()
def is_binary_file(path):
def is_binary_file(path: str) -> bool:
ext = os.path.splitext(path)[1]
return ext in BIN_FILE_EXTENSIONS
def template_path(*path, templates_root=TEMPLATES_ROOT):
def template_path(*path: str, templates_root: str = TEMPLATES_ROOT) -> str:
"""
Return the template file's absolute path.
"""
return os.path.join(templates_root, *path)
def data_path(root, *path):
def data_path(root: str, *path: str) -> str:
"""
Return the file's absolute path inside the data directory.
"""
return os.path.join(root_dir(root), "data", *path)
def pathjoin(root, *path):
def pathjoin(root: str, *path: str) -> str:
"""
Return the file's absolute path inside the environment.
"""
return os.path.join(base_dir(root), *path)
def base_dir(root):
def base_dir(root: str) -> str:
"""
Return the environment base directory.
"""
return os.path.join(root_dir(root), "env")
def root_dir(root):
def root_dir(root: str) -> str:
"""
Return the project root directory.
"""

20
tutor/fmt.py
View File

@ -3,7 +3,7 @@ import click
STDOUT = None
def title(text):
def title(text: str) -> str:
indent = 8
separator = "=" * (len(text) + 2 * indent)
message = "{separator}\n{indent}{text}\n{separator}".format(
@ -12,37 +12,37 @@ def title(text):
return click.style(message, fg="green")
def echo_info(text):
def echo_info(text: str) -> None:
echo(info(text))
def info(text):
def info(text: str) -> str:
return click.style(text, fg="blue")
def error(text):
def error(text: str) -> str:
return click.style(text, fg="red")
def echo_error(text):
def echo_error(text: str) -> None:
echo(error(text), err=True)
def command(text):
def command(text: str) -> str:
return click.style(text, fg="magenta")
def question(text):
def question(text: str) -> str:
return click.style(text, fg="yellow")
def echo_alert(text):
def echo_alert(text: str) -> None:
echo_error(alert(text))
def alert(text):
def alert(text: str) -> str:
return click.style("⚠️ " + text, fg="yellow", bold=True)
def echo(text, err=False):
def echo(text: str, err: bool = False) -> None:
click.echo(text, file=STDOUT, err=err)

10
tutor/images.py
View File

@ -1,21 +1,23 @@
from typing import Any, Dict
from . import fmt
from . import utils
def get_tag(config, name):
def get_tag(config: Dict[str, Any], name: str) -> Any:
return config["DOCKER_IMAGE_" + name.upper().replace("-", "_")]
def build(path, tag, *args):
def build(path: str, tag: str, *args: str) -> None:
fmt.echo_info("Building image {}".format(tag))
utils.docker("build", "-t", tag, *args, path)
def pull(tag):
def pull(tag: str) -> None:
fmt.echo_info("Pulling image {}".format(tag))
utils.docker("pull", tag)
def push(tag):
def push(tag: str) -> None:
fmt.echo_info("Pushing image {}".format(tag))
utils.docker("push", tag)

25
tutor/interactive.py
View File

@ -1,3 +1,4 @@
from typing import Any, Dict, List, Tuple
import click
from . import config as tutor_config
@ -7,7 +8,7 @@ from . import fmt
from .__about__ import __version__
def update(root, interactive=True):
def update(root: str, interactive: bool = True) -> Dict[str, Any]:
"""
Load and save the configuration.
"""
@ -17,7 +18,9 @@ def update(root, interactive=True):
return config
def load_all(root, interactive=True):
def load_all(
root: str, interactive: bool = True
) -> Tuple[Dict[str, Any], Dict[str, Any]]:
"""
Load configuration and interactively ask questions to collect param values from the user.
"""
@ -27,7 +30,7 @@ def load_all(root, interactive=True):
return config, defaults
def ask_questions(config, defaults):
def ask_questions(config: Dict[str, Any], defaults: Dict[str, Any]) -> None:
run_for_prod = config.get("LMS_HOST") != "local.overhang.io"
run_for_prod = click.confirm(
fmt.question(
@ -156,21 +159,31 @@ def ask_questions(config, defaults):
)
def ask(question, key, config, defaults):
def ask(
question: str, key: str, config: Dict[str, Any], defaults: Dict[str, Any]
) -> None:
default = env.render_str(config, config.get(key, defaults[key]))
config[key] = click.prompt(
fmt.question(question), prompt_suffix=" ", default=default, show_default=True
)
def ask_bool(question, key, config, defaults):
def ask_bool(
question: str, key: str, config: Dict[str, Any], defaults: Dict[str, Any]
) -> None:
default = config.get(key, defaults[key])
config[key] = click.confirm(
fmt.question(question), prompt_suffix=" ", default=default
)
def ask_choice(question, key, config, defaults, choices):
def ask_choice(
question: str,
key: str,
config: Dict[str, Any],
defaults: Dict[str, Any],
choices: List[str],
) -> None:
default = config.get(key, defaults[key])
answer = click.prompt(
fmt.question(question),

33
tutor/jobs.py
View File

@ -1,3 +1,5 @@
from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
from . import env
from . import fmt
from . import plugins
@ -9,25 +11,30 @@ echo "Loading settings $DJANGO_SETTINGS_MODULE"
class BaseJobRunner:
def __init__(self, root, config):
def __init__(self, root: str, config: Dict[str, Any]):
self.root = root
self.config = config
def run_job_from_template(self, service, *path):
def run_job_from_template(self, service: str, *path: str) -> None:
command = self.render(*path)
self.run_job(service, command)
def render(self, *path):
return env.render_file(self.config, *path).strip()
def render(self, *path: str) -> str:
rendered = env.render_file(self.config, *path).strip()
if isinstance(rendered, bytes):
raise TypeError("Cannot load job from binary file")
return rendered
def run_job(self, service, command):
def run_job(self, service: str, command: str) -> int:
raise NotImplementedError
def iter_plugin_hooks(self, hook):
def iter_plugin_hooks(
self, hook: str
) -> Iterator[Tuple[str, Union[Dict[str, str], List[str]]]]:
yield from plugins.iter_hooks(self.config, hook)
def initialise(runner, limit_to=None):
def initialise(runner: BaseJobRunner, limit_to: Optional[str] = None) -> None:
fmt.echo_info("Initialising all services...")
if limit_to is None or limit_to == "mysql":
runner.run_job_from_template("mysql", "hooks", "mysql", "init")
@ -60,7 +67,13 @@ def initialise(runner, limit_to=None):
fmt.echo_info("All services initialised.")
def create_user_command(superuser, staff, username, email, password=None):
def create_user_command(
superuser: str,
staff: bool,
username: str,
email: str,
password: Optional[str] = None,
) -> str:
command = BASE_OPENEDX_COMMAND
opts = ""
@ -86,11 +99,11 @@ u.save()"
return command.format(opts=opts, username=username, email=email, password=password)
def import_demo_course(runner):
def import_demo_course(runner: BaseJobRunner) -> None:
runner.run_job_from_template("cms", "hooks", "cms", "importdemocourse")
def set_theme(theme_name, domain_name, runner):
def set_theme(theme_name: str, domain_name: str, runner: BaseJobRunner) -> None:
command = BASE_OPENEDX_COMMAND
command += """
echo "Assigning theme {theme_name} to {domain_name}..."

119
tutor/plugins.py
View File

@ -3,9 +3,10 @@ from copy import deepcopy
from glob import glob
import importlib
import os
import pkg_resources
from typing import cast, Any, Dict, Iterator, List, Optional, Tuple, Type, Union
import appdirs
import pkg_resources
from . import exceptions
from . import fmt
@ -47,41 +48,50 @@ class BasePlugin:
`command` (click.Command): if a plugin exposes a `command` attribute, users will be able to run it from the command line as `tutor pluginname`.
"""
INSTALLED = []
INSTALLED: List["BasePlugin"] = []
_IS_LOADED = False
def __init__(self, name, obj):
def __init__(self, name: str, obj: Any) -> None:
self.name = name
self.config = get_callable_attr(obj, "config", {})
self.patches = get_callable_attr(obj, "patches", default={})
self.hooks = get_callable_attr(obj, "hooks", default={})
self.templates_root = get_callable_attr(obj, "templates", default=None)
self.config = cast(
Dict[str, Dict[str, Any]], get_callable_attr(obj, "config", {})
)
self.patches = cast(
Dict[str, str], get_callable_attr(obj, "patches", default={})
)
self.hooks = cast(
Dict[str, Union[Dict[str, str], List[str]]],
get_callable_attr(obj, "hooks", default={}),
)
self.templates_root = cast(
Optional[str], get_callable_attr(obj, "templates", default=None)
)
self.command = getattr(obj, "command", None)
def config_key(self, key):
def config_key(self, key: str) -> str:
"""
Config keys in the "add" and "defaults" dicts should be prefixed by the plugin name, in uppercase.
"""
return self.name.upper() + "_" + key
@property
def config_add(self):
def config_add(self) -> Dict[str, Any]:
return self.config.get("add", {})
@property
def config_set(self):
def config_set(self) -> Dict[str, Any]:
return self.config.get("set", {})
@property
def config_defaults(self):
def config_defaults(self) -> Dict[str, Any]:
return self.config.get("defaults", {})
@property
def version(self):
def version(self) -> str:
raise NotImplementedError
@classmethod
def iter_installed(cls):
def iter_installed(cls) -> Iterator["BasePlugin"]:
if not cls._IS_LOADED:
for plugin in cls.iter_load():
cls.INSTALLED.append(plugin)
@ -89,7 +99,7 @@ class BasePlugin:
yield from cls.INSTALLED
@classmethod
def iter_load(cls):
def iter_load(cls) -> Iterator["BasePlugin"]:
raise NotImplementedError
@ -103,16 +113,18 @@ class EntrypointPlugin(BasePlugin):
ENTRYPOINT = "tutor.plugin.v0"
def __init__(self, entrypoint):
def __init__(self, entrypoint: pkg_resources.EntryPoint) -> None:
super().__init__(entrypoint.name, entrypoint.load())
self.entrypoint = entrypoint
@property
def version(self):
def version(self) -> str:
if not self.entrypoint.dist:
return "0.0.0"
return self.entrypoint.dist.version
@classmethod
def iter_load(cls):
def iter_load(cls) -> Iterator["EntrypointPlugin"]:
for entrypoint in pkg_resources.iter_entry_points(cls.ENTRYPOINT):
yield cls(entrypoint)
@ -124,21 +136,24 @@ class OfficialPlugin(BasePlugin):
"""
@classmethod
def load(cls, name):
def load(cls, name: str) -> BasePlugin:
plugin = cls(name)
cls.INSTALLED.append(plugin)
return plugin
def __init__(self, name):
def __init__(self, name: str):
self.module = importlib.import_module("tutor{}.plugin".format(name))
super().__init__(name, self.module)
@property
def version(self):
return self.module.__version__
def version(self) -> str:
version = getattr(self.module, "__version__")
if not isinstance(version, str):
raise TypeError("OfficialPlugin __version__ must be 'str'")
return version
@classmethod
def iter_load(cls):
def iter_load(cls) -> Iterator[BasePlugin]:
yield from []
@ -148,18 +163,20 @@ class DictPlugin(BasePlugin):
os.environ.get(ROOT_ENV_VAR_NAME, "")
) or appdirs.user_data_dir(appname="tutor-plugins")
def __init__(self, data):
Module = namedtuple("Module", data.keys())
obj = Module(**data)
def __init__(self, data: Dict[str, Any]):
Module = namedtuple("Module", data.keys()) # type: ignore
obj = Module(**data) # type: ignore
super().__init__(data["name"], obj)
self._version = data["version"]
@property
def version(self):
def version(self) -> str:
if not isinstance(self._version, str):
raise TypeError("DictPlugin.__version__ must be str")
return self._version
@classmethod
def iter_load(cls):
def iter_load(cls) -> Iterator[BasePlugin]:
for path in glob(os.path.join(cls.ROOT, "*.yml")):
with open(path) as f:
data = serialize.load(f)
@ -176,13 +193,17 @@ class DictPlugin(BasePlugin):
class Plugins:
PLUGIN_CLASSES = [OfficialPlugin, EntrypointPlugin, DictPlugin]
PLUGIN_CLASSES: List[Type[BasePlugin]] = [
OfficialPlugin,
EntrypointPlugin,
DictPlugin,
]
def __init__(self, config):
def __init__(self, config: Dict[str, Any]):
self.config = deepcopy(config)
self.patches = {}
self.hooks = {}
self.template_roots = {}
self.patches: Dict[str, Dict[str, str]] = {}
self.hooks: Dict[str, Dict[str, Union[Dict[str, str], List[str]]]] = {}
self.template_roots: Dict[str, str] = {}
for plugin in self.iter_enabled():
for patch_name, content in plugin.patches.items():
@ -196,12 +217,12 @@ class Plugins:
self.hooks[hook_name][plugin.name] = services
@classmethod
def clear(cls):
def clear(cls) -> None:
for PluginClass in cls.PLUGIN_CLASSES:
PluginClass.INSTALLED.clear()
@classmethod
def iter_installed(cls):
def iter_installed(cls) -> Iterator[BasePlugin]:
"""
Iterate on all installed plugins. Plugins are deduplicated by name. The list of installed plugins is cached to
prevent too many re-computations, which happens a lot.
@ -213,40 +234,44 @@ class Plugins:
installed_plugin_names.add(plugin.name)
yield plugin
def iter_enabled(self):
def iter_enabled(self) -> Iterator[BasePlugin]:
for plugin in self.iter_installed():
if is_enabled(self.config, plugin.name):
yield plugin
def iter_patches(self, name):
def iter_patches(self, name: str) -> Iterator[Tuple[str, str]]:
plugin_patches = self.patches.get(name, {})
plugins = sorted(plugin_patches.keys())
for plugin in plugins:
yield plugin, plugin_patches[plugin]
def iter_hooks(self, hook_name):
def iter_hooks(
self, hook_name: str
) -> Iterator[Tuple[str, Union[Dict[str, str], List[str]]]]:
yield from self.hooks.get(hook_name, {}).items()
def get_callable_attr(plugin, attr_name, default=None):
def get_callable_attr(
plugin: Any, attr_name: str, default: Optional[Any] = None
) -> Optional[Any]:
attr = getattr(plugin, attr_name, default)
if callable(attr):
attr = attr()
return attr
def is_installed(name):
def is_installed(name: str) -> bool:
for plugin in iter_installed():
if name == plugin.name:
return True
return False
def iter_installed():
def iter_installed() -> Iterator[BasePlugin]:
yield from Plugins.iter_installed()
def enable(config, name):
def enable(config: Dict[str, Any], name: str) -> None:
if not is_installed(name):
raise exceptions.TutorError("plugin '{}' is not installed.".format(name))
if is_enabled(config, name):
@ -257,7 +282,7 @@ def enable(config, name):
config[CONFIG_KEY].sort()
def disable(config, name):
def disable(config: Dict[str, Any], name: str) -> None:
fmt.echo_info("Disabling plugin {}...".format(name))
for plugin in Plugins(config).iter_enabled():
if name == plugin.name:
@ -271,17 +296,19 @@ def disable(config, name):
fmt.echo_info(" Plugin disabled")
def iter_enabled(config):
def iter_enabled(config: Dict[str, Any]) -> Iterator[BasePlugin]:
yield from Plugins(config).iter_enabled()
def is_enabled(config, name):
def is_enabled(config: Dict[str, Any], name: str) -> bool:
return name in config.get(CONFIG_KEY, [])
def iter_patches(config, name):
def iter_patches(config: Dict[str, str], name: str) -> Iterator[Tuple[str, str]]:
yield from Plugins(config).iter_patches(name)
def iter_hooks(config, hook_name):
def iter_hooks(
config: Dict[str, Any], hook_name: str
) -> Iterator[Tuple[str, Union[Dict[str, str], List[str]]]]:
yield from Plugins(config).iter_hooks(hook_name)

19
tutor/serialize.py
View File

@ -1,28 +1,27 @@
import re
from typing import cast, Any, Dict, IO, Iterator, Tuple, Union
import yaml
from _io import TextIOWrapper
from yaml.parser import ParserError
from yaml.scanner import ScannerError
import click
def load(stream):
return yaml.load(stream, Loader=yaml.SafeLoader)
def load(stream: Union[str, IO[str]]) -> Dict[str, str]:
return cast(Dict[str, str], yaml.load(stream, Loader=yaml.SafeLoader))
def load_all(stream):
def load_all(stream: str) -> Iterator[Any]:
return yaml.load_all(stream, Loader=yaml.SafeLoader)
def dump(content, fileobj):
def dump(content: Dict[str, str], fileobj: TextIOWrapper) -> None:
yaml.dump(content, stream=fileobj, default_flow_style=False)
def dumps(content):
return yaml.dump(content, stream=None, default_flow_style=False)
def parse(v):
def parse(v: Union[str, IO[str]]) -> Any:
"""
Parse a yaml-formatted string.
"""
@ -37,7 +36,7 @@ class YamlParamType(click.ParamType):
name = "yaml"
PARAM_REGEXP = r"(?P<key>[a-zA-Z0-9_-]+)=(?P<value>.*)"
def convert(self, value, param, ctx):
def convert(self, value: str, param: Any, ctx: Any) -> Tuple[str, Any]:
match = re.match(self.PARAM_REGEXP, value)
if not match:
self.fail("'{}' is not of the form 'key=value'.".format(value), param, ctx)

59
tutor/utils.py
View File

@ -7,16 +7,17 @@ import string
import struct
import subprocess
import sys
from typing import List, Tuple
import click
from Crypto.PublicKey import RSA
from Crypto.Protocol.KDF import bcrypt, bcrypt_check
from Crypto.PublicKey import RSA
from Crypto.PublicKey.RSA import RsaKey
from . import exceptions
from . import fmt
from . import exceptions, fmt
def encrypt(text):
def encrypt(text: str) -> str:
"""
Encrypt some textual content with bcrypt.
https://pycryptodome.readthedocs.io/en/latest/src/protocol/kdf.html#bcrypt
@ -26,7 +27,7 @@ def encrypt(text):
return bcrypt(text.encode(), 12).decode()
def verify_encrypted(encrypted, text):
def verify_encrypted(encrypted: str, text: str) -> bool:
"""
Return True/False if the encrypted content corresponds to the unencrypted text.
"""
@ -37,7 +38,7 @@ def verify_encrypted(encrypted, text):
return False
def ensure_file_directory_exists(path):
def ensure_file_directory_exists(path: str) -> None:
"""
Create file's base directory if it does not exist.
"""
@ -46,17 +47,17 @@ def ensure_file_directory_exists(path):
os.makedirs(directory)
def random_string(length):
def random_string(length: int) -> str:
return "".join(
[random.choice(string.ascii_letters + string.digits) for _ in range(length)]
)
def list_if(services):
def list_if(services: List[Tuple[str, bool]]) -> str:
return json.dumps([service[0] for service in services if service[1]])
def common_domain(d1, d2):
def common_domain(d1: str, d2: str) -> str:
"""
Return the common domain between two domain names.
@ -73,7 +74,7 @@ def common_domain(d1, d2):
return ".".join(common[::-1])
def reverse_host(domain):
def reverse_host(domain: str) -> str:
"""
Return the reverse domain name, java-style.
@ -82,7 +83,7 @@ def reverse_host(domain):
return ".".join(domain.split(".")[::-1])
def rsa_private_key(bits=2048):
def rsa_private_key(bits: int = 2048) -> str:
"""
Export an RSA private key in PEM format.
"""
@ -90,20 +91,20 @@ def rsa_private_key(bits=2048):
return key.export_key().decode()
def rsa_import_key(key):
def rsa_import_key(key: str) -> RsaKey:
"""
Import PEM-formatted RSA key and return the corresponding object.
"""
return RSA.import_key(key.encode())
def long_to_base64(n):
def long_to_base64(n: int) -> str:
"""
Borrowed from jwkest.__init__
"""
def long2intarr(long_int):
_bytes = []
def long2intarr(long_int: int) -> List[int]:
_bytes: List[int] = []
while long_int:
long_int, r = divmod(long_int, 256)
_bytes.insert(0, r)
@ -117,16 +118,7 @@ def long_to_base64(n):
return s.decode("ascii")
def walk_files(path):
"""
Iterate on file paths located in directory.
"""
for dirpath, _, filenames in os.walk(path):
for filename in filenames:
yield os.path.join(dirpath, filename)
def is_root():
def is_root() -> bool:
"""
Check whether tutor is being run as root/sudo.
"""
@ -136,7 +128,7 @@ def is_root():
return get_user_id() == 0
def get_user_id():
def get_user_id() -> int:
"""
Portable way to get user ID. Note: I have no idea if it actually works on windows...
"""
@ -146,14 +138,14 @@ def get_user_id():
return os.getuid()
def docker_run(*command):
def docker_run(*command: str) -> int:
args = ["run", "--rm"]
if is_a_tty():
args.append("-it")
return docker(*args, *command)
def docker(*command):
def docker(*command: str) -> int:
if shutil.which("docker") is None:
raise exceptions.TutorError(
"docker is not installed. Please follow instructions from https://docs.docker.com/install/"
@ -161,7 +153,7 @@ def docker(*command):
return execute("docker", *command)
def docker_compose(*command):
def docker_compose(*command: str) -> int:
if shutil.which("docker-compose") is None:
raise exceptions.TutorError(
"docker-compose is not installed. Please follow instructions from https://docs.docker.com/compose/install/"
@ -169,7 +161,7 @@ def docker_compose(*command):
return execute("docker-compose", *command)
def kubectl(*command):
def kubectl(*command: str) -> int:
if shutil.which("kubectl") is None:
raise exceptions.TutorError(
"kubectl is not installed. Please follow instructions from https://kubernetes.io/docs/tasks/tools/install-kubectl/"
@ -177,7 +169,7 @@ def kubectl(*command):
return execute("kubectl", *command)
def is_a_tty():
def is_a_tty() -> bool:
"""
Return True if stdin is able to allocate a tty. Tty allocation sometimes cannot be
enabled, for instance in cron jobs
@ -185,7 +177,7 @@ def is_a_tty():
return os.isatty(sys.stdin.fileno())
def execute(*command):
def execute(*command: str) -> int:
click.echo(fmt.command(" ".join(command)))
with subprocess.Popen(command) as p:
try:
@ -204,9 +196,10 @@ def execute(*command):
raise exceptions.TutorError(
"Command failed with status {}: {}".format(result, " ".join(command))
)
return result
def check_output(*command):
def check_output(*command: str) -> bytes:
click.echo(fmt.command(" ".join(command)))
try:
return subprocess.check_output(command)