diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f99421..0f66a06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ Note: Breaking changes between versions are indicated by "💥". ## Unreleased +- [Improvement] Annotate types all over the Tutor code base. - [Bugfix] Fix parsing of YAML CLI arguments that include equal "=" signs. - [Bugfix] Fix minor edge case in `long_to_base64` utility function. - [Improvement] Add openedx patches to add settings during build process. diff --git a/Makefile b/Makefile index 077aba0..d547630 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ build-pythonpackage: ## Build a python package ready to upload to pypi push-pythonpackage: ## Push python packages to pypi twine upload --skip-existing dist/tutor-*.tar.gz -test: test-lint test-unit test-format test-pythonpackage ## Run all tests by decreasing order or priority +test: test-lint test-unit test-types test-format test-pythonpackage ## Run all tests by decreasing order or priority test-format: ## Run code formatting tests black --check --diff $(BLACK_OPTS) @@ -35,6 +35,9 @@ test-lint: ## Run code linting tests test-unit: ## Run unit tests python -m unittest discover tests +test-types: ## Check type definitions + mypy --exclude=templates --ignore-missing-imports --strict tutor/ tests/ + test-pythonpackage: build-pythonpackage ## Test that package can be uploaded to pypi twine check dist/tutor-openedx-$(shell make version).tar.gz diff --git a/requirements/base.in b/requirements/base.in index 63afad5..31e1aac 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -1,6 +1,7 @@ appdirs click>=7.0 click_repl +mypy pycryptodome jinja2>=2.9 kubernetes diff --git a/requirements/base.txt b/requirements/base.txt index d1ed4a0..ad13178 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -30,6 +30,10 @@ kubernetes==12.0.1 # via -r requirements/base.in markupsafe==1.1.1 # via jinja2 +mypy-extensions==0.4.3 + # via mypy +mypy==0.812 + # via -r requirements/base.in oauthlib==3.1.0 # via requests-oauthlib prompt-toolkit==3.0.14 @@ -63,6 +67,10 @@ six==1.15.0 # kubernetes # python-dateutil # websocket-client +typed-ast==1.4.2 + # via mypy +typing-extensions==3.7.4.3 + # via mypy urllib3==1.25.11 # via # -r requirements/base.in diff --git a/requirements/dev.txt b/requirements/dev.txt index 80cd7f0..b29e740 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -53,6 +53,11 @@ idna==2.10 # via # -r requirements/base.txt # requests +importlib-metadata==3.7.0 + # via + # keyring + # pyinstaller + # twine isort==5.7.0 # via pylint jeepney==0.6.0 @@ -74,7 +79,12 @@ markupsafe==1.1.1 mccabe==0.6.1 # via pylint mypy-extensions==0.4.3 - # via black + # via + # -r requirements/base.txt + # black + # mypy +mypy==0.812 + # via -r requirements/base.txt oauthlib==3.1.0 # via # -r requirements/base.txt @@ -167,9 +177,17 @@ tqdm==4.56.1 twine==3.3.0 # via -r requirements/dev.in typed-ast==1.4.2 - # via black + # via + # -r requirements/base.txt + # astroid + # black + # mypy typing-extensions==3.7.4.3 - # via black + # via + # -r requirements/base.txt + # black + # importlib-metadata + # mypy urllib3==1.25.11 # via # -r requirements/base.txt @@ -187,6 +205,8 @@ websocket-client==0.57.0 # kubernetes wrapt==1.12.1 # via astroid +zipp==3.4.0 + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/requirements/docs.txt b/requirements/docs.txt index b9f94c3..d18e1a5 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -51,6 +51,12 @@ markupsafe==1.1.1 # via # -r requirements/base.txt # jinja2 +mypy-extensions==0.4.3 + # via + # -r requirements/base.txt + # mypy +mypy==0.812 + # via -r requirements/base.txt oauthlib==3.1.0 # via # -r requirements/base.txt @@ -128,6 +134,14 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.4 # via sphinx +typed-ast==1.4.2 + # via + # -r requirements/base.txt + # mypy +typing-extensions==3.7.4.3 + # via + # -r requirements/base.txt + # mypy urllib3==1.25.11 # via # -r requirements/base.txt diff --git a/tests/test_bindmounts.py b/tests/test_bindmounts.py index da2aa0f..35fe1cb 100644 --- a/tests/test_bindmounts.py +++ b/tests/test_bindmounts.py @@ -5,17 +5,17 @@ from tutor.exceptions import TutorError class BindMountsTests(unittest.TestCase): - def test_get_name(self): + def test_get_name(self) -> None: self.assertEqual("venv", bindmounts.get_name("/openedx/venv")) self.assertEqual("venv", bindmounts.get_name("/openedx/venv/")) - def test_get_name_root_folder(self): + def test_get_name_root_folder(self) -> None: with self.assertRaises(TutorError): bindmounts.get_name("/") with self.assertRaises(TutorError): bindmounts.get_name("") - def test_parse_volumes(self): + def test_parse_volumes(self) -> None: volume_args, non_volume_args = bindmounts.parse_volumes( [ "run", diff --git a/tests/test_config.py b/tests/test_config.py index bdd3f41..9fc7911 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,39 +1,33 @@ +from typing import Any, Dict import unittest -import unittest.mock +from unittest.mock import Mock, patch import tempfile from tutor import config as tutor_config -from tutor import env from tutor import interactive class ConfigTests(unittest.TestCase): - def setUp(self): - # This is necessary to avoid cached mocks - env.Renderer.reset() - - def test_version(self): + def test_version(self) -> None: defaults = tutor_config.load_defaults() self.assertNotIn("TUTOR_VERSION", defaults) - def test_merge(self): + def test_merge(self) -> None: config1 = {"x": "y"} config2 = {"x": "z"} tutor_config.merge(config1, config2) self.assertEqual({"x": "y"}, config1) - def test_merge_render(self): - config = {} + def test_merge_render(self) -> None: + config: Dict[str, Any] = {} defaults = tutor_config.load_defaults() - with unittest.mock.patch.object( - tutor_config.utils, "random_string", return_value="abcd" - ): + with patch.object(tutor_config.utils, "random_string", return_value="abcd"): tutor_config.merge(config, defaults) self.assertEqual("abcd", config["MYSQL_ROOT_PASSWORD"]) - @unittest.mock.patch.object(tutor_config.fmt, "echo") - def test_update_twice(self, _): + @patch.object(tutor_config.fmt, "echo") + def test_update_twice(self, _: Mock) -> None: with tempfile.TemporaryDirectory() as root: tutor_config.update(root) config1 = tutor_config.load_user(root) @@ -43,27 +37,27 @@ class ConfigTests(unittest.TestCase): self.assertEqual(config1, config2) - @unittest.mock.patch.object(tutor_config.fmt, "echo") - def test_removed_entry_is_added_on_save(self, _): + @patch.object(tutor_config.fmt, "echo") + def test_removed_entry_is_added_on_save(self, _: Mock) -> None: with tempfile.TemporaryDirectory() as root: - with unittest.mock.patch.object( + with patch.object( tutor_config.utils, "random_string" ) as mock_random_string: mock_random_string.return_value = "abcd" - config1, _ = tutor_config.load_all(root) + config1, _defaults1 = tutor_config.load_all(root) password1 = config1["MYSQL_ROOT_PASSWORD"] config1.pop("MYSQL_ROOT_PASSWORD") tutor_config.save_config_file(root, config1) mock_random_string.return_value = "efgh" - config2, _ = tutor_config.load_all(root) + config2, _defaults2 = tutor_config.load_all(root) password2 = config2["MYSQL_ROOT_PASSWORD"] self.assertEqual("abcd", password1) self.assertEqual("efgh", password2) - def test_interactive_load_all(self): + def test_interactive_load_all(self) -> None: with tempfile.TemporaryDirectory() as rootdir: config, defaults = interactive.load_all(rootdir, interactive=False) @@ -73,7 +67,7 @@ class ConfigTests(unittest.TestCase): self.assertEqual("www.myopenedx.com", defaults["LMS_HOST"]) self.assertEqual("studio.{{ LMS_HOST }}", defaults["CMS_HOST"]) - def test_is_service_activated(self): + def test_is_service_activated(self) -> None: config = {"RUN_SERVICE1": True, "RUN_SERVICE2": False} self.assertTrue(tutor_config.is_service_activated(config, "service1")) diff --git a/tests/test_env.py b/tests/test_env.py index baa424d..c49316f 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -1,7 +1,8 @@ import os import tempfile +from typing import Any, Dict import unittest -import unittest.mock +from unittest.mock import patch, Mock from tutor import config as tutor_config from tutor import env @@ -10,41 +11,38 @@ from tutor import exceptions class EnvTests(unittest.TestCase): - def setUp(self): - env.Renderer.reset() - - def test_walk_templates(self): + def test_walk_templates(self) -> None: renderer = env.Renderer({}, [env.TEMPLATES_ROOT]) templates = list(renderer.walk_templates("local")) self.assertIn("local/docker-compose.yml", templates) - def test_walk_templates_partials_are_ignored(self): + def test_walk_templates_partials_are_ignored(self) -> None: template_name = "apps/openedx/settings/partials/common_all.py" renderer = env.Renderer({}, [env.TEMPLATES_ROOT], ignore_folders=["partials"]) templates = list(renderer.walk_templates("apps")) - self.assertIn(template_name, renderer.environment.loader.list_templates()) + self.assertIn(template_name, renderer.environment.loader.list_templates()) # type: ignore self.assertNotIn(template_name, templates) - def test_is_binary_file(self): + def test_is_binary_file(self) -> None: self.assertTrue(env.is_binary_file("/home/somefile.ico")) - def test_find_os_path(self): + def test_find_os_path(self) -> None: renderer = env.Renderer({}, [env.TEMPLATES_ROOT]) path = renderer.find_os_path("local/docker-compose.yml") self.assertTrue(os.path.exists(path)) - def test_pathjoin(self): + def test_pathjoin(self) -> None: self.assertEqual( "/tmp/env/target/dummy", env.pathjoin("/tmp", "target", "dummy") ) self.assertEqual("/tmp/env/dummy", env.pathjoin("/tmp", "dummy")) - def test_render_str(self): + def test_render_str(self) -> None: self.assertEqual( "hello world", env.render_str({"name": "world"}, "hello {{ name }}") ) - def test_common_domain(self): + def test_common_domain(self) -> None: self.assertEqual( "mydomain.com", env.render_str( @@ -53,64 +51,62 @@ class EnvTests(unittest.TestCase): ), ) - def test_render_str_missing_configuration(self): + def test_render_str_missing_configuration(self) -> None: self.assertRaises(exceptions.TutorError, env.render_str, {}, "hello {{ name }}") - def test_render_file(self): - config = {} + def test_render_file(self) -> None: + config: Dict[str, Any] = {} tutor_config.merge(config, tutor_config.load_defaults()) config["MYSQL_ROOT_PASSWORD"] = "testpassword" rendered = env.render_file(config, "hooks", "mysql", "init") self.assertIn("testpassword", rendered) - @unittest.mock.patch.object(tutor_config.fmt, "echo") - def test_render_file_missing_configuration(self, _): + @patch.object(tutor_config.fmt, "echo") + def test_render_file_missing_configuration(self, _: Mock) -> None: self.assertRaises( exceptions.TutorError, env.render_file, {}, "local", "docker-compose.yml" ) - def test_save_full(self): + def test_save_full(self) -> None: defaults = tutor_config.load_defaults() with tempfile.TemporaryDirectory() as root: config = tutor_config.load_current(root, defaults) tutor_config.merge(config, defaults) - with unittest.mock.patch.object(fmt, "STDOUT"): + with patch.object(fmt, "STDOUT"): env.save(root, config) self.assertTrue( os.path.exists(os.path.join(root, "env", "local", "docker-compose.yml")) ) - def test_save_full_with_https(self): + def test_save_full_with_https(self) -> None: defaults = tutor_config.load_defaults() with tempfile.TemporaryDirectory() as root: config = tutor_config.load_current(root, defaults) tutor_config.merge(config, defaults) config["ENABLE_HTTPS"] = True - with unittest.mock.patch.object(fmt, "STDOUT"): + with patch.object(fmt, "STDOUT"): env.save(root, config) with open(os.path.join(root, "env", "apps", "caddy", "Caddyfile")) as f: self.assertIn("www.myopenedx.com {", f.read()) - def test_patch(self): + def test_patch(self) -> None: patches = {"plugin1": "abcd", "plugin2": "efgh"} - with unittest.mock.patch.object( + with patch.object( env.plugins, "iter_patches", return_value=patches.items() ) as mock_iter_patches: rendered = env.render_str({}, '{{ patch("location") }}') mock_iter_patches.assert_called_once_with({}, "location") self.assertEqual("abcd\nefgh", rendered) - def test_patch_separator_suffix(self): + def test_patch_separator_suffix(self) -> None: patches = {"plugin1": "abcd", "plugin2": "efgh"} - with unittest.mock.patch.object( - env.plugins, "iter_patches", return_value=patches.items() - ): + with patch.object(env.plugins, "iter_patches", return_value=patches.items()): rendered = env.render_str( {}, '{{ patch("location", separator=",\n", suffix=",") }}' ) self.assertEqual("abcd,\nefgh,", rendered) - def test_plugin_templates(self): + def test_plugin_templates(self) -> None: with tempfile.TemporaryDirectory() as plugin_templates: # Create plugin plugin1 = env.plugins.DictPlugin( @@ -132,7 +128,7 @@ class EnvTests(unittest.TestCase): config = {"ID": "abcd"} # Render templates - with unittest.mock.patch.object( + with patch.object( env.plugins, "iter_enabled", return_value=[plugin1], @@ -153,7 +149,7 @@ class EnvTests(unittest.TestCase): with open(dst_rendered) as f: self.assertEqual("Hello my ID is abcd", f.read()) - def test_renderer_is_reset_on_config_change(self): + def test_renderer_is_reset_on_config_change(self) -> None: with tempfile.TemporaryDirectory() as plugin_templates: plugin1 = env.plugins.DictPlugin( {"name": "plugin1", "version": "0", "templates": plugin_templates} @@ -166,10 +162,10 @@ class EnvTests(unittest.TestCase): f.write("some content") # Load env once - config = {"PLUGINS": []} + config: Dict[str, Any] = {"PLUGINS": []} env1 = env.Renderer.instance(config).environment - with unittest.mock.patch.object( + with patch.object( env.plugins, "iter_enabled", return_value=[plugin1], @@ -178,5 +174,5 @@ class EnvTests(unittest.TestCase): config["PLUGINS"].append("myplugin") env2 = env.Renderer.instance(config).environment - self.assertNotIn("plugin1/myplugin.txt", env1.loader.list_templates()) - self.assertIn("plugin1/myplugin.txt", env2.loader.list_templates()) + self.assertNotIn("plugin1/myplugin.txt", env1.loader.list_templates()) # type: ignore + self.assertIn("plugin1/myplugin.txt", env2.loader.list_templates()) # type: ignore diff --git a/tests/test_images.py b/tests/test_images.py index 525dac9..149866d 100644 --- a/tests/test_images.py +++ b/tests/test_images.py @@ -3,7 +3,7 @@ from tutor import images class ImagesTests(unittest.TestCase): - def test_get_tag(self): + def test_get_tag(self) -> None: config = { "DOCKER_IMAGE_OPENEDX": "registry/openedx", "DOCKER_IMAGE_OPENEDX_DEV": "registry/openedxdev", diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 19eb566..3b8e5f5 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1,5 +1,6 @@ +from typing import Any, Dict import unittest -from unittest.mock import patch +from unittest.mock import Mock, patch from tutor import config as tutor_config from tutor import exceptions @@ -8,22 +9,22 @@ from tutor import plugins class PluginsTests(unittest.TestCase): - def setUp(self): + def setUp(self) -> None: plugins.Plugins.clear() @patch.object(plugins.DictPlugin, "iter_installed", return_value=[]) - def test_iter_installed(self, _dict_plugin_iter_installed): - with patch.object(plugins.pkg_resources, "iter_entry_points", return_value=[]): + def test_iter_installed(self, _dict_plugin_iter_installed: Mock) -> None: + with patch.object(plugins.pkg_resources, "iter_entry_points", return_value=[]): # type: ignore self.assertEqual([], list(plugins.iter_installed())) - def test_is_installed(self): + def test_is_installed(self) -> None: self.assertFalse(plugins.is_installed("dummy")) @patch.object(plugins.DictPlugin, "iter_installed", return_value=[]) - def test_official_plugins(self, _dict_plugin_iter_installed): - with patch.object(plugins.importlib, "import_module", return_value=42): + def test_official_plugins(self, _dict_plugin_iter_installed: Mock) -> None: + with patch.object(plugins.importlib, "import_module", return_value=42): # type: ignore plugin1 = plugins.OfficialPlugin.load("plugin1") - with patch.object(plugins.importlib, "import_module", return_value=43): + with patch.object(plugins.importlib, "import_module", return_value=43): # type: ignore plugin2 = plugins.OfficialPlugin.load("plugin2") with patch.object( plugins.EntrypointPlugin, @@ -35,32 +36,32 @@ class PluginsTests(unittest.TestCase): list(plugins.iter_installed()), ) - def test_enable(self): - config = {plugins.CONFIG_KEY: []} + def test_enable(self) -> None: + config: Dict[str, Any] = {plugins.CONFIG_KEY: []} with patch.object(plugins, "is_installed", return_value=True): plugins.enable(config, "plugin2") plugins.enable(config, "plugin1") self.assertEqual(["plugin1", "plugin2"], config[plugins.CONFIG_KEY]) - def test_enable_twice(self): - config = {plugins.CONFIG_KEY: []} + def test_enable_twice(self) -> None: + config: Dict[str, Any] = {plugins.CONFIG_KEY: []} with patch.object(plugins, "is_installed", return_value=True): plugins.enable(config, "plugin1") plugins.enable(config, "plugin1") self.assertEqual(["plugin1"], config[plugins.CONFIG_KEY]) - def test_enable_not_installed_plugin(self): - config = {"PLUGINS": []} + def test_enable_not_installed_plugin(self) -> None: + config: Dict[str, Any] = {"PLUGINS": []} with patch.object(plugins, "is_installed", return_value=False): self.assertRaises(exceptions.TutorError, plugins.enable, config, "plugin1") - def test_disable(self): - config = {"PLUGINS": ["plugin1", "plugin2"]} + def test_disable(self) -> None: + config: Dict[str, Any] = {"PLUGINS": ["plugin1", "plugin2"]} with patch.object(fmt, "STDOUT"): plugins.disable(config, "plugin1") self.assertEqual(["plugin2"], config["PLUGINS"]) - def test_disable_removes_set_config(self): + def test_disable_removes_set_config(self) -> None: with patch.object( plugins.Plugins, "iter_enabled", @@ -80,7 +81,7 @@ class PluginsTests(unittest.TestCase): self.assertEqual([], config["PLUGINS"]) self.assertNotIn("KEY", config) - def test_patches(self): + def test_patches(self) -> None: class plugin1: patches = {"patch1": "Hello {{ ID }}"} @@ -92,7 +93,7 @@ class PluginsTests(unittest.TestCase): patches = list(plugins.iter_patches({}, "patch1")) self.assertEqual([("plugin1", "Hello {{ ID }}")], patches) - def test_plugin_without_patches(self): + def test_plugin_without_patches(self) -> None: with patch.object( plugins.Plugins, "iter_enabled", @@ -101,9 +102,9 @@ class PluginsTests(unittest.TestCase): patches = list(plugins.iter_patches({}, "patch1")) self.assertEqual([], patches) - def test_configure(self): + def test_configure(self) -> None: config = {"ID": "id"} - defaults = {} + defaults: Dict[str, Any] = {} class plugin1: config = { @@ -130,7 +131,7 @@ class PluginsTests(unittest.TestCase): ) self.assertEqual({"PLUGIN1_PARAM4": "value4"}, defaults) - def test_configure_set_does_not_override(self): + def test_configure_set_does_not_override(self) -> None: config = {"ID": "oldid"} class plugin1: @@ -145,8 +146,8 @@ class PluginsTests(unittest.TestCase): self.assertEqual({"ID": "oldid"}, config) - def test_configure_set_random_string(self): - config = {} + def test_configure_set_random_string(self) -> None: + config: Dict[str, Any] = {} class plugin1: config = {"set": {"PARAM1": "{{ 128|random_string }}"}} @@ -159,8 +160,8 @@ class PluginsTests(unittest.TestCase): tutor_config.load_plugins(config, {}) self.assertEqual(128, len(config["PARAM1"])) - def test_configure_default_value_with_previous_definition(self): - config = {} + def test_configure_default_value_with_previous_definition(self) -> None: + config: Dict[str, Any] = {} defaults = {"PARAM1": "value"} class plugin1: @@ -174,8 +175,8 @@ class PluginsTests(unittest.TestCase): tutor_config.load_plugins(config, defaults) self.assertEqual("{{ PARAM1 }}", defaults["PLUGIN1_PARAM2"]) - def test_configure_add_twice(self): - config = {} + def test_configure_add_twice(self) -> None: + config: Dict[str, Any] = {} class plugin1: config = {"add": {"PARAM1": "{{ 10|random_string }}"}} @@ -199,7 +200,7 @@ class PluginsTests(unittest.TestCase): self.assertEqual(10, len(value2)) self.assertEqual(value1, value2) - def test_hooks(self): + def test_hooks(self) -> None: class plugin1: hooks = {"init": ["myclient"]} @@ -212,8 +213,8 @@ class PluginsTests(unittest.TestCase): [("plugin1", ["myclient"])], list(plugins.iter_hooks({}, "init")) ) - def test_plugins_are_updated_on_config_change(self): - config = {"PLUGINS": []} + def test_plugins_are_updated_on_config_change(self) -> None: + config: Dict[str, Any] = {"PLUGINS": []} plugins1 = plugins.Plugins(config) self.assertEqual(0, len(list(plugins1.iter_enabled()))) config["PLUGINS"].append("plugin1") @@ -225,7 +226,7 @@ class PluginsTests(unittest.TestCase): plugins2 = plugins.Plugins(config) self.assertEqual(1, len(list(plugins2.iter_enabled()))) - def test_dict_plugin(self): + def test_dict_plugin(self) -> None: plugin = plugins.DictPlugin( {"name": "myplugin", "config": {"set": {"KEY": "value"}}, "version": "0.1"} ) diff --git a/tests/test_serialize.py b/tests/test_serialize.py index dc5b871..d17b327 100644 --- a/tests/test_serialize.py +++ b/tests/test_serialize.py @@ -6,32 +6,32 @@ from tutor import serialize class SerializeTests(unittest.TestCase): - def test_parse_str(self): + def test_parse_str(self) -> None: self.assertEqual("abcd", serialize.parse("abcd")) - def test_parse_int(self): + def test_parse_int(self) -> None: self.assertEqual(1, serialize.parse("1")) - def test_parse_bool(self): + def test_parse_bool(self) -> None: self.assertEqual(True, serialize.parse("true")) self.assertEqual(False, serialize.parse("false")) - def test_parse_null(self): + def test_parse_null(self) -> None: self.assertIsNone(serialize.parse("null")) - def test_parse_invalid_format(self): + def test_parse_invalid_format(self) -> None: self.assertEqual('["abcd"', serialize.parse('["abcd"')) - def test_parse_list(self): + def test_parse_list(self) -> None: self.assertEqual(["abcd"], serialize.parse('["abcd"]')) - def test_parse_weird_chars(self): + def test_parse_weird_chars(self) -> None: self.assertEqual("*@google.com", serialize.parse("*@google.com")) - def test_parse_empty_string(self): + def test_parse_empty_string(self) -> None: self.assertEqual("", serialize.parse("''")) - def test_yaml_param_type(self): + def test_yaml_param_type(self) -> None: param = serialize.YamlParamType() self.assertEqual(("name", True), param.convert("name=true", "param", {})) self.assertEqual(("name", "abcd"), param.convert("name=abcd", "param", {})) diff --git a/tests/test_utils.py b/tests/test_utils.py index 85f4c2f..d02d723 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -5,7 +5,7 @@ from tutor import utils class UtilsTests(unittest.TestCase): - def test_common_domain(self): + def test_common_domain(self) -> None: self.assertEqual( "domain.com", utils.common_domain("sub1.domain.com", "sub2.domain.com") ) @@ -18,13 +18,13 @@ class UtilsTests(unittest.TestCase): "domain.com", utils.common_domain("sub.domain.com", "ub.domain.com") ) - def test_reverse_host(self): + def test_reverse_host(self) -> None: self.assertEqual("com.google.www", utils.reverse_host("www.google.com")) - def test_list_if(self): + def test_list_if(self) -> None: self.assertEqual('["cms"]', utils.list_if([("lms", False), ("cms", True)])) - def test_encrypt_decrypt(self): + def test_encrypt_decrypt(self) -> None: password = "passw0rd" encrypted1 = utils.encrypt(password) encrypted2 = utils.encrypt(password) @@ -32,7 +32,7 @@ class UtilsTests(unittest.TestCase): self.assertTrue(utils.verify_encrypted(encrypted1, password)) self.assertTrue(utils.verify_encrypted(encrypted2, password)) - def test_long_to_base64(self): + def test_long_to_base64(self) -> None: self.assertEqual( b"\x00", base64.urlsafe_b64decode(utils.long_to_base64(0) + "==") ) diff --git a/tutor/bindmounts.py b/tutor/bindmounts.py index 013922b..56534a5 100644 --- a/tutor/bindmounts.py +++ b/tutor/bindmounts.py @@ -1,12 +1,20 @@ import os +from typing import Any, Callable, Dict, List, Tuple import click +from mypy_extensions import VarArg from .exceptions import TutorError from .utils import get_user_id -def create(root, config, docker_compose_func, service, path): +def create( + root: str, + config: Dict[str, Any], + docker_compose_func: Callable[[str, Dict[str, Any], VarArg(str)], int], + service: str, + path: str, +) -> str: volumes_root_path = get_root_path(root) volume_name = get_name(path) container_volumes_root_path = "/tmp/volumes" @@ -41,12 +49,12 @@ chown -R {user_id} {volumes_path}/{volume_name}""".format( return os.path.join(volumes_root_path, volume_name) -def get_path(root, container_bind_path): +def get_path(root: str, container_bind_path: str) -> str: bind_basename = get_name(container_bind_path) return os.path.join(get_root_path(root), bind_basename) -def get_name(container_bind_path): +def get_name(container_bind_path: str) -> str: # We rstrip slashes, otherwise os.path.basename returns an empty string # We don't use basename here as it will not work on Windows name = container_bind_path.rstrip("/").split("/")[-1] @@ -55,11 +63,11 @@ def get_name(container_bind_path): return name -def get_root_path(root): +def get_root_path(root: str) -> str: return os.path.join(root, "volumes") -def parse_volumes(docker_compose_args): +def parse_volumes(docker_compose_args: List[str]) -> Tuple[List[str], List[str]]: """ Parse `-v/--volume` options from an arbitrary list of arguments. """ @@ -67,7 +75,9 @@ def parse_volumes(docker_compose_args): @click.command(context_settings={"ignore_unknown_options": True}) @click.option("-v", "--volume", "volumes", multiple=True) @click.argument("args", nargs=-1, required=True) - def custom_docker_compose(volumes, args): # pylint: disable=unused-argument + def custom_docker_compose( + volumes: List[str], args: List[str] + ) -> None: # pylint: disable=unused-argument pass if isinstance(docker_compose_args, tuple): diff --git a/tutor/commands/android.py b/tutor/commands/android.py index 280965f..2e4b2a8 100644 --- a/tutor/commands/android.py +++ b/tutor/commands/android.py @@ -1,3 +1,5 @@ +from typing import Dict + import click from .compose import ComposeJobRunner @@ -5,17 +7,18 @@ from .local import docker_compose as local_docker_compose from .. import config as tutor_config from .. import env as tutor_env from .. import fmt +from .context import Context @click.group(help="Build an Android app for your Open edX platform [BETA FEATURE]") -def android(): +def android() -> None: pass @click.command(help="Build the application") @click.argument("mode", type=click.Choice(["debug", "release"])) @click.pass_obj -def build(context, mode): +def build(context: Context, mode: str) -> None: config = tutor_config.load(context.root) docker_run(context.root, build_command(config, mode)) fmt.echo_info( @@ -25,7 +28,7 @@ def build(context, mode): ) -def build_command(config, target): +def build_command(config: Dict[str, str], target: str) -> str: gradle_target = { "debug": "assembleProdDebuggable", "release": "assembleProdRelease", @@ -41,7 +44,7 @@ cp OpenEdXMobile/build/outputs/apk/prod/{apk_folder}/*.apk /openedx/data/""" return command -def docker_run(root, command): +def docker_run(root: str, command: str) -> None: config = tutor_config.load(root) runner = ComposeJobRunner(root, config, local_docker_compose) runner.run_job("android", command) diff --git a/tutor/commands/cli.py b/tutor/commands/cli.py index d6a3bd3..1dd9a92 100755 --- a/tutor/commands/cli.py +++ b/tutor/commands/cli.py @@ -21,7 +21,7 @@ from .. import fmt from .. import utils -def main(): +def main() -> None: try: click_repl.register_repl(cli, name="ui") cli.add_command(images_command) @@ -55,7 +55,7 @@ def main(): help="Root project directory (environment variable: TUTOR_ROOT)", ) @click.pass_context -def cli(context, root): +def cli(context: click.Context, root: str) -> None: if utils.is_root(): fmt.echo_alert( "You are running Tutor as root. This is strongly not recommended. If you are doing this in order to access" @@ -66,9 +66,9 @@ def cli(context, root): @click.command(help="Print this help", name="help") -def print_help(): - with click.Context(cli) as context: - click.echo(cli.get_help(context)) +def print_help() -> None: + context = click.Context(cli) + click.echo(cli.get_help(context)) if __name__ == "__main__": diff --git a/tutor/commands/compose.py b/tutor/commands/compose.py index 561c85c..4540949 100644 --- a/tutor/commands/compose.py +++ b/tutor/commands/compose.py @@ -1,6 +1,8 @@ import os +from typing import Any, Callable, Dict, List import click +from mypy_extensions import VarArg from .. import bindmounts from .. import config as tutor_config @@ -10,14 +12,20 @@ from .. import fmt from .. import jobs from .. import serialize from .. import utils +from .context import Context class ComposeJobRunner(jobs.BaseJobRunner): - def __init__(self, root, config, docker_compose_func): + def __init__( + self, + root: str, + config: Dict[str, Any], + docker_compose_func: Callable[[str, Dict[str, Any], VarArg(str)], int], + ): super().__init__(root, config) self.docker_compose_func = docker_compose_func - def run_job(self, service, command): + def run_job(self, service: str, command: str) -> int: """ Run the "{{ service }}-job" service from local/docker-compose.jobs.yml with the specified command. For backward-compatibility reasons, if the corresponding @@ -28,7 +36,7 @@ class ComposeJobRunner(jobs.BaseJobRunner): job_service_name = "{}-job".format(service) opts = [] if utils.is_a_tty() else ["-T"] if job_service_name in serialize.load(open(jobs_path).read())["services"]: - self.docker_compose_func( + return self.docker_compose_func( self.root, self.config, "-f", @@ -42,44 +50,43 @@ class ComposeJobRunner(jobs.BaseJobRunner): "-c", command, ) - else: - fmt.echo_alert( - ( - "The '{job_service_name}' service does not exist in {jobs_path}. " - "This might be caused by an older plugin. Tutor switched to a job " - "runner model for running one-time commands, such as database" - " initialisation. For the record, this is the command that we are " - "running:\n" - "\n" - " {command}\n" - "\n" - "Old-style job running will be deprecated soon. Please inform " - "your plugin maintainer!" - ).format( - job_service_name=job_service_name, - jobs_path=jobs_path, - command=command.replace("\n", "\n "), - ) - ) - self.docker_compose_func( - self.root, - self.config, - "run", - *opts, - "--rm", - service, - "sh", - "-e", - "-c", - command, + fmt.echo_alert( + ( + "The '{job_service_name}' service does not exist in {jobs_path}. " + "This might be caused by an older plugin. Tutor switched to a job " + "runner model for running one-time commands, such as database" + " initialisation. For the record, this is the command that we are " + "running:\n" + "\n" + " {command}\n" + "\n" + "Old-style job running will be deprecated soon. Please inform " + "your plugin maintainer!" + ).format( + job_service_name=job_service_name, + jobs_path=jobs_path, + command=command.replace("\n", "\n "), ) + ) + return self.docker_compose_func( + self.root, + self.config, + "run", + *opts, + "--rm", + service, + "sh", + "-e", + "-c", + command, + ) @click.command(help="Run all or a selection of configured Open edX services") @click.option("-d", "--detach", is_flag=True, help="Start in daemon mode") @click.argument("services", metavar="service", nargs=-1) @click.pass_obj -def start(context, detach, services): +def start(context: Context, detach: bool, services: List[str]) -> None: command = ["up", "--remove-orphans"] if detach: command.append("-d") @@ -91,7 +98,7 @@ def start(context, detach, services): @click.command(help="Stop a running platform") @click.argument("services", metavar="service", nargs=-1) @click.pass_obj -def stop(context, services): +def stop(context: Context, services: List[str]) -> None: config = tutor_config.load(context.root) context.docker_compose(context.root, config, "stop", *services) @@ -102,9 +109,10 @@ def stop(context, services): ) @click.option("-d", "--detach", is_flag=True, help="Start in daemon mode") @click.argument("services", metavar="service", nargs=-1) -def reboot(detach, services): - stop.callback(services) - start.callback(detach, services) +@click.pass_context +def reboot(context: click.Context, detach: bool, services: List[str]) -> None: + context.invoke(stop, services=services) + context.invoke(start, detach=detach, services=services) @click.command( @@ -116,7 +124,7 @@ fully stop the platform, use the 'reboot' command.""", ) @click.argument("services", metavar="service", nargs=-1) @click.pass_obj -def restart(context, services): +def restart(context: Context, services: List[str]) -> None: config = tutor_config.load(context.root) command = ["restart"] if "all" in services: @@ -136,7 +144,7 @@ def restart(context, services): @click.command(help="Initialise all applications") @click.option("-l", "--limit", help="Limit initialisation to this service or plugin") @click.pass_obj -def init(context, limit): +def init(context: Context, limit: str) -> None: config = tutor_config.load(context.root) runner = ComposeJobRunner(context.root, config, context.docker_compose) jobs.initialise(runner, limit_to=limit) @@ -153,7 +161,9 @@ def init(context, limit): @click.argument("name") @click.argument("email") @click.pass_obj -def createuser(context, superuser, staff, password, name, email): +def createuser( + context: Context, superuser: str, staff: bool, password: str, name: str, email: str +) -> None: config = tutor_config.load(context.root) runner = ComposeJobRunner(context.root, config, context.docker_compose) command = jobs.create_user_command(superuser, staff, name, email, password=password) @@ -166,7 +176,7 @@ def createuser(context, superuser, staff, password, name, email): @click.argument("theme_name") @click.argument("domain_names", metavar="domain_name", nargs=-1) @click.pass_obj -def settheme(context, theme_name, domain_names): +def settheme(context: Context, theme_name: str, domain_names: List[str]) -> None: config = tutor_config.load(context.root) runner = ComposeJobRunner(context.root, config, context.docker_compose) for domain_name in domain_names: @@ -175,7 +185,7 @@ def settheme(context, theme_name, domain_names): @click.command(help="Import the demo course") @click.pass_obj -def importdemocourse(context): +def importdemocourse(context: Context) -> None: config = tutor_config.load(context.root) runner = ComposeJobRunner(context.root, config, context.docker_compose) fmt.echo_info("Importing demo course") @@ -192,11 +202,12 @@ def importdemocourse(context): context_settings={"ignore_unknown_options": True}, ) @click.argument("args", nargs=-1, required=True) -def run(args): +@click.pass_context +def run(context: click.Context, args: List[str]) -> None: extra_args = ["--rm"] if not utils.is_a_tty(): extra_args.append("-T") - dc_command.callback("run", [*extra_args, *args]) + context.invoke(dc_command, command="run", args=[*extra_args, *args]) @click.command( @@ -208,7 +219,7 @@ def run(args): ) @click.argument("path") @click.pass_obj -def bindmount_command(context, service, path): +def bindmount_command(context: Context, service: str, path: str) -> None: config = tutor_config.load(context.root) host_path = bindmounts.create( context.root, config, context.docker_compose, service, path @@ -231,8 +242,9 @@ def bindmount_command(context, service, path): name="exec", ) @click.argument("args", nargs=-1, required=True) -def execute(args): - dc_command.callback("exec", args) +@click.pass_context +def execute(context: click.Context, args: List[str]) -> None: + context.invoke(dc_command, command="exec", args=args) @click.command( @@ -242,14 +254,15 @@ def execute(args): @click.option("-f", "--follow", is_flag=True, help="Follow log output") @click.option("--tail", type=int, help="Number of lines to show from each container") @click.argument("service", nargs=-1) -def logs(follow, tail, service): +@click.pass_context +def logs(context: click.Context, follow: bool, tail: bool, service: str) -> None: args = [] if follow: args.append("--follow") if tail is not None: args += ["--tail", str(tail)] args += service - dc_command.callback("logs", args) + context.invoke(dc_command, command="logs", args=args) @click.command( @@ -264,7 +277,7 @@ def logs(follow, tail, service): @click.argument("command") @click.argument("args", nargs=-1, required=True) @click.pass_obj -def dc_command(context, command, args): +def dc_command(context: Context, command: str, args: List[str]) -> None: config = tutor_config.load(context.root) volumes, non_volume_args = bindmounts.parse_volumes(args) volume_args = [] @@ -286,7 +299,7 @@ def dc_command(context, command, args): ) -def add_commands(command_group): +def add_commands(command_group: click.Group) -> None: command_group.add_command(start) command_group.add_command(stop) command_group.add_command(restart) diff --git a/tutor/commands/config.py b/tutor/commands/config.py index 167af66..17b9c62 100644 --- a/tutor/commands/config.py +++ b/tutor/commands/config.py @@ -1,3 +1,5 @@ +from typing import Any, Dict, List + import click from .. import config as tutor_config @@ -6,6 +8,7 @@ from .. import exceptions from .. import fmt from .. import interactive as interactive_config from .. import serialize +from .context import Context @click.group( @@ -13,7 +16,7 @@ from .. import serialize short_help="Configure Open edX", help="""Configure Open edX and store configuration values in $TUTOR_ROOT/config.yml""", ) -def config_command(): +def config_command() -> None: pass @@ -36,7 +39,9 @@ def config_command(): help="Remove a configuration value (can be used multiple times)", ) @click.pass_obj -def save(context, interactive, set_vars, unset_vars): +def save( + context: Context, interactive: bool, set_vars: Dict[str, Any], unset_vars: List[str] +) -> None: config, defaults = interactive_config.load_all( context.root, interactive=interactive ) @@ -61,7 +66,7 @@ def save(context, interactive, set_vars, unset_vars): @click.argument("src", type=click.Path(exists=True, resolve_path=True)) @click.argument("dst") @click.pass_obj -def render(context, extra_configs, src, dst): +def render(context: Context, extra_configs: List[str], src: str, dst: str) -> None: config = tutor_config.load(context.root) for extra_config in extra_configs: tutor_config.merge( @@ -75,14 +80,14 @@ def render(context, extra_configs, src, dst): @click.command(help="Print the project root") @click.pass_obj -def printroot(context): +def printroot(context: Context) -> None: click.echo(context.root) @click.command(help="Print a configuration value") @click.argument("key") @click.pass_obj -def printvalue(context, key): +def printvalue(context: Context, key: str) -> None: config = tutor_config.load(context.root) try: # Note that this will incorrectly print None values diff --git a/tutor/commands/context.py b/tutor/commands/context.py index fea60b6..38530fb 100644 --- a/tutor/commands/context.py +++ b/tutor/commands/context.py @@ -1,8 +1,17 @@ +from typing import Any, Dict + + +def unimplemented_docker_compose( + root: str, config: Dict[str, Any], *command: str +) -> int: + raise NotImplementedError + + # pylint: disable=too-few-public-methods class Context: - def __init__(self, root): + def __init__(self, root: str) -> None: self.root = root + self.docker_compose_func = unimplemented_docker_compose - @staticmethod - def docker_compose(root, config, *command): - raise NotImplementedError + def docker_compose(self, root: str, config: Dict[str, Any], *command: str) -> int: + return self.docker_compose_func(root, config, *command) diff --git a/tutor/commands/dev.py b/tutor/commands/dev.py index 61f5b80..3935a8e 100644 --- a/tutor/commands/dev.py +++ b/tutor/commands/dev.py @@ -1,15 +1,17 @@ import os +from typing import Any, Dict, List import click -from . import compose from .. import config as tutor_config from .. import env as tutor_env from .. import fmt from .. import utils +from . import compose +from .context import Context -def docker_compose(root, config, *command): +def docker_compose(root: str, config: Dict[str, Any], *command: str) -> int: """ Run docker-compose with dev arguments. """ @@ -27,15 +29,15 @@ def docker_compose(root, config, *command): return utils.docker_compose( *args, "--project-name", - config["DEV_PROJECT_NAME"], + str(config["DEV_PROJECT_NAME"]), *command, ) @click.group(help="Run Open edX locally with development settings") @click.pass_obj -def dev(context): - context.docker_compose = docker_compose +def dev(context: Context) -> None: + context.docker_compose_func = docker_compose @click.command( @@ -44,9 +46,9 @@ def dev(context): ) @click.argument("options", nargs=-1, required=False) @click.argument("service") -@click.pass_obj -def runserver(context, options, service): - config = tutor_config.load(context.root) +@click.pass_context +def runserver(context: click.Context, options: List[str], service: str) -> None: + config = tutor_config.load(context.obj.root) if service in ["lms", "cms"]: port = 8000 if service == "lms" else 8001 host = config["LMS_HOST"] if service == "lms" else config["CMS_HOST"] @@ -56,7 +58,7 @@ def runserver(context, options, service): ) ) args = ["--service-ports", *options, service] - compose.run.callback(args) + context.invoke(compose.run, args=args) dev.add_command(runserver) diff --git a/tutor/commands/images.py b/tutor/commands/images.py index ac2d6d3..3e01de4 100644 --- a/tutor/commands/images.py +++ b/tutor/commands/images.py @@ -1,3 +1,5 @@ +from typing import cast, Any, Dict, Iterator, List, Tuple + import click from .. import config as tutor_config @@ -5,6 +7,7 @@ from .. import env as tutor_env from .. import images from .. import plugins from .. import utils +from .context import Context BASE_IMAGE_NAMES = ["openedx", "forum", "android"] DEV_IMAGE_NAMES = ["openedx-dev"] @@ -20,7 +23,7 @@ VENDOR_IMAGES = [ @click.group(name="images", short_help="Manage docker images") -def images_command(): +def images_command() -> None: pass @@ -50,7 +53,14 @@ def images_command(): help="Set the target build stage to build.", ) @click.pass_obj -def build(context, image_names, no_cache, build_args, add_hosts, target): +def build( + context: Context, + image_names: List[str], + no_cache: bool, + build_args: List[str], + add_hosts: List[str], + target: str, +) -> None: config = tutor_config.load(context.root) command_args = [] if no_cache: @@ -68,7 +78,7 @@ def build(context, image_names, no_cache, build_args, add_hosts, target): @click.command(short_help="Pull images from the Docker registry") @click.argument("image_names", metavar="image", nargs=-1) @click.pass_obj -def pull(context, image_names): +def pull(context: Context, image_names: List[str]) -> None: config = tutor_config.load(context.root) for image in image_names: pull_image(config, image) @@ -77,7 +87,7 @@ def pull(context, image_names): @click.command(short_help="Push images to the Docker registry") @click.argument("image_names", metavar="image", nargs=-1) @click.pass_obj -def push(context, image_names): +def push(context: Context, image_names: List[str]) -> None: config = tutor_config.load(context.root) for image in image_names: push_image(config, image) @@ -86,16 +96,16 @@ def push(context, image_names): @click.command(short_help="Print tag associated to a Docker image") @click.argument("image_names", metavar="image", nargs=-1) @click.pass_obj -def printtag(context, image_names): +def printtag(context: Context, image_names: List[str]) -> None: config = tutor_config.load(context.root) for image in image_names: for _img, tag in iter_images(config, image, BASE_IMAGE_NAMES): print(tag) - for _img, tag in iter_plugin_images(config, image, "build-image"): + for _plugin, _img, tag in iter_plugin_images(config, image, "build-image"): print(tag) -def build_image(root, config, image, *args): +def build_image(root: str, config: Dict[str, Any], image: str, *args: str) -> None: # Build base images for img, tag in iter_images(config, image, BASE_IMAGE_NAMES): images.build(tutor_env.pathjoin(root, "build", img), tag, *args) @@ -112,40 +122,45 @@ def build_image(root, config, image, *args): images.build(tutor_env.pathjoin(root, "build", img), tag, *dev_build_arg, *args) -def pull_image(config, image): +def pull_image(config: Dict[str, Any], image: str) -> None: for _img, tag in iter_images(config, image, all_image_names(config)): images.pull(tag) for _plugin, _img, tag in iter_plugin_images(config, image, "remote-image"): images.pull(tag) -def push_image(config, image): +def push_image(config: Dict[str, Any], image: str) -> None: for _img, tag in iter_images(config, image, BASE_IMAGE_NAMES): images.push(tag) for _plugin, _img, tag in iter_plugin_images(config, image, "remote-image"): images.push(tag) -def iter_images(config, image, image_list): +def iter_images( + config: Dict[str, Any], image: str, image_list: List[str] +) -> Iterator[Tuple[str, str]]: for img in image_list: if image in [img, "all"]: tag = images.get_tag(config, img) yield img, tag -def iter_plugin_images(config, image, hook_name): +def iter_plugin_images( + config: Dict[str, Any], image: str, hook_name: str +) -> Iterator[Tuple[str, str, str]]: for plugin, hook in plugins.iter_hooks(config, hook_name): + hook = cast(Dict[str, str], hook) for img, tag in hook.items(): if image in [img, "all"]: tag = tutor_env.render_str(config, tag) yield plugin, img, tag -def all_image_names(config): +def all_image_names(config: Dict[str, Any]) -> List[str]: return BASE_IMAGE_NAMES + vendor_image_names(config) -def vendor_image_names(config): +def vendor_image_names(config: Dict[str, Any]) -> List[str]: vendor_images = VENDOR_IMAGES[:] for image in VENDOR_IMAGES: if not config.get("RUN_" + image.upper(), True): diff --git a/tutor/commands/k8s.py b/tutor/commands/k8s.py index e3fbc64..2d859bf 100644 --- a/tutor/commands/k8s.py +++ b/tutor/commands/k8s.py @@ -1,5 +1,6 @@ from datetime import datetime from time import sleep +from typing import cast, Any, Dict, List, Optional, Type import click @@ -11,286 +12,13 @@ from .. import interactive as interactive_config from .. import jobs from .. import serialize from .. import utils - - -@click.group(help="Run Open edX on Kubernetes") -def k8s(): - pass - - -@click.command(help="Configure and run Open edX from scratch") -@click.option("-I", "--non-interactive", is_flag=True, help="Run non-interactively") -@click.pass_obj -def quickstart(context, non_interactive): - click.echo(fmt.title("Interactive platform configuration")) - config = interactive_config.update(context.root, interactive=(not non_interactive)) - if not config["RUN_CADDY"]: - fmt.echo_alert( - "Potentially invalid configuration: RUN_CADDY=false\n" - "This setting might have been defined because you previously set WEB_PROXY=true. This is no longer" - " necessary in order to get Tutor to work on Kubernetes. In Tutor v11+ a Caddy-based load balancer is" - " provided out of the box to handle SSL/TLS certificate generation at runtime. If you disable this" - " service, you will have to configure an Ingress resource and a certificate manager yourself to redirect" - " traffic to the nginx service. See the Kubernetes section in the Tutor documentation for more" - " information." - ) - click.echo(fmt.title("Updating the current environment")) - tutor_env.save(context.root, config) - click.echo(fmt.title("Starting the platform")) - start.callback() - click.echo(fmt.title("Database creation and migrations")) - init.callback(limit=None) - fmt.echo_info( - """Your Open edX platform is ready and can be accessed at the following urls: - - {http}://{lms_host} - {http}://{cms_host} - """.format( - http="https" if config["ENABLE_HTTPS"] else "http", - lms_host=config["LMS_HOST"], - cms_host=config["CMS_HOST"], - ) - ) - - -@click.command(help="Run all configured Open edX services") -@click.pass_obj -def start(context): - # Create namespace - utils.kubectl( - "apply", - "--kustomize", - tutor_env.pathjoin(context.root), - "--wait", - "--selector", - "app.kubernetes.io/component=namespace", - ) - # Create volumes - utils.kubectl( - "apply", - "--kustomize", - tutor_env.pathjoin(context.root), - "--wait", - "--selector", - "app.kubernetes.io/component=volume", - ) - # Create everything else except jobs - utils.kubectl( - "apply", - "--kustomize", - tutor_env.pathjoin(context.root), - "--selector", - # Here use `notin (job, xxx)` when there are other components to ignore - "app.kubernetes.io/component!=job", - ) - - -@click.command(help="Stop a running platform") -@click.pass_obj -def stop(context): - config = tutor_config.load(context.root) - utils.kubectl( - "delete", - *resource_selector(config), - "deployments,services,configmaps,jobs", - ) - - -@click.command(help="Reboot an existing platform") -def reboot(): - stop.callback() - start.callback() - - -def resource_selector(config, *selectors): - """ - Convenient utility for filtering only the resources that belong to this project. - """ - selector = ",".join( - ["app.kubernetes.io/instance=openedx-" + config["ID"]] + list(selectors) - ) - return ["--namespace", config["K8S_NAMESPACE"], "--selector=" + selector] - - -@click.command(help="Completely delete an existing platform") -@click.option("-y", "--yes", is_flag=True, help="Do not ask for confirmation") -@click.pass_obj -def delete(context, yes): - if not yes: - click.confirm( - "Are you sure you want to delete the platform? All data will be removed.", - abort=True, - ) - utils.kubectl( - "delete", - "-k", - tutor_env.pathjoin(context.root), - "--ignore-not-found=true", - "--wait", - ) - - -@click.command(help="Initialise all applications") -@click.option("-l", "--limit", help="Limit initialisation to this service or plugin") -@click.pass_obj -def init(context, limit): - config = tutor_config.load(context.root) - runner = K8sJobRunner(context.root, config) - for service in ["mysql", "elasticsearch", "mongodb"]: - if tutor_config.is_service_activated(config, service): - wait_for_pod_ready(config, service) - jobs.initialise(runner, limit_to=limit) - - -@click.command(help="Create an Open edX user and interactively set their password") -@click.option("--superuser", is_flag=True, help="Make superuser") -@click.option("--staff", is_flag=True, help="Make staff user") -@click.option( - "-p", - "--password", - help="Specify password from the command line. If undefined, you will be prompted to input a password", -) -@click.argument("name") -@click.argument("email") -@click.pass_obj -def createuser(context, superuser, staff, password, name, email): - config = tutor_config.load(context.root) - command = jobs.create_user_command(superuser, staff, name, email, password=password) - # This needs to be interactive in case the user needs to type a password - kubectl_exec(config, "lms", command, attach=True) - - -@click.command(help="Import the demo course") -@click.pass_obj -def importdemocourse(context): - fmt.echo_info("Importing demo course") - config = tutor_config.load(context.root) - runner = K8sJobRunner(context.root, config) - jobs.import_demo_course(runner) - - -@click.command( - help="Set a theme for a given domain name. To reset to the default theme , use 'default' as the theme name." -) -@click.argument("theme_name") -@click.argument("domain_names", metavar="domain_name", nargs=-1) -@click.pass_obj -def settheme(context, theme_name, domain_names): - config = tutor_config.load(context.root) - runner = K8sJobRunner(context.root, config) - for domain_name in domain_names: - jobs.set_theme(theme_name, domain_name, runner) - - -@click.command(name="exec", help="Execute a command in a pod of the given application") -@click.argument("service") -@click.argument("command") -@click.pass_obj -def exec_command(context, service, command): - config = tutor_config.load(context.root) - kubectl_exec(config, service, command, attach=True) - - -@click.command(help="View output from containers") -@click.option("-c", "--container", help="Print the logs of this specific container") -@click.option("-f", "--follow", is_flag=True, help="Follow log output") -@click.option("--tail", type=int, help="Number of lines to show from each container") -@click.argument("service") -@click.pass_obj -def logs(context, container, follow, tail, service): - config = tutor_config.load(context.root) - - command = ["logs"] - selectors = ["app.kubernetes.io/name=" + service] if service else [] - command += resource_selector(config, *selectors) - - if container: - command += ["-c", container] - if follow: - command += ["--follow"] - if tail is not None: - command += ["--tail", str(tail)] - - utils.kubectl(*command) - - -@click.command(help="Wait for a pod to become ready") -@click.argument("name") -@click.pass_obj -def wait(context, name): - config = tutor_config.load(context.root) - wait_for_pod_ready(config, name) - - -@click.command(help="Upgrade from a previous Open edX named release") -@click.option( - "--from", "from_version", default="ironwood", type=click.Choice(["ironwood"]) -) -@click.pass_obj -def upgrade(context, from_version): - config = tutor_config.load(context.root) - - running_version = from_version - if running_version == "ironwood": - upgrade_from_ironwood(config) - running_version = "juniper" - - if running_version == "juniper": - - running_version = "koa" - - -def upgrade_from_ironwood(config): - if not config["RUN_MONGODB"]: - fmt.echo_info( - "You are not running MongDB (RUN_MONGODB=false). It is your " - "responsibility to upgrade your MongoDb instance to v3.6. There is " - "nothing left to do to upgrade from Ironwood." - ) - return - message = """Automatic release upgrade is unsupported in Kubernetes. To upgrade from Ironwood, you should upgrade -your MongoDb cluster from v3.2 to v3.6. You should run something similar to: - - # Upgrade from v3.2 to v3.4 - tutor k8s stop - tutor config save --set DOCKER_IMAGE_MONGODB=mongo:3.4.24 - tutor k8s start - tutor k8s exec mongodb mongo --eval 'db.adminCommand({ setFeatureCompatibilityVersion: "3.4" })' - - # Upgrade from v3.4 to v3.6 - tutor k8s stop - tutor config save --set DOCKER_IMAGE_MONGODB=mongo:3.6.18 - tutor k8s start - tutor k8s exec mongodb mongo --eval 'db.adminCommand({ setFeatureCompatibilityVersion: "3.6" })' - - tutor config save --unset DOCKER_IMAGE_MONGODB""" - fmt.echo_info(message) - - -def upgrade_from_juniper(config): - if not config["RUN_MYSQL"]: - fmt.echo_info( - "You are not running MySQL (RUN_MYSQL=false). It is your " - "responsibility to upgrade your MySQL instance to v5.7. There is " - "nothing left to do to upgrade from Juniper." - ) - return - - message = """Automatic release upgrade is unsupported in Kubernetes. To upgrade from Juniper, you should upgrade -your MySQL database from v5.6 to v5.7. You should run something similar to: - - tutor k8s start - tutor k8s exec mysql bash -e -c "mysql_upgrade \ - -u $(tutor config printvalue MYSQL_ROOT_USERNAME) \ - --password='$(tutor config printvalue MYSQL_ROOT_PASSWORD)' -""" - fmt.echo_info(message) +from .context import Context class K8sClients: _instance = None - def __init__(self): + def __init__(self) -> None: # Loading the kubernetes module here to avoid import overhead from kubernetes import client, config # pylint: disable=import-outside-toplevel @@ -300,33 +28,34 @@ class K8sClients: self._client = client @classmethod - def instance(cls): + def instance(cls: Type["K8sClients"]) -> "K8sClients": if cls._instance is None: cls._instance = cls() return cls._instance @property - def batch_api(self): + def batch_api(self): # type: ignore if self._batch_api is None: self._batch_api = self._client.BatchV1Api() return self._batch_api @property - def core_api(self): + def core_api(self): # type: ignore if self._core_api is None: self._core_api = self._client.CoreV1Api() return self._core_api class K8sJobRunner(jobs.BaseJobRunner): - def load_job(self, name): - jobs = self.render("k8s", "jobs.yml") - for job in serialize.load_all(jobs): - if job["metadata"]["name"] == name: + def load_job(self, name: str) -> Any: + all_jobs = self.render("k8s", "jobs.yml") + for job in serialize.load_all(all_jobs): + job_name = cast(str, job["metadata"]["name"]) + if job_name == name: return job raise ValueError("Could not find job '{}'".format(name)) - def active_job_names(self): + def active_job_names(self) -> List[str]: """ Return a list of active job names Docs: @@ -339,7 +68,7 @@ class K8sJobRunner(jobs.BaseJobRunner): if job.status.active ] - def run_job(self, service, command): + def run_job(self, service: str, command: str) -> int: job_name = "{}-job".format(service) try: job = self.load_job(job_name) @@ -361,8 +90,7 @@ class K8sJobRunner(jobs.BaseJobRunner): ) fmt.echo_alert(message) wait_for_pod_ready(self.config, service) - kubectl_exec(self.config, service, command) - return + return kubectl_exec(self.config, service, command) # Create a unique job name to make it deduplicate jobs and make it easier to # find later. Logs of older jobs will remain available for some time. job_name += "-" + datetime.now().strftime("%Y%m%d%H%M%S") @@ -417,12 +145,12 @@ class K8sJobRunner(jobs.BaseJobRunner): # Wait for completion field_selector = "metadata.name={}".format(job_name) while True: - jobs = K8sClients.instance().batch_api.list_namespaced_job( + namespaced_jobs = K8sClients.instance().batch_api.list_namespaced_job( self.config["K8S_NAMESPACE"], field_selector=field_selector ) - if not jobs.items: + if not namespaced_jobs.items: continue - job = jobs.items[0] + job = namespaced_jobs.items[0] if not job.status.active: if job.status.succeeded: fmt.echo_info("Job {} successful.".format(job_name)) @@ -434,9 +162,292 @@ class K8sJobRunner(jobs.BaseJobRunner): ) ) sleep(5) + return 0 -def kubectl_exec(config, service, command, attach=False): +@click.group(help="Run Open edX on Kubernetes") +def k8s() -> None: + pass + + +@click.command(help="Configure and run Open edX from scratch") +@click.option("-I", "--non-interactive", is_flag=True, help="Run non-interactively") +@click.pass_context +def quickstart(context: click.Context, non_interactive: bool) -> None: + click.echo(fmt.title("Interactive platform configuration")) + config = interactive_config.update( + context.obj.root, interactive=(not non_interactive) + ) + if not config["RUN_CADDY"]: + fmt.echo_alert( + "Potentially invalid configuration: RUN_CADDY=false\n" + "This setting might have been defined because you previously set WEB_PROXY=true. This is no longer" + " necessary in order to get Tutor to work on Kubernetes. In Tutor v11+ a Caddy-based load balancer is" + " provided out of the box to handle SSL/TLS certificate generation at runtime. If you disable this" + " service, you will have to configure an Ingress resource and a certificate manager yourself to redirect" + " traffic to the nginx service. See the Kubernetes section in the Tutor documentation for more" + " information." + ) + click.echo(fmt.title("Updating the current environment")) + tutor_env.save(context.obj.root, config) + click.echo(fmt.title("Starting the platform")) + context.invoke(start) + click.echo(fmt.title("Database creation and migrations")) + context.invoke(init, limit=None) + fmt.echo_info( + """Your Open edX platform is ready and can be accessed at the following urls: + + {http}://{lms_host} + {http}://{cms_host} + """.format( + http="https" if config["ENABLE_HTTPS"] else "http", + lms_host=config["LMS_HOST"], + cms_host=config["CMS_HOST"], + ) + ) + + +@click.command(help="Run all configured Open edX services") +@click.pass_obj +def start(context: Context) -> None: + # Create namespace + utils.kubectl( + "apply", + "--kustomize", + tutor_env.pathjoin(context.root), + "--wait", + "--selector", + "app.kubernetes.io/component=namespace", + ) + # Create volumes + utils.kubectl( + "apply", + "--kustomize", + tutor_env.pathjoin(context.root), + "--wait", + "--selector", + "app.kubernetes.io/component=volume", + ) + # Create everything else except jobs + utils.kubectl( + "apply", + "--kustomize", + tutor_env.pathjoin(context.root), + "--selector", + # Here use `notin (job, xxx)` when there are other components to ignore + "app.kubernetes.io/component!=job", + ) + + +@click.command(help="Stop a running platform") +@click.pass_obj +def stop(context: Context) -> None: + config = tutor_config.load(context.root) + utils.kubectl( + "delete", + *resource_selector(config), + "deployments,services,configmaps,jobs", + ) + + +@click.command(help="Reboot an existing platform") +@click.pass_context +def reboot(context: click.Context) -> None: + context.invoke(stop) + context.invoke(start) + + +def resource_selector(config: Dict[str, str], *selectors: str) -> List[str]: + """ + Convenient utility for filtering only the resources that belong to this project. + """ + selector = ",".join( + ["app.kubernetes.io/instance=openedx-" + config["ID"]] + list(selectors) + ) + return ["--namespace", config["K8S_NAMESPACE"], "--selector=" + selector] + + +@click.command(help="Completely delete an existing platform") +@click.option("-y", "--yes", is_flag=True, help="Do not ask for confirmation") +@click.pass_obj +def delete(context: Context, yes: bool) -> None: + if not yes: + click.confirm( + "Are you sure you want to delete the platform? All data will be removed.", + abort=True, + ) + utils.kubectl( + "delete", + "-k", + tutor_env.pathjoin(context.root), + "--ignore-not-found=true", + "--wait", + ) + + +@click.command(help="Initialise all applications") +@click.option("-l", "--limit", help="Limit initialisation to this service or plugin") +@click.pass_obj +def init(context: Context, limit: Optional[str]) -> None: + config = tutor_config.load(context.root) + runner = K8sJobRunner(context.root, config) + for service in ["mysql", "elasticsearch", "mongodb"]: + if tutor_config.is_service_activated(config, service): + wait_for_pod_ready(config, service) + jobs.initialise(runner, limit_to=limit) + + +@click.command(help="Create an Open edX user and interactively set their password") +@click.option("--superuser", is_flag=True, help="Make superuser") +@click.option("--staff", is_flag=True, help="Make staff user") +@click.option( + "-p", + "--password", + help="Specify password from the command line. If undefined, you will be prompted to input a password", +) +@click.argument("name") +@click.argument("email") +@click.pass_obj +def createuser( + context: Context, superuser: str, staff: bool, password: str, name: str, email: str +) -> None: + config = tutor_config.load(context.root) + command = jobs.create_user_command(superuser, staff, name, email, password=password) + # This needs to be interactive in case the user needs to type a password + kubectl_exec(config, "lms", command, attach=True) + + +@click.command(help="Import the demo course") +@click.pass_obj +def importdemocourse(context: Context) -> None: + fmt.echo_info("Importing demo course") + config = tutor_config.load(context.root) + runner = K8sJobRunner(context.root, config) + jobs.import_demo_course(runner) + + +@click.command( + help="Set a theme for a given domain name. To reset to the default theme , use 'default' as the theme name." +) +@click.argument("theme_name") +@click.argument("domain_names", metavar="domain_name", nargs=-1) +@click.pass_obj +def settheme(context: Context, theme_name: str, domain_names: List[str]) -> None: + config = tutor_config.load(context.root) + runner = K8sJobRunner(context.root, config) + for domain_name in domain_names: + jobs.set_theme(theme_name, domain_name, runner) + + +@click.command(name="exec", help="Execute a command in a pod of the given application") +@click.argument("service") +@click.argument("command") +@click.pass_obj +def exec_command(context: Context, service: str, command: str) -> None: + config = tutor_config.load(context.root) + kubectl_exec(config, service, command, attach=True) + + +@click.command(help="View output from containers") +@click.option("-c", "--container", help="Print the logs of this specific container") +@click.option("-f", "--follow", is_flag=True, help="Follow log output") +@click.option("--tail", type=int, help="Number of lines to show from each container") +@click.argument("service") +@click.pass_obj +def logs( + context: Context, container: str, follow: bool, tail: bool, service: str +) -> None: + config = tutor_config.load(context.root) + + command = ["logs"] + selectors = ["app.kubernetes.io/name=" + service] if service else [] + command += resource_selector(config, *selectors) + + if container: + command += ["-c", container] + if follow: + command += ["--follow"] + if tail is not None: + command += ["--tail", str(tail)] + + utils.kubectl(*command) + + +@click.command(help="Wait for a pod to become ready") +@click.argument("name") +@click.pass_obj +def wait(context: Context, name: str) -> None: + config = tutor_config.load(context.root) + wait_for_pod_ready(config, name) + + +@click.command(help="Upgrade from a previous Open edX named release") +@click.option( + "--from", "from_version", default="ironwood", type=click.Choice(["ironwood"]) +) +@click.pass_obj +def upgrade(context: Context, from_version: str) -> None: + config = tutor_config.load(context.root) + + running_version = from_version + if running_version == "ironwood": + upgrade_from_ironwood(config) + running_version = "juniper" + + if running_version == "juniper": + running_version = "koa" + + +def upgrade_from_ironwood(config: Dict[str, Any]) -> None: + if not config["RUN_MONGODB"]: + fmt.echo_info( + "You are not running MongDB (RUN_MONGODB=false). It is your " + "responsibility to upgrade your MongoDb instance to v3.6. There is " + "nothing left to do to upgrade from Ironwood." + ) + return + message = """Automatic release upgrade is unsupported in Kubernetes. To upgrade from Ironwood, you should upgrade +your MongoDb cluster from v3.2 to v3.6. You should run something similar to: + + # Upgrade from v3.2 to v3.4 + tutor k8s stop + tutor config save --set DOCKER_IMAGE_MONGODB=mongo:3.4.24 + tutor k8s start + tutor k8s exec mongodb mongo --eval 'db.adminCommand({ setFeatureCompatibilityVersion: "3.4" })' + + # Upgrade from v3.4 to v3.6 + tutor k8s stop + tutor config save --set DOCKER_IMAGE_MONGODB=mongo:3.6.18 + tutor k8s start + tutor k8s exec mongodb mongo --eval 'db.adminCommand({ setFeatureCompatibilityVersion: "3.6" })' + + tutor config save --unset DOCKER_IMAGE_MONGODB""" + fmt.echo_info(message) + + +def upgrade_from_juniper(config: Dict[str, Any]) -> None: + if not config["RUN_MYSQL"]: + fmt.echo_info( + "You are not running MySQL (RUN_MYSQL=false). It is your " + "responsibility to upgrade your MySQL instance to v5.7. There is " + "nothing left to do to upgrade from Juniper." + ) + return + + message = """Automatic release upgrade is unsupported in Kubernetes. To upgrade from Juniper, you should upgrade +your MySQL database from v5.6 to v5.7. You should run something similar to: + + tutor k8s start + tutor k8s exec mysql bash -e -c "mysql_upgrade \ + -u $(tutor config printvalue MYSQL_ROOT_USERNAME) \ + --password='$(tutor config printvalue MYSQL_ROOT_PASSWORD)' +""" + fmt.echo_info(message) + + +def kubectl_exec( + config: Dict[str, Any], service: str, command: str, attach: bool = False +) -> int: selector = "app.kubernetes.io/name={}".format(service) pods = K8sClients.instance().core_api.list_namespaced_pod( namespace=config["K8S_NAMESPACE"], label_selector=selector @@ -449,7 +460,7 @@ def kubectl_exec(config, service, command, attach=False): # Run command attach_opts = ["-i", "-t"] if attach else [] - utils.kubectl( + return utils.kubectl( "exec", *attach_opts, "--namespace", @@ -463,7 +474,7 @@ def kubectl_exec(config, service, command, attach=False): ) -def wait_for_pod_ready(config, service): +def wait_for_pod_ready(config: Dict[str, str], service: str) -> None: fmt.echo_info("Waiting for a {} pod to be ready...".format(service)) utils.kubectl( "wait", diff --git a/tutor/commands/local.py b/tutor/commands/local.py index db6fe9c..655c09e 100644 --- a/tutor/commands/local.py +++ b/tutor/commands/local.py @@ -1,15 +1,18 @@ import os +from typing import Dict, Any import click from .. import config as tutor_config from .. import env as tutor_env -from .. import fmt, utils +from .. import fmt +from .. import utils from . import compose from .config import save as config_save_command +from .context import Context -def docker_compose(root, config, *command): +def docker_compose(root: str, config: Dict[str, Any], *command: str) -> int: """ Run docker-compose with local and production yml files. """ @@ -31,39 +34,41 @@ def docker_compose(root, config, *command): @click.group(help="Run Open edX locally with docker-compose") @click.pass_obj -def local(context): - context.docker_compose = docker_compose +def local(context: Context) -> None: + context.docker_compose_func = docker_compose @click.command(help="Configure and run Open edX from scratch") @click.option("-I", "--non-interactive", is_flag=True, help="Run non-interactively") -@click.option( - "-p", "--pullimages", "pullimages_", is_flag=True, help="Update docker images" -) -@click.pass_obj -def quickstart(context, non_interactive, pullimages_): - if tutor_env.needs_major_upgrade(context.root): +@click.option("-p", "--pullimages", is_flag=True, help="Update docker images") +@click.pass_context +def quickstart(context: click.Context, non_interactive: bool, pullimages: bool) -> None: + if tutor_env.needs_major_upgrade(context.obj.root): click.echo(fmt.title("Upgrading from an older release")) - upgrade.callback( - from_version=tutor_env.current_release(context.root), + context.invoke( + upgrade, + from_version=tutor_env.current_release(context.obj.root), non_interactive=non_interactive, ) click.echo(fmt.title("Interactive platform configuration")) - config_save_command.callback( - interactive=(not non_interactive), set_vars=[], unset_vars=[] + context.invoke( + config_save_command, + interactive=(not non_interactive), + set_vars=[], + unset_vars=[], ) click.echo(fmt.title("Stopping any existing platform")) - compose.stop.callback([]) - if pullimages_: + context.invoke(compose.stop) + if pullimages: click.echo(fmt.title("Docker image updates")) - compose.dc_command.callback(["pull"]) + context.invoke(compose.dc_command, command="pull") click.echo(fmt.title("Starting the platform in detached mode")) - compose.start.callback(True, []) + context.invoke(compose.start, detach=True) click.echo(fmt.title("Database creation and migrations")) - compose.init.callback(limit=None) + context.invoke(compose.init) - config = tutor_config.load(context.root) + config = tutor_config.load(context.obj.root) fmt.echo_info( """The Open edX platform is now running in detached mode Your Open edX platform is ready and can be accessed at the following urls: @@ -86,9 +91,9 @@ Your Open edX platform is ready and can be accessed at the following urls: type=click.Choice(["ironwood", "juniper"]), ) @click.option("-I", "--non-interactive", is_flag=True, help="Run non-interactively") -@click.pass_obj -def upgrade(context, from_version, non_interactive): - config = tutor_config.load_no_check(context.root) +@click.pass_context +def upgrade(context: click.Context, from_version: str, non_interactive: bool) -> None: + config = tutor_config.load_no_check(context.obj.root) if not non_interactive: question = """You are about to upgrade your Open edX platform. It is strongly recommended to make a backup before upgrading. To do so, run: @@ -113,12 +118,12 @@ Are you sure you want to continue?""" running_version = "koa" -def upgrade_from_ironwood(context, config): +def upgrade_from_ironwood(context: click.Context, config: Dict[str, Any]) -> None: click.echo(fmt.title("Upgrading from Ironwood")) - tutor_env.save(context.root, config) + tutor_env.save(context.obj.root, config) click.echo(fmt.title("Stopping any existing platform")) - compose.stop.callback([]) + context.invoke(compose.stop) if not config["RUN_MONGODB"]: fmt.echo_info( @@ -132,39 +137,41 @@ def upgrade_from_ironwood(context, config): # environment, not the configuration. click.echo(fmt.title("Upgrading MongoDb from v3.2 to v3.4")) config["DOCKER_IMAGE_MONGODB"] = "mongo:3.4.24" - tutor_env.save(context.root, config) - compose.start.callback(detach=True, services=["mongodb"]) - compose.execute.callback( - [ + tutor_env.save(context.obj.root, config) + context.invoke(compose.start, detach=True, services=["mongodb"]) + context.invoke( + compose.execute, + args=[ "mongodb", "mongo", "--eval", 'db.adminCommand({ setFeatureCompatibilityVersion: "3.4" })', - ] + ], ) - compose.stop.callback([]) + context.invoke(compose.stop) click.echo(fmt.title("Upgrading MongoDb from v3.4 to v3.6")) config["DOCKER_IMAGE_MONGODB"] = "mongo:3.6.18" - tutor_env.save(context.root, config) - compose.start.callback(detach=True, services=["mongodb"]) - compose.execute.callback( - [ + tutor_env.save(context.obj.root, config) + context.invoke(compose.start, detach=True, services=["mongodb"]) + context.invoke( + compose.execute, + args=[ "mongodb", "mongo", "--eval", 'db.adminCommand({ setFeatureCompatibilityVersion: "3.6" })', - ] + ], ) - compose.stop.callback([]) + context.invoke(compose.stop) -def upgrade_from_juniper(context, config): +def upgrade_from_juniper(context: click.Context, config: Dict[str, Any]) -> None: click.echo(fmt.title("Upgrading from Juniper")) - tutor_env.save(context.root, config) + tutor_env.save(context.obj.root, config) click.echo(fmt.title("Stopping any existing platform")) - compose.stop.callback([]) + context.invoke(compose.stop) if not config["RUN_MYSQL"]: fmt.echo_info( @@ -175,9 +182,10 @@ def upgrade_from_juniper(context, config): return click.echo(fmt.title("Upgrading MySQL from v5.6 to v5.7")) - compose.start.callback(detach=True, services=["mysql"]) - compose.execute.callback( - [ + context.invoke(compose.start, detach=True, services=["mysql"]) + context.invoke( + compose.execute, + args=[ "mysql", "bash", "-e", @@ -185,9 +193,9 @@ def upgrade_from_juniper(context, config): "mysql_upgrade -u {} --password='{}'".format( config["MYSQL_ROOT_USERNAME"], config["MYSQL_ROOT_PASSWORD"] ), - ] + ], ) - compose.stop.callback([]) + context.invoke(compose.stop) local.add_command(quickstart) diff --git a/tutor/commands/plugins.py b/tutor/commands/plugins.py index 4a356a7..aea002a 100644 --- a/tutor/commands/plugins.py +++ b/tutor/commands/plugins.py @@ -1,5 +1,6 @@ import os import shutil +from typing import List import urllib.request import click @@ -9,6 +10,7 @@ from .. import env as tutor_env from .. import exceptions from .. import fmt from .. import plugins +from .context import Context @click.group( @@ -16,7 +18,7 @@ from .. import plugins short_help="Manage Tutor plugins", help="Manage Tutor plugins to add new features and customize your Open edX platform", ) -def plugins_command(): +def plugins_command() -> None: """ All plugin commands should work even if there is no existing config file. This is because users might enable plugins prior to configuration or environment generation. @@ -25,7 +27,7 @@ def plugins_command(): @click.command(name="list", help="List installed plugins") @click.pass_obj -def list_command(context): +def list_command(context: Context) -> None: config = tutor_config.load_user(context.root) for plugin in plugins.iter_installed(): status = "" if plugins.is_enabled(config, plugin.name) else " (disabled)" @@ -39,7 +41,7 @@ def list_command(context): @click.command(help="Enable a plugin") @click.argument("plugin_names", metavar="plugin", nargs=-1) @click.pass_obj -def enable(context, plugin_names): +def enable(context: Context, plugin_names: List[str]) -> None: config = tutor_config.load_user(context.root) for plugin in plugin_names: plugins.enable(config, plugin) @@ -56,7 +58,7 @@ def enable(context, plugin_names): ) @click.argument("plugin_names", metavar="plugin", nargs=-1) @click.pass_obj -def disable(context, plugin_names): +def disable(context: Context, plugin_names: List[str]) -> None: config = tutor_config.load_user(context.root) if "all" in plugin_names: plugin_names = [plugin.name for plugin in plugins.iter_enabled(config)] @@ -70,7 +72,7 @@ def disable(context, plugin_names): ) -def delete_plugin(root, name): +def delete_plugin(root: str, name: str) -> None: plugin_dir = tutor_env.pathjoin(root, "plugins", name) if os.path.exists(plugin_dir): try: @@ -90,7 +92,7 @@ defined by setting the {} environment variable""".format( plugins.DictPlugin.ROOT_ENV_VAR_NAME ), ) -def printroot(): +def printroot() -> None: fmt.echo(plugins.DictPlugin.ROOT) @@ -102,7 +104,7 @@ location. The plugin will be installed to {}.""".format( ), ) @click.argument("location") -def install(location): +def install(location: str) -> None: basename = os.path.basename(location) if not basename.endswith(".yml"): basename += ".yml" @@ -127,7 +129,7 @@ def install(location): fmt.echo_info("Plugin installed at {}".format(plugin_path)) -def add_plugin_commands(command_group): +def add_plugin_commands(command_group: click.Group) -> None: """ Add commands provided by all plugins to the given command group. Each command is added with a name that is equal to the plugin name. diff --git a/tutor/commands/ui.py b/tutor/commands/ui.py index c6ce17e..6b06056 100644 --- a/tutor/commands/ui.py +++ b/tutor/commands/ui.py @@ -6,7 +6,7 @@ import click_repl short_help="Interactive shell", help="Launch an interactive shell for launching Tutor commands", ) -def ui(): +def ui() -> None: click.echo( """Welcome to the Tutor interactive shell UI! Type "help" to view all available commands. diff --git a/tutor/commands/webui.py b/tutor/commands/webui.py index 896910e..b210bd7 100644 --- a/tutor/commands/webui.py +++ b/tutor/commands/webui.py @@ -4,6 +4,7 @@ import platform import subprocess import sys import tarfile +from typing import Any, Dict from urllib.request import urlopen import click @@ -13,12 +14,13 @@ import click from .. import fmt from .. import env as tutor_env from .. import serialize +from .context import Context @click.group( short_help="Web user interface", help="""Run Tutor commands from a web terminal""" ) -def webui(): +def webui() -> None: pass @@ -35,7 +37,7 @@ def webui(): "-h", "--host", default="0.0.0.0", show_default=True, help="Host address to listen" ) @click.pass_obj -def start(context, port, host): +def start(context: Context, port: int, host: str) -> None: check_gotty_binary(context.root) fmt.echo_info("Access the Tutor web UI at http://{}:{}".format(host, port)) while True: @@ -86,7 +88,7 @@ def start(context, port, host): help="Authentication password", ) @click.pass_obj -def configure(context, user, password): +def configure(context: Context, user: str, password: str) -> None: save_webui_config_file(context.root, {"user": user, "password": password}) fmt.echo_info( "The web UI configuration has been updated. " @@ -95,7 +97,7 @@ def configure(context, user, password): ) -def check_gotty_binary(root): +def check_gotty_binary(root: str) -> None: path = gotty_path(root) if os.path.exists(path): return @@ -119,7 +121,7 @@ def check_gotty_binary(root): compressed.extract("./gotty", dirname) -def load_config(root): +def load_config(root: str) -> Dict[str, Any]: path = config_path(root) if not os.path.exists(path): save_webui_config_file(root, {"user": None, "password": None}) @@ -127,7 +129,7 @@ def load_config(root): return serialize.load(f) -def save_webui_config_file(root, config): +def save_webui_config_file(root: str, config: Dict[str, Any]) -> None: path = config_path(root) directory = os.path.dirname(path) if not os.path.exists(directory): @@ -136,15 +138,15 @@ def save_webui_config_file(root, config): serialize.dump(config, of) -def gotty_path(root): +def gotty_path(root: str) -> str: return get_path(root, "gotty") -def config_path(root): +def config_path(root: str) -> str: return get_path(root, "config.yml") -def get_path(root, filename): +def get_path(root: str, filename: str) -> str: return tutor_env.pathjoin(root, "webui", filename) diff --git a/tutor/config.py b/tutor/config.py index a0f98f0..7e9def7 100644 --- a/tutor/config.py +++ b/tutor/config.py @@ -1,4 +1,5 @@ import os +from typing import Dict, Any, Tuple from . import exceptions from . import env @@ -8,7 +9,7 @@ from . import serialize from . import utils -def update(root): +def update(root: str) -> Dict[str, Any]: """ Load and save the configuration. """ @@ -18,7 +19,7 @@ def update(root): return config -def load(root): +def load(root: str) -> Dict[str, Any]: """ Load full configuration. This will raise an exception if there is no current configuration in the project root. @@ -27,13 +28,13 @@ def load(root): return load_no_check(root) -def load_no_check(root): +def load_no_check(root: str) -> Dict[str, Any]: config, defaults = load_all(root) merge(config, defaults) return config -def load_all(root): +def load_all(root: str) -> Tuple[Dict[str, Any], Dict[str, Any]]: """ Return: current (dict): params currently saved in config.yml @@ -45,7 +46,9 @@ def load_all(root): return current, defaults -def merge(config, defaults, force=False): +def merge( + config: Dict[str, str], defaults: Dict[str, str], force: bool = False +) -> None: """ Merge default values with user configuration and perform rendering of "{{...}}" values. @@ -55,16 +58,16 @@ def merge(config, defaults, force=False): config[key] = env.render_unknown(config, value) -def load_defaults(): +def load_defaults() -> Dict[str, Any]: return serialize.load(env.read_template_file("config.yml")) -def load_config_file(path): +def load_config_file(path: str) -> Dict[str, Any]: with open(path) as f: return serialize.load(f.read()) -def load_current(root, defaults): +def load_current(root: str, defaults: Dict[str, str]) -> Dict[str, Any]: """ Load the configuration currently stored on disk. Note: this modifies the defaults with the plugin default values. @@ -77,7 +80,7 @@ def load_current(root, defaults): return config -def load_user(root): +def load_user(root: str) -> Dict[str, Any]: path = config_path(root) if not os.path.exists(path): return {} @@ -87,14 +90,14 @@ def load_user(root): return config -def load_env(config, defaults): +def load_env(config: Dict[str, str], defaults: Dict[str, str]) -> None: for k in defaults.keys(): env_var = "TUTOR_" + k if env_var in os.environ: config[k] = serialize.parse(os.environ[env_var]) -def load_required(config, defaults): +def load_required(config: Dict[str, str], defaults: Dict[str, str]) -> None: """ All these keys must be present in the user's config.yml. This includes all values that are generated once and must be kept after that, such as passwords. @@ -111,7 +114,7 @@ def load_required(config, defaults): config[key] = env.render_unknown(config, defaults[key]) -def load_plugins(config, defaults): +def load_plugins(config: Dict[str, str], defaults: Dict[str, str]) -> None: """ Add, override and set new defaults from plugins. """ @@ -133,11 +136,11 @@ def load_plugins(config, defaults): config[key] = env.render_unknown(config, value) -def is_service_activated(config, service): - return config["RUN_" + service.upper()] +def is_service_activated(config: Dict[str, Any], service: str) -> bool: + return config["RUN_" + service.upper()] is not False -def upgrade_obsolete(config): +def upgrade_obsolete(config: Dict[str, Any]) -> None: # Openedx-specific mysql passwords if "MYSQL_PASSWORD" in config: config["MYSQL_ROOT_PASSWORD"] = config["MYSQL_PASSWORD"] @@ -178,7 +181,7 @@ def upgrade_obsolete(config): config[name.replace("ACTIVATE_", "RUN_")] = config.pop(name) -def convert_json2yml(root): +def convert_json2yml(root: str) -> None: """ Older versions of tutor used to have json config files. """ @@ -199,7 +202,7 @@ def convert_json2yml(root): ) -def save_config_file(root, config): +def save_config_file(root: str, config: Dict[str, str]) -> None: path = config_path(root) utils.ensure_file_directory_exists(path) with open(path, "w") as of: @@ -207,7 +210,7 @@ def save_config_file(root, config): fmt.echo_info("Configuration saved to {}".format(path)) -def check_existing_config(root): +def check_existing_config(root: str) -> None: """ Check there is a configuration on disk and the current environment is up-to-date. """ @@ -220,5 +223,5 @@ def check_existing_config(root): env.check_is_up_to_date(root) -def config_path(root): +def config_path(root: str) -> str: return os.path.join(root, "config.yml") diff --git a/tutor/env.py b/tutor/env.py index d0382d6..c990405 100644 --- a/tutor/env.py +++ b/tutor/env.py @@ -1,6 +1,7 @@ import codecs from copy import deepcopy import os +from typing import Dict, Any, Iterable, List, Optional, Type, Union import jinja2 import pkg_resources @@ -19,7 +20,7 @@ BIN_FILE_EXTENSIONS = [".ico", ".jpg", ".png", ".ttf", ".woff", ".woff2"] class Renderer: @classmethod - def instance(cls, config): + def instance(cls: Type["Renderer"], config: Dict[str, Any]) -> "Renderer": # Load template roots: these are required to be able to use # {% include .. %} directives template_roots = [TEMPLATES_ROOT] @@ -29,11 +30,12 @@ class Renderer: return cls(config, template_roots, ignore_folders=["partials"]) - @classmethod - def reset(cls): - cls.INSTANCE = None - - def __init__(self, config, template_roots, ignore_folders=None): + def __init__( + self, + config: Dict[str, Any], + template_roots: List[str], + ignore_folders: Optional[List[str]] = None, + ): self.config = deepcopy(config) self.template_roots = template_roots self.ignore_folders = ignore_folders or [] @@ -57,16 +59,16 @@ class Renderer: environment.globals["TUTOR_VERSION"] = __version__ self.environment = environment - def iter_templates_in(self, *prefix): + def iter_templates_in(self, *prefix: str) -> Iterable[str]: """ The elements of `prefix` must contain only "/", and not os.sep. """ - prefix = "/".join(prefix) - for template in self.environment.loader.list_templates(): - if template.startswith(prefix) and self.is_part_of_env(template): + full_prefix = "/".join(prefix) + for template in self.environment.loader.list_templates(): # type: ignore + if template.startswith(full_prefix) and self.is_part_of_env(template): yield template - def walk_templates(self, subdir): + def walk_templates(self, subdir: str) -> Iterable[str]: """ Iterate on the template files from `templates/`. @@ -75,7 +77,7 @@ class Renderer: """ yield from self.iter_templates_in(subdir + "/") - def is_part_of_env(self, path): + def is_part_of_env(self, path: str) -> bool: """ Determines whether a template should be rendered or not. Note that here we don't rely on the OS separator, as we are handling templates @@ -91,7 +93,7 @@ class Renderer: is_excluded = is_excluded or ignore_folder in parts return not is_excluded - def find_os_path(self, template_name): + def find_os_path(self, template_name: str) -> str: path = template_name.replace("/", os.sep) for templates_root in self.template_roots: full_path = os.path.join(templates_root, path) @@ -99,7 +101,7 @@ class Renderer: return full_path raise ValueError("Template path does not exist") - def patch(self, name, separator="\n", suffix=""): + def patch(self, name: str, separator: str = "\n", suffix: str = "") -> str: """ Render calls to {{ patch("...") }} in environment templates from plugin patches. """ @@ -119,11 +121,11 @@ class Renderer: rendered += suffix return rendered - def render_str(self, text): + def render_str(self, text: str) -> str: template = self.environment.from_string(text) return self.__render(template) - def render_template(self, template_name): + def render_template(self, template_name: str) -> Union[str, bytes]: """ Render a template file. Return the corresponding string. If it's a binary file (as indicated by its path), return bytes. @@ -151,7 +153,7 @@ class Renderer: fmt.echo_error("Unknown error rendering template " + template_name) raise - def render_all_to(self, root, *prefix): + def render_all_to(self, root: str, *prefix: str) -> None: """ `prefix` can be used to limit the templates to render. """ @@ -160,7 +162,7 @@ class Renderer: dst = os.path.join(root, template_name.replace("/", os.sep)) write_to(rendered, dst) - def __render(self, template): + def __render(self, template: jinja2.Template) -> str: try: return template.render(**self.config) except jinja2.exceptions.UndefinedError as e: @@ -169,7 +171,7 @@ class Renderer: ) -def save(root, config): +def save(root: str, config: Dict[str, Any]) -> None: """ Save the full environment, including version information. """ @@ -195,7 +197,7 @@ def save(root, config): fmt.echo_info("Environment generated in {}".format(base_dir(root))) -def upgrade_obsolete(root): +def upgrade_obsolete(root: str) -> None: # tutor.conf was renamed to _tutor.conf in order to be the first config file loaded # by nginx nginx_tutor_conf = pathjoin(root, "apps", "nginx", "tutor.conf") @@ -203,7 +205,9 @@ def upgrade_obsolete(root): os.remove(nginx_tutor_conf) -def save_plugin_templates(plugin, root, config): +def save_plugin_templates( + plugin: plugins.BasePlugin, root: str, config: Dict[str, Any] +) -> None: """ Save plugin templates to plugins//*. Only the "apps" and "build" subfolders are rendered. @@ -214,7 +218,7 @@ def save_plugin_templates(plugin, root, config): save_all_from(subdir_path, plugins_root, config) -def save_all_from(prefix, root, config): +def save_all_from(prefix: str, root: str, config: Dict[str, Any]) -> None: """ Render the templates that start with `prefix` and store them with the same hierarchy at `root`. Here, `prefix` can be the result of os.path.join(...). @@ -223,23 +227,20 @@ def save_all_from(prefix, root, config): renderer.render_all_to(root, prefix.replace(os.sep, "/")) -def write_to(content, path): +def write_to(content: Union[str, bytes], path: str) -> None: """ Write some content to a path. Content can be either str or bytes. """ - open_kwargs = {"mode": "w"} - if isinstance(content, bytes): - open_kwargs["mode"] += "b" - else: - # Make files readable by Docker on Windows - open_kwargs["encoding"] = "utf8" - open_kwargs["newline"] = "\n" utils.ensure_file_directory_exists(path) - with open(path, **open_kwargs) as of: - of.write(content) + if isinstance(content, bytes): + with open(path, mode="wb") as of_binary: + of_binary.write(content) + else: + with open(path, mode="w", encoding="utf8", newline="\n") as of_text: + of_text.write(content) -def render_file(config, *path): +def render_file(config: Dict[str, Any], *path: str) -> Union[str, bytes]: """ Return the rendered contents of a template. """ @@ -248,7 +249,7 @@ def render_file(config, *path): return renderer.render_template(template_name) -def render_dict(config): +def render_dict(config: Dict[str, Any]) -> None: """ Render the values from the dict. This is useful for rendering the default values from config.yml. @@ -266,13 +267,13 @@ def render_dict(config): config[k] = v -def render_unknown(config, value): +def render_unknown(config: Dict[str, Any], value: Any) -> Any: if isinstance(value, str): return render_str(config, value) return value -def render_str(config, text): +def render_str(config: Dict[str, Any], text: str) -> str: """ Args: text (str) @@ -284,7 +285,7 @@ def render_str(config, text): return Renderer.instance(config).render_str(text) -def check_is_up_to_date(root): +def check_is_up_to_date(root: str) -> None: if not is_up_to_date(root): message = ( "The current environment stored at {} is not up-to-date: it is at " @@ -298,14 +299,14 @@ def check_is_up_to_date(root): ) -def is_up_to_date(root): +def is_up_to_date(root: str) -> bool: """ Check if the currently rendered version is equal to the current tutor version. """ return current_version(root) == __version__ -def needs_major_upgrade(root): +def needs_major_upgrade(root: str) -> bool: """ Return the current version as a tuple of int. E.g: (1, 0, 2). """ @@ -314,7 +315,7 @@ def needs_major_upgrade(root): return 0 < current < required -def current_release(root): +def current_release(root: str) -> str: """ Return the name of the current Open edX release. """ @@ -323,7 +324,7 @@ def current_release(root): ] -def current_version(root): +def current_version(root: str) -> str: """ Return the current environment version. If the current environment has no version, return "0.0.0". @@ -334,7 +335,7 @@ def current_version(root): return open(path).read().strip() -def read_template_file(*path): +def read_template_file(*path: str) -> str: """ Read raw content of template located at `path`. """ @@ -343,40 +344,40 @@ def read_template_file(*path): return fi.read() -def is_binary_file(path): +def is_binary_file(path: str) -> bool: ext = os.path.splitext(path)[1] return ext in BIN_FILE_EXTENSIONS -def template_path(*path, templates_root=TEMPLATES_ROOT): +def template_path(*path: str, templates_root: str = TEMPLATES_ROOT) -> str: """ Return the template file's absolute path. """ return os.path.join(templates_root, *path) -def data_path(root, *path): +def data_path(root: str, *path: str) -> str: """ Return the file's absolute path inside the data directory. """ return os.path.join(root_dir(root), "data", *path) -def pathjoin(root, *path): +def pathjoin(root: str, *path: str) -> str: """ Return the file's absolute path inside the environment. """ return os.path.join(base_dir(root), *path) -def base_dir(root): +def base_dir(root: str) -> str: """ Return the environment base directory. """ return os.path.join(root_dir(root), "env") -def root_dir(root): +def root_dir(root: str) -> str: """ Return the project root directory. """ diff --git a/tutor/fmt.py b/tutor/fmt.py index 76f6bbb..bffcb8a 100644 --- a/tutor/fmt.py +++ b/tutor/fmt.py @@ -3,7 +3,7 @@ import click STDOUT = None -def title(text): +def title(text: str) -> str: indent = 8 separator = "=" * (len(text) + 2 * indent) message = "{separator}\n{indent}{text}\n{separator}".format( @@ -12,37 +12,37 @@ def title(text): return click.style(message, fg="green") -def echo_info(text): +def echo_info(text: str) -> None: echo(info(text)) -def info(text): +def info(text: str) -> str: return click.style(text, fg="blue") -def error(text): +def error(text: str) -> str: return click.style(text, fg="red") -def echo_error(text): +def echo_error(text: str) -> None: echo(error(text), err=True) -def command(text): +def command(text: str) -> str: return click.style(text, fg="magenta") -def question(text): +def question(text: str) -> str: return click.style(text, fg="yellow") -def echo_alert(text): +def echo_alert(text: str) -> None: echo_error(alert(text)) -def alert(text): +def alert(text: str) -> str: return click.style("⚠️ " + text, fg="yellow", bold=True) -def echo(text, err=False): +def echo(text: str, err: bool = False) -> None: click.echo(text, file=STDOUT, err=err) diff --git a/tutor/images.py b/tutor/images.py index fb412b9..19f0aa3 100644 --- a/tutor/images.py +++ b/tutor/images.py @@ -1,21 +1,23 @@ +from typing import Any, Dict + from . import fmt from . import utils -def get_tag(config, name): +def get_tag(config: Dict[str, Any], name: str) -> Any: return config["DOCKER_IMAGE_" + name.upper().replace("-", "_")] -def build(path, tag, *args): +def build(path: str, tag: str, *args: str) -> None: fmt.echo_info("Building image {}".format(tag)) utils.docker("build", "-t", tag, *args, path) -def pull(tag): +def pull(tag: str) -> None: fmt.echo_info("Pulling image {}".format(tag)) utils.docker("pull", tag) -def push(tag): +def push(tag: str) -> None: fmt.echo_info("Pushing image {}".format(tag)) utils.docker("push", tag) diff --git a/tutor/interactive.py b/tutor/interactive.py index e9ce268..79681e5 100644 --- a/tutor/interactive.py +++ b/tutor/interactive.py @@ -1,3 +1,4 @@ +from typing import Any, Dict, List, Tuple import click from . import config as tutor_config @@ -7,7 +8,7 @@ from . import fmt from .__about__ import __version__ -def update(root, interactive=True): +def update(root: str, interactive: bool = True) -> Dict[str, Any]: """ Load and save the configuration. """ @@ -17,7 +18,9 @@ def update(root, interactive=True): return config -def load_all(root, interactive=True): +def load_all( + root: str, interactive: bool = True +) -> Tuple[Dict[str, Any], Dict[str, Any]]: """ Load configuration and interactively ask questions to collect param values from the user. """ @@ -27,7 +30,7 @@ def load_all(root, interactive=True): return config, defaults -def ask_questions(config, defaults): +def ask_questions(config: Dict[str, Any], defaults: Dict[str, Any]) -> None: run_for_prod = config.get("LMS_HOST") != "local.overhang.io" run_for_prod = click.confirm( fmt.question( @@ -156,21 +159,31 @@ def ask_questions(config, defaults): ) -def ask(question, key, config, defaults): +def ask( + question: str, key: str, config: Dict[str, Any], defaults: Dict[str, Any] +) -> None: default = env.render_str(config, config.get(key, defaults[key])) config[key] = click.prompt( fmt.question(question), prompt_suffix=" ", default=default, show_default=True ) -def ask_bool(question, key, config, defaults): +def ask_bool( + question: str, key: str, config: Dict[str, Any], defaults: Dict[str, Any] +) -> None: default = config.get(key, defaults[key]) config[key] = click.confirm( fmt.question(question), prompt_suffix=" ", default=default ) -def ask_choice(question, key, config, defaults, choices): +def ask_choice( + question: str, + key: str, + config: Dict[str, Any], + defaults: Dict[str, Any], + choices: List[str], +) -> None: default = config.get(key, defaults[key]) answer = click.prompt( fmt.question(question), diff --git a/tutor/jobs.py b/tutor/jobs.py index e12a9ea..4910718 100644 --- a/tutor/jobs.py +++ b/tutor/jobs.py @@ -1,3 +1,5 @@ +from typing import Any, Dict, Iterator, List, Optional, Tuple, Union + from . import env from . import fmt from . import plugins @@ -9,25 +11,30 @@ echo "Loading settings $DJANGO_SETTINGS_MODULE" class BaseJobRunner: - def __init__(self, root, config): + def __init__(self, root: str, config: Dict[str, Any]): self.root = root self.config = config - def run_job_from_template(self, service, *path): + def run_job_from_template(self, service: str, *path: str) -> None: command = self.render(*path) self.run_job(service, command) - def render(self, *path): - return env.render_file(self.config, *path).strip() + def render(self, *path: str) -> str: + rendered = env.render_file(self.config, *path).strip() + if isinstance(rendered, bytes): + raise TypeError("Cannot load job from binary file") + return rendered - def run_job(self, service, command): + def run_job(self, service: str, command: str) -> int: raise NotImplementedError - def iter_plugin_hooks(self, hook): + def iter_plugin_hooks( + self, hook: str + ) -> Iterator[Tuple[str, Union[Dict[str, str], List[str]]]]: yield from plugins.iter_hooks(self.config, hook) -def initialise(runner, limit_to=None): +def initialise(runner: BaseJobRunner, limit_to: Optional[str] = None) -> None: fmt.echo_info("Initialising all services...") if limit_to is None or limit_to == "mysql": runner.run_job_from_template("mysql", "hooks", "mysql", "init") @@ -60,7 +67,13 @@ def initialise(runner, limit_to=None): fmt.echo_info("All services initialised.") -def create_user_command(superuser, staff, username, email, password=None): +def create_user_command( + superuser: str, + staff: bool, + username: str, + email: str, + password: Optional[str] = None, +) -> str: command = BASE_OPENEDX_COMMAND opts = "" @@ -86,11 +99,11 @@ u.save()" return command.format(opts=opts, username=username, email=email, password=password) -def import_demo_course(runner): +def import_demo_course(runner: BaseJobRunner) -> None: runner.run_job_from_template("cms", "hooks", "cms", "importdemocourse") -def set_theme(theme_name, domain_name, runner): +def set_theme(theme_name: str, domain_name: str, runner: BaseJobRunner) -> None: command = BASE_OPENEDX_COMMAND command += """ echo "Assigning theme {theme_name} to {domain_name}..." diff --git a/tutor/plugins.py b/tutor/plugins.py index 52e5894..d343898 100644 --- a/tutor/plugins.py +++ b/tutor/plugins.py @@ -3,9 +3,10 @@ from copy import deepcopy from glob import glob import importlib import os -import pkg_resources +from typing import cast, Any, Dict, Iterator, List, Optional, Tuple, Type, Union import appdirs +import pkg_resources from . import exceptions from . import fmt @@ -47,41 +48,50 @@ class BasePlugin: `command` (click.Command): if a plugin exposes a `command` attribute, users will be able to run it from the command line as `tutor pluginname`. """ - INSTALLED = [] + INSTALLED: List["BasePlugin"] = [] _IS_LOADED = False - def __init__(self, name, obj): + def __init__(self, name: str, obj: Any) -> None: self.name = name - self.config = get_callable_attr(obj, "config", {}) - self.patches = get_callable_attr(obj, "patches", default={}) - self.hooks = get_callable_attr(obj, "hooks", default={}) - self.templates_root = get_callable_attr(obj, "templates", default=None) + self.config = cast( + Dict[str, Dict[str, Any]], get_callable_attr(obj, "config", {}) + ) + self.patches = cast( + Dict[str, str], get_callable_attr(obj, "patches", default={}) + ) + self.hooks = cast( + Dict[str, Union[Dict[str, str], List[str]]], + get_callable_attr(obj, "hooks", default={}), + ) + self.templates_root = cast( + Optional[str], get_callable_attr(obj, "templates", default=None) + ) self.command = getattr(obj, "command", None) - def config_key(self, key): + def config_key(self, key: str) -> str: """ Config keys in the "add" and "defaults" dicts should be prefixed by the plugin name, in uppercase. """ return self.name.upper() + "_" + key @property - def config_add(self): + def config_add(self) -> Dict[str, Any]: return self.config.get("add", {}) @property - def config_set(self): + def config_set(self) -> Dict[str, Any]: return self.config.get("set", {}) @property - def config_defaults(self): + def config_defaults(self) -> Dict[str, Any]: return self.config.get("defaults", {}) @property - def version(self): + def version(self) -> str: raise NotImplementedError @classmethod - def iter_installed(cls): + def iter_installed(cls) -> Iterator["BasePlugin"]: if not cls._IS_LOADED: for plugin in cls.iter_load(): cls.INSTALLED.append(plugin) @@ -89,7 +99,7 @@ class BasePlugin: yield from cls.INSTALLED @classmethod - def iter_load(cls): + def iter_load(cls) -> Iterator["BasePlugin"]: raise NotImplementedError @@ -103,16 +113,18 @@ class EntrypointPlugin(BasePlugin): ENTRYPOINT = "tutor.plugin.v0" - def __init__(self, entrypoint): + def __init__(self, entrypoint: pkg_resources.EntryPoint) -> None: super().__init__(entrypoint.name, entrypoint.load()) self.entrypoint = entrypoint @property - def version(self): + def version(self) -> str: + if not self.entrypoint.dist: + return "0.0.0" return self.entrypoint.dist.version @classmethod - def iter_load(cls): + def iter_load(cls) -> Iterator["EntrypointPlugin"]: for entrypoint in pkg_resources.iter_entry_points(cls.ENTRYPOINT): yield cls(entrypoint) @@ -124,21 +136,24 @@ class OfficialPlugin(BasePlugin): """ @classmethod - def load(cls, name): + def load(cls, name: str) -> BasePlugin: plugin = cls(name) cls.INSTALLED.append(plugin) return plugin - def __init__(self, name): + def __init__(self, name: str): self.module = importlib.import_module("tutor{}.plugin".format(name)) super().__init__(name, self.module) @property - def version(self): - return self.module.__version__ + def version(self) -> str: + version = getattr(self.module, "__version__") + if not isinstance(version, str): + raise TypeError("OfficialPlugin __version__ must be 'str'") + return version @classmethod - def iter_load(cls): + def iter_load(cls) -> Iterator[BasePlugin]: yield from [] @@ -148,18 +163,20 @@ class DictPlugin(BasePlugin): os.environ.get(ROOT_ENV_VAR_NAME, "") ) or appdirs.user_data_dir(appname="tutor-plugins") - def __init__(self, data): - Module = namedtuple("Module", data.keys()) - obj = Module(**data) + def __init__(self, data: Dict[str, Any]): + Module = namedtuple("Module", data.keys()) # type: ignore + obj = Module(**data) # type: ignore super().__init__(data["name"], obj) self._version = data["version"] @property - def version(self): + def version(self) -> str: + if not isinstance(self._version, str): + raise TypeError("DictPlugin.__version__ must be str") return self._version @classmethod - def iter_load(cls): + def iter_load(cls) -> Iterator[BasePlugin]: for path in glob(os.path.join(cls.ROOT, "*.yml")): with open(path) as f: data = serialize.load(f) @@ -176,13 +193,17 @@ class DictPlugin(BasePlugin): class Plugins: - PLUGIN_CLASSES = [OfficialPlugin, EntrypointPlugin, DictPlugin] + PLUGIN_CLASSES: List[Type[BasePlugin]] = [ + OfficialPlugin, + EntrypointPlugin, + DictPlugin, + ] - def __init__(self, config): + def __init__(self, config: Dict[str, Any]): self.config = deepcopy(config) - self.patches = {} - self.hooks = {} - self.template_roots = {} + self.patches: Dict[str, Dict[str, str]] = {} + self.hooks: Dict[str, Dict[str, Union[Dict[str, str], List[str]]]] = {} + self.template_roots: Dict[str, str] = {} for plugin in self.iter_enabled(): for patch_name, content in plugin.patches.items(): @@ -196,12 +217,12 @@ class Plugins: self.hooks[hook_name][plugin.name] = services @classmethod - def clear(cls): + def clear(cls) -> None: for PluginClass in cls.PLUGIN_CLASSES: PluginClass.INSTALLED.clear() @classmethod - def iter_installed(cls): + def iter_installed(cls) -> Iterator[BasePlugin]: """ Iterate on all installed plugins. Plugins are deduplicated by name. The list of installed plugins is cached to prevent too many re-computations, which happens a lot. @@ -213,40 +234,44 @@ class Plugins: installed_plugin_names.add(plugin.name) yield plugin - def iter_enabled(self): + def iter_enabled(self) -> Iterator[BasePlugin]: for plugin in self.iter_installed(): if is_enabled(self.config, plugin.name): yield plugin - def iter_patches(self, name): + def iter_patches(self, name: str) -> Iterator[Tuple[str, str]]: plugin_patches = self.patches.get(name, {}) plugins = sorted(plugin_patches.keys()) for plugin in plugins: yield plugin, plugin_patches[plugin] - def iter_hooks(self, hook_name): + def iter_hooks( + self, hook_name: str + ) -> Iterator[Tuple[str, Union[Dict[str, str], List[str]]]]: yield from self.hooks.get(hook_name, {}).items() -def get_callable_attr(plugin, attr_name, default=None): +def get_callable_attr( + plugin: Any, attr_name: str, default: Optional[Any] = None +) -> Optional[Any]: attr = getattr(plugin, attr_name, default) if callable(attr): attr = attr() return attr -def is_installed(name): +def is_installed(name: str) -> bool: for plugin in iter_installed(): if name == plugin.name: return True return False -def iter_installed(): +def iter_installed() -> Iterator[BasePlugin]: yield from Plugins.iter_installed() -def enable(config, name): +def enable(config: Dict[str, Any], name: str) -> None: if not is_installed(name): raise exceptions.TutorError("plugin '{}' is not installed.".format(name)) if is_enabled(config, name): @@ -257,7 +282,7 @@ def enable(config, name): config[CONFIG_KEY].sort() -def disable(config, name): +def disable(config: Dict[str, Any], name: str) -> None: fmt.echo_info("Disabling plugin {}...".format(name)) for plugin in Plugins(config).iter_enabled(): if name == plugin.name: @@ -271,17 +296,19 @@ def disable(config, name): fmt.echo_info(" Plugin disabled") -def iter_enabled(config): +def iter_enabled(config: Dict[str, Any]) -> Iterator[BasePlugin]: yield from Plugins(config).iter_enabled() -def is_enabled(config, name): +def is_enabled(config: Dict[str, Any], name: str) -> bool: return name in config.get(CONFIG_KEY, []) -def iter_patches(config, name): +def iter_patches(config: Dict[str, str], name: str) -> Iterator[Tuple[str, str]]: yield from Plugins(config).iter_patches(name) -def iter_hooks(config, hook_name): +def iter_hooks( + config: Dict[str, Any], hook_name: str +) -> Iterator[Tuple[str, Union[Dict[str, str], List[str]]]]: yield from Plugins(config).iter_hooks(hook_name) diff --git a/tutor/serialize.py b/tutor/serialize.py index 73cb496..9b29d9c 100644 --- a/tutor/serialize.py +++ b/tutor/serialize.py @@ -1,28 +1,27 @@ import re +from typing import cast, Any, Dict, IO, Iterator, Tuple, Union + import yaml +from _io import TextIOWrapper from yaml.parser import ParserError from yaml.scanner import ScannerError import click -def load(stream): - return yaml.load(stream, Loader=yaml.SafeLoader) +def load(stream: Union[str, IO[str]]) -> Dict[str, str]: + return cast(Dict[str, str], yaml.load(stream, Loader=yaml.SafeLoader)) -def load_all(stream): +def load_all(stream: str) -> Iterator[Any]: return yaml.load_all(stream, Loader=yaml.SafeLoader) -def dump(content, fileobj): +def dump(content: Dict[str, str], fileobj: TextIOWrapper) -> None: yaml.dump(content, stream=fileobj, default_flow_style=False) -def dumps(content): - return yaml.dump(content, stream=None, default_flow_style=False) - - -def parse(v): +def parse(v: Union[str, IO[str]]) -> Any: """ Parse a yaml-formatted string. """ @@ -37,7 +36,7 @@ class YamlParamType(click.ParamType): name = "yaml" PARAM_REGEXP = r"(?P[a-zA-Z0-9_-]+)=(?P.*)" - def convert(self, value, param, ctx): + def convert(self, value: str, param: Any, ctx: Any) -> Tuple[str, Any]: match = re.match(self.PARAM_REGEXP, value) if not match: self.fail("'{}' is not of the form 'key=value'.".format(value), param, ctx) diff --git a/tutor/utils.py b/tutor/utils.py index 9705f01..a7cb7cb 100644 --- a/tutor/utils.py +++ b/tutor/utils.py @@ -7,16 +7,17 @@ import string import struct import subprocess import sys +from typing import List, Tuple import click -from Crypto.PublicKey import RSA from Crypto.Protocol.KDF import bcrypt, bcrypt_check +from Crypto.PublicKey import RSA +from Crypto.PublicKey.RSA import RsaKey -from . import exceptions -from . import fmt +from . import exceptions, fmt -def encrypt(text): +def encrypt(text: str) -> str: """ Encrypt some textual content with bcrypt. https://pycryptodome.readthedocs.io/en/latest/src/protocol/kdf.html#bcrypt @@ -26,7 +27,7 @@ def encrypt(text): return bcrypt(text.encode(), 12).decode() -def verify_encrypted(encrypted, text): +def verify_encrypted(encrypted: str, text: str) -> bool: """ Return True/False if the encrypted content corresponds to the unencrypted text. """ @@ -37,7 +38,7 @@ def verify_encrypted(encrypted, text): return False -def ensure_file_directory_exists(path): +def ensure_file_directory_exists(path: str) -> None: """ Create file's base directory if it does not exist. """ @@ -46,17 +47,17 @@ def ensure_file_directory_exists(path): os.makedirs(directory) -def random_string(length): +def random_string(length: int) -> str: return "".join( [random.choice(string.ascii_letters + string.digits) for _ in range(length)] ) -def list_if(services): +def list_if(services: List[Tuple[str, bool]]) -> str: return json.dumps([service[0] for service in services if service[1]]) -def common_domain(d1, d2): +def common_domain(d1: str, d2: str) -> str: """ Return the common domain between two domain names. @@ -73,7 +74,7 @@ def common_domain(d1, d2): return ".".join(common[::-1]) -def reverse_host(domain): +def reverse_host(domain: str) -> str: """ Return the reverse domain name, java-style. @@ -82,7 +83,7 @@ def reverse_host(domain): return ".".join(domain.split(".")[::-1]) -def rsa_private_key(bits=2048): +def rsa_private_key(bits: int = 2048) -> str: """ Export an RSA private key in PEM format. """ @@ -90,20 +91,20 @@ def rsa_private_key(bits=2048): return key.export_key().decode() -def rsa_import_key(key): +def rsa_import_key(key: str) -> RsaKey: """ Import PEM-formatted RSA key and return the corresponding object. """ return RSA.import_key(key.encode()) -def long_to_base64(n): +def long_to_base64(n: int) -> str: """ Borrowed from jwkest.__init__ """ - def long2intarr(long_int): - _bytes = [] + def long2intarr(long_int: int) -> List[int]: + _bytes: List[int] = [] while long_int: long_int, r = divmod(long_int, 256) _bytes.insert(0, r) @@ -117,16 +118,7 @@ def long_to_base64(n): return s.decode("ascii") -def walk_files(path): - """ - Iterate on file paths located in directory. - """ - for dirpath, _, filenames in os.walk(path): - for filename in filenames: - yield os.path.join(dirpath, filename) - - -def is_root(): +def is_root() -> bool: """ Check whether tutor is being run as root/sudo. """ @@ -136,7 +128,7 @@ def is_root(): return get_user_id() == 0 -def get_user_id(): +def get_user_id() -> int: """ Portable way to get user ID. Note: I have no idea if it actually works on windows... """ @@ -146,14 +138,14 @@ def get_user_id(): return os.getuid() -def docker_run(*command): +def docker_run(*command: str) -> int: args = ["run", "--rm"] if is_a_tty(): args.append("-it") return docker(*args, *command) -def docker(*command): +def docker(*command: str) -> int: if shutil.which("docker") is None: raise exceptions.TutorError( "docker is not installed. Please follow instructions from https://docs.docker.com/install/" @@ -161,7 +153,7 @@ def docker(*command): return execute("docker", *command) -def docker_compose(*command): +def docker_compose(*command: str) -> int: if shutil.which("docker-compose") is None: raise exceptions.TutorError( "docker-compose is not installed. Please follow instructions from https://docs.docker.com/compose/install/" @@ -169,7 +161,7 @@ def docker_compose(*command): return execute("docker-compose", *command) -def kubectl(*command): +def kubectl(*command: str) -> int: if shutil.which("kubectl") is None: raise exceptions.TutorError( "kubectl is not installed. Please follow instructions from https://kubernetes.io/docs/tasks/tools/install-kubectl/" @@ -177,7 +169,7 @@ def kubectl(*command): return execute("kubectl", *command) -def is_a_tty(): +def is_a_tty() -> bool: """ Return True if stdin is able to allocate a tty. Tty allocation sometimes cannot be enabled, for instance in cron jobs @@ -185,7 +177,7 @@ def is_a_tty(): return os.isatty(sys.stdin.fileno()) -def execute(*command): +def execute(*command: str) -> int: click.echo(fmt.command(" ".join(command))) with subprocess.Popen(command) as p: try: @@ -204,9 +196,10 @@ def execute(*command): raise exceptions.TutorError( "Command failed with status {}: {}".format(result, " ".join(command)) ) + return result -def check_output(*command): +def check_output(*command: str) -> bytes: click.echo(fmt.command(" ".join(command))) try: return subprocess.check_output(command)