7
0
mirror of https://github.com/ChristianLight/tutor.git synced 2024-05-29 20:30:48 +00:00

refactor: add type annotations

Annotations were generated with pyannotate:
https://github.com/dropbox/pyannotate

We are running in strict mode, which is awesome!

This affects a large part of the code base, which might be an issue for
people running a fork of Tutor. Nonetheless, the behavior should not be
affected. If anything, this process has helped find and resolve a few
type-related bugs. Thus, this is not considered as a breaking change.
This commit is contained in:
Régis Behmo 2021-02-25 09:09:14 +01:00
parent 1d4ab79863
commit 0a670d7ead
35 changed files with 919 additions and 750 deletions

View File

@ -4,6 +4,7 @@ Note: Breaking changes between versions are indicated by "💥".
## Unreleased ## Unreleased
- [Improvement] Annotate types all over the Tutor code base.
- [Bugfix] Fix parsing of YAML CLI arguments that include equal "=" signs. - [Bugfix] Fix parsing of YAML CLI arguments that include equal "=" signs.
- [Bugfix] Fix minor edge case in `long_to_base64` utility function. - [Bugfix] Fix minor edge case in `long_to_base64` utility function.
- [Improvement] Add openedx patches to add settings during build process. - [Improvement] Add openedx patches to add settings during build process.

View File

@ -24,7 +24,7 @@ build-pythonpackage: ## Build a python package ready to upload to pypi
push-pythonpackage: ## Push python packages to pypi push-pythonpackage: ## Push python packages to pypi
twine upload --skip-existing dist/tutor-*.tar.gz twine upload --skip-existing dist/tutor-*.tar.gz
test: test-lint test-unit test-format test-pythonpackage ## Run all tests by decreasing order or priority test: test-lint test-unit test-types test-format test-pythonpackage ## Run all tests by decreasing order or priority
test-format: ## Run code formatting tests test-format: ## Run code formatting tests
black --check --diff $(BLACK_OPTS) black --check --diff $(BLACK_OPTS)
@ -35,6 +35,9 @@ test-lint: ## Run code linting tests
test-unit: ## Run unit tests test-unit: ## Run unit tests
python -m unittest discover tests python -m unittest discover tests
test-types: ## Check type definitions
mypy --exclude=templates --ignore-missing-imports --strict tutor/ tests/
test-pythonpackage: build-pythonpackage ## Test that package can be uploaded to pypi test-pythonpackage: build-pythonpackage ## Test that package can be uploaded to pypi
twine check dist/tutor-openedx-$(shell make version).tar.gz twine check dist/tutor-openedx-$(shell make version).tar.gz

View File

@ -1,6 +1,7 @@
appdirs appdirs
click>=7.0 click>=7.0
click_repl click_repl
mypy
pycryptodome pycryptodome
jinja2>=2.9 jinja2>=2.9
kubernetes kubernetes

View File

@ -30,6 +30,10 @@ kubernetes==12.0.1
# via -r requirements/base.in # via -r requirements/base.in
markupsafe==1.1.1 markupsafe==1.1.1
# via jinja2 # via jinja2
mypy-extensions==0.4.3
# via mypy
mypy==0.812
# via -r requirements/base.in
oauthlib==3.1.0 oauthlib==3.1.0
# via requests-oauthlib # via requests-oauthlib
prompt-toolkit==3.0.14 prompt-toolkit==3.0.14
@ -63,6 +67,10 @@ six==1.15.0
# kubernetes # kubernetes
# python-dateutil # python-dateutil
# websocket-client # websocket-client
typed-ast==1.4.2
# via mypy
typing-extensions==3.7.4.3
# via mypy
urllib3==1.25.11 urllib3==1.25.11
# via # via
# -r requirements/base.in # -r requirements/base.in

View File

@ -53,6 +53,11 @@ idna==2.10
# via # via
# -r requirements/base.txt # -r requirements/base.txt
# requests # requests
importlib-metadata==3.7.0
# via
# keyring
# pyinstaller
# twine
isort==5.7.0 isort==5.7.0
# via pylint # via pylint
jeepney==0.6.0 jeepney==0.6.0
@ -74,7 +79,12 @@ markupsafe==1.1.1
mccabe==0.6.1 mccabe==0.6.1
# via pylint # via pylint
mypy-extensions==0.4.3 mypy-extensions==0.4.3
# via black # via
# -r requirements/base.txt
# black
# mypy
mypy==0.812
# via -r requirements/base.txt
oauthlib==3.1.0 oauthlib==3.1.0
# via # via
# -r requirements/base.txt # -r requirements/base.txt
@ -167,9 +177,17 @@ tqdm==4.56.1
twine==3.3.0 twine==3.3.0
# via -r requirements/dev.in # via -r requirements/dev.in
typed-ast==1.4.2 typed-ast==1.4.2
# via black # via
# -r requirements/base.txt
# astroid
# black
# mypy
typing-extensions==3.7.4.3 typing-extensions==3.7.4.3
# via black # via
# -r requirements/base.txt
# black
# importlib-metadata
# mypy
urllib3==1.25.11 urllib3==1.25.11
# via # via
# -r requirements/base.txt # -r requirements/base.txt
@ -187,6 +205,8 @@ websocket-client==0.57.0
# kubernetes # kubernetes
wrapt==1.12.1 wrapt==1.12.1
# via astroid # via astroid
zipp==3.4.0
# via importlib-metadata
# The following packages are considered to be unsafe in a requirements file: # The following packages are considered to be unsafe in a requirements file:
# pip # pip

View File

@ -51,6 +51,12 @@ markupsafe==1.1.1
# via # via
# -r requirements/base.txt # -r requirements/base.txt
# jinja2 # jinja2
mypy-extensions==0.4.3
# via
# -r requirements/base.txt
# mypy
mypy==0.812
# via -r requirements/base.txt
oauthlib==3.1.0 oauthlib==3.1.0
# via # via
# -r requirements/base.txt # -r requirements/base.txt
@ -128,6 +134,14 @@ sphinxcontrib-qthelp==1.0.3
# via sphinx # via sphinx
sphinxcontrib-serializinghtml==1.1.4 sphinxcontrib-serializinghtml==1.1.4
# via sphinx # via sphinx
typed-ast==1.4.2
# via
# -r requirements/base.txt
# mypy
typing-extensions==3.7.4.3
# via
# -r requirements/base.txt
# mypy
urllib3==1.25.11 urllib3==1.25.11
# via # via
# -r requirements/base.txt # -r requirements/base.txt

View File

@ -5,17 +5,17 @@ from tutor.exceptions import TutorError
class BindMountsTests(unittest.TestCase): class BindMountsTests(unittest.TestCase):
def test_get_name(self): def test_get_name(self) -> None:
self.assertEqual("venv", bindmounts.get_name("/openedx/venv")) self.assertEqual("venv", bindmounts.get_name("/openedx/venv"))
self.assertEqual("venv", bindmounts.get_name("/openedx/venv/")) self.assertEqual("venv", bindmounts.get_name("/openedx/venv/"))
def test_get_name_root_folder(self): def test_get_name_root_folder(self) -> None:
with self.assertRaises(TutorError): with self.assertRaises(TutorError):
bindmounts.get_name("/") bindmounts.get_name("/")
with self.assertRaises(TutorError): with self.assertRaises(TutorError):
bindmounts.get_name("") bindmounts.get_name("")
def test_parse_volumes(self): def test_parse_volumes(self) -> None:
volume_args, non_volume_args = bindmounts.parse_volumes( volume_args, non_volume_args = bindmounts.parse_volumes(
[ [
"run", "run",

View File

@ -1,39 +1,33 @@
from typing import Any, Dict
import unittest import unittest
import unittest.mock from unittest.mock import Mock, patch
import tempfile import tempfile
from tutor import config as tutor_config from tutor import config as tutor_config
from tutor import env
from tutor import interactive from tutor import interactive
class ConfigTests(unittest.TestCase): class ConfigTests(unittest.TestCase):
def setUp(self): def test_version(self) -> None:
# This is necessary to avoid cached mocks
env.Renderer.reset()
def test_version(self):
defaults = tutor_config.load_defaults() defaults = tutor_config.load_defaults()
self.assertNotIn("TUTOR_VERSION", defaults) self.assertNotIn("TUTOR_VERSION", defaults)
def test_merge(self): def test_merge(self) -> None:
config1 = {"x": "y"} config1 = {"x": "y"}
config2 = {"x": "z"} config2 = {"x": "z"}
tutor_config.merge(config1, config2) tutor_config.merge(config1, config2)
self.assertEqual({"x": "y"}, config1) self.assertEqual({"x": "y"}, config1)
def test_merge_render(self): def test_merge_render(self) -> None:
config = {} config: Dict[str, Any] = {}
defaults = tutor_config.load_defaults() defaults = tutor_config.load_defaults()
with unittest.mock.patch.object( with patch.object(tutor_config.utils, "random_string", return_value="abcd"):
tutor_config.utils, "random_string", return_value="abcd"
):
tutor_config.merge(config, defaults) tutor_config.merge(config, defaults)
self.assertEqual("abcd", config["MYSQL_ROOT_PASSWORD"]) self.assertEqual("abcd", config["MYSQL_ROOT_PASSWORD"])
@unittest.mock.patch.object(tutor_config.fmt, "echo") @patch.object(tutor_config.fmt, "echo")
def test_update_twice(self, _): def test_update_twice(self, _: Mock) -> None:
with tempfile.TemporaryDirectory() as root: with tempfile.TemporaryDirectory() as root:
tutor_config.update(root) tutor_config.update(root)
config1 = tutor_config.load_user(root) config1 = tutor_config.load_user(root)
@ -43,27 +37,27 @@ class ConfigTests(unittest.TestCase):
self.assertEqual(config1, config2) self.assertEqual(config1, config2)
@unittest.mock.patch.object(tutor_config.fmt, "echo") @patch.object(tutor_config.fmt, "echo")
def test_removed_entry_is_added_on_save(self, _): def test_removed_entry_is_added_on_save(self, _: Mock) -> None:
with tempfile.TemporaryDirectory() as root: with tempfile.TemporaryDirectory() as root:
with unittest.mock.patch.object( with patch.object(
tutor_config.utils, "random_string" tutor_config.utils, "random_string"
) as mock_random_string: ) as mock_random_string:
mock_random_string.return_value = "abcd" mock_random_string.return_value = "abcd"
config1, _ = tutor_config.load_all(root) config1, _defaults1 = tutor_config.load_all(root)
password1 = config1["MYSQL_ROOT_PASSWORD"] password1 = config1["MYSQL_ROOT_PASSWORD"]
config1.pop("MYSQL_ROOT_PASSWORD") config1.pop("MYSQL_ROOT_PASSWORD")
tutor_config.save_config_file(root, config1) tutor_config.save_config_file(root, config1)
mock_random_string.return_value = "efgh" mock_random_string.return_value = "efgh"
config2, _ = tutor_config.load_all(root) config2, _defaults2 = tutor_config.load_all(root)
password2 = config2["MYSQL_ROOT_PASSWORD"] password2 = config2["MYSQL_ROOT_PASSWORD"]
self.assertEqual("abcd", password1) self.assertEqual("abcd", password1)
self.assertEqual("efgh", password2) self.assertEqual("efgh", password2)
def test_interactive_load_all(self): def test_interactive_load_all(self) -> None:
with tempfile.TemporaryDirectory() as rootdir: with tempfile.TemporaryDirectory() as rootdir:
config, defaults = interactive.load_all(rootdir, interactive=False) config, defaults = interactive.load_all(rootdir, interactive=False)
@ -73,7 +67,7 @@ class ConfigTests(unittest.TestCase):
self.assertEqual("www.myopenedx.com", defaults["LMS_HOST"]) self.assertEqual("www.myopenedx.com", defaults["LMS_HOST"])
self.assertEqual("studio.{{ LMS_HOST }}", defaults["CMS_HOST"]) self.assertEqual("studio.{{ LMS_HOST }}", defaults["CMS_HOST"])
def test_is_service_activated(self): def test_is_service_activated(self) -> None:
config = {"RUN_SERVICE1": True, "RUN_SERVICE2": False} config = {"RUN_SERVICE1": True, "RUN_SERVICE2": False}
self.assertTrue(tutor_config.is_service_activated(config, "service1")) self.assertTrue(tutor_config.is_service_activated(config, "service1"))

View File

@ -1,7 +1,8 @@
import os import os
import tempfile import tempfile
from typing import Any, Dict
import unittest import unittest
import unittest.mock from unittest.mock import patch, Mock
from tutor import config as tutor_config from tutor import config as tutor_config
from tutor import env from tutor import env
@ -10,41 +11,38 @@ from tutor import exceptions
class EnvTests(unittest.TestCase): class EnvTests(unittest.TestCase):
def setUp(self): def test_walk_templates(self) -> None:
env.Renderer.reset()
def test_walk_templates(self):
renderer = env.Renderer({}, [env.TEMPLATES_ROOT]) renderer = env.Renderer({}, [env.TEMPLATES_ROOT])
templates = list(renderer.walk_templates("local")) templates = list(renderer.walk_templates("local"))
self.assertIn("local/docker-compose.yml", templates) self.assertIn("local/docker-compose.yml", templates)
def test_walk_templates_partials_are_ignored(self): def test_walk_templates_partials_are_ignored(self) -> None:
template_name = "apps/openedx/settings/partials/common_all.py" template_name = "apps/openedx/settings/partials/common_all.py"
renderer = env.Renderer({}, [env.TEMPLATES_ROOT], ignore_folders=["partials"]) renderer = env.Renderer({}, [env.TEMPLATES_ROOT], ignore_folders=["partials"])
templates = list(renderer.walk_templates("apps")) templates = list(renderer.walk_templates("apps"))
self.assertIn(template_name, renderer.environment.loader.list_templates()) self.assertIn(template_name, renderer.environment.loader.list_templates()) # type: ignore
self.assertNotIn(template_name, templates) self.assertNotIn(template_name, templates)
def test_is_binary_file(self): def test_is_binary_file(self) -> None:
self.assertTrue(env.is_binary_file("/home/somefile.ico")) self.assertTrue(env.is_binary_file("/home/somefile.ico"))
def test_find_os_path(self): def test_find_os_path(self) -> None:
renderer = env.Renderer({}, [env.TEMPLATES_ROOT]) renderer = env.Renderer({}, [env.TEMPLATES_ROOT])
path = renderer.find_os_path("local/docker-compose.yml") path = renderer.find_os_path("local/docker-compose.yml")
self.assertTrue(os.path.exists(path)) self.assertTrue(os.path.exists(path))
def test_pathjoin(self): def test_pathjoin(self) -> None:
self.assertEqual( self.assertEqual(
"/tmp/env/target/dummy", env.pathjoin("/tmp", "target", "dummy") "/tmp/env/target/dummy", env.pathjoin("/tmp", "target", "dummy")
) )
self.assertEqual("/tmp/env/dummy", env.pathjoin("/tmp", "dummy")) self.assertEqual("/tmp/env/dummy", env.pathjoin("/tmp", "dummy"))
def test_render_str(self): def test_render_str(self) -> None:
self.assertEqual( self.assertEqual(
"hello world", env.render_str({"name": "world"}, "hello {{ name }}") "hello world", env.render_str({"name": "world"}, "hello {{ name }}")
) )
def test_common_domain(self): def test_common_domain(self) -> None:
self.assertEqual( self.assertEqual(
"mydomain.com", "mydomain.com",
env.render_str( env.render_str(
@ -53,64 +51,62 @@ class EnvTests(unittest.TestCase):
), ),
) )
def test_render_str_missing_configuration(self): def test_render_str_missing_configuration(self) -> None:
self.assertRaises(exceptions.TutorError, env.render_str, {}, "hello {{ name }}") self.assertRaises(exceptions.TutorError, env.render_str, {}, "hello {{ name }}")
def test_render_file(self): def test_render_file(self) -> None:
config = {} config: Dict[str, Any] = {}
tutor_config.merge(config, tutor_config.load_defaults()) tutor_config.merge(config, tutor_config.load_defaults())
config["MYSQL_ROOT_PASSWORD"] = "testpassword" config["MYSQL_ROOT_PASSWORD"] = "testpassword"
rendered = env.render_file(config, "hooks", "mysql", "init") rendered = env.render_file(config, "hooks", "mysql", "init")
self.assertIn("testpassword", rendered) self.assertIn("testpassword", rendered)
@unittest.mock.patch.object(tutor_config.fmt, "echo") @patch.object(tutor_config.fmt, "echo")
def test_render_file_missing_configuration(self, _): def test_render_file_missing_configuration(self, _: Mock) -> None:
self.assertRaises( self.assertRaises(
exceptions.TutorError, env.render_file, {}, "local", "docker-compose.yml" exceptions.TutorError, env.render_file, {}, "local", "docker-compose.yml"
) )
def test_save_full(self): def test_save_full(self) -> None:
defaults = tutor_config.load_defaults() defaults = tutor_config.load_defaults()
with tempfile.TemporaryDirectory() as root: with tempfile.TemporaryDirectory() as root:
config = tutor_config.load_current(root, defaults) config = tutor_config.load_current(root, defaults)
tutor_config.merge(config, defaults) tutor_config.merge(config, defaults)
with unittest.mock.patch.object(fmt, "STDOUT"): with patch.object(fmt, "STDOUT"):
env.save(root, config) env.save(root, config)
self.assertTrue( self.assertTrue(
os.path.exists(os.path.join(root, "env", "local", "docker-compose.yml")) os.path.exists(os.path.join(root, "env", "local", "docker-compose.yml"))
) )
def test_save_full_with_https(self): def test_save_full_with_https(self) -> None:
defaults = tutor_config.load_defaults() defaults = tutor_config.load_defaults()
with tempfile.TemporaryDirectory() as root: with tempfile.TemporaryDirectory() as root:
config = tutor_config.load_current(root, defaults) config = tutor_config.load_current(root, defaults)
tutor_config.merge(config, defaults) tutor_config.merge(config, defaults)
config["ENABLE_HTTPS"] = True config["ENABLE_HTTPS"] = True
with unittest.mock.patch.object(fmt, "STDOUT"): with patch.object(fmt, "STDOUT"):
env.save(root, config) env.save(root, config)
with open(os.path.join(root, "env", "apps", "caddy", "Caddyfile")) as f: with open(os.path.join(root, "env", "apps", "caddy", "Caddyfile")) as f:
self.assertIn("www.myopenedx.com {", f.read()) self.assertIn("www.myopenedx.com {", f.read())
def test_patch(self): def test_patch(self) -> None:
patches = {"plugin1": "abcd", "plugin2": "efgh"} patches = {"plugin1": "abcd", "plugin2": "efgh"}
with unittest.mock.patch.object( with patch.object(
env.plugins, "iter_patches", return_value=patches.items() env.plugins, "iter_patches", return_value=patches.items()
) as mock_iter_patches: ) as mock_iter_patches:
rendered = env.render_str({}, '{{ patch("location") }}') rendered = env.render_str({}, '{{ patch("location") }}')
mock_iter_patches.assert_called_once_with({}, "location") mock_iter_patches.assert_called_once_with({}, "location")
self.assertEqual("abcd\nefgh", rendered) self.assertEqual("abcd\nefgh", rendered)
def test_patch_separator_suffix(self): def test_patch_separator_suffix(self) -> None:
patches = {"plugin1": "abcd", "plugin2": "efgh"} patches = {"plugin1": "abcd", "plugin2": "efgh"}
with unittest.mock.patch.object( with patch.object(env.plugins, "iter_patches", return_value=patches.items()):
env.plugins, "iter_patches", return_value=patches.items()
):
rendered = env.render_str( rendered = env.render_str(
{}, '{{ patch("location", separator=",\n", suffix=",") }}' {}, '{{ patch("location", separator=",\n", suffix=",") }}'
) )
self.assertEqual("abcd,\nefgh,", rendered) self.assertEqual("abcd,\nefgh,", rendered)
def test_plugin_templates(self): def test_plugin_templates(self) -> None:
with tempfile.TemporaryDirectory() as plugin_templates: with tempfile.TemporaryDirectory() as plugin_templates:
# Create plugin # Create plugin
plugin1 = env.plugins.DictPlugin( plugin1 = env.plugins.DictPlugin(
@ -132,7 +128,7 @@ class EnvTests(unittest.TestCase):
config = {"ID": "abcd"} config = {"ID": "abcd"}
# Render templates # Render templates
with unittest.mock.patch.object( with patch.object(
env.plugins, env.plugins,
"iter_enabled", "iter_enabled",
return_value=[plugin1], return_value=[plugin1],
@ -153,7 +149,7 @@ class EnvTests(unittest.TestCase):
with open(dst_rendered) as f: with open(dst_rendered) as f:
self.assertEqual("Hello my ID is abcd", f.read()) self.assertEqual("Hello my ID is abcd", f.read())
def test_renderer_is_reset_on_config_change(self): def test_renderer_is_reset_on_config_change(self) -> None:
with tempfile.TemporaryDirectory() as plugin_templates: with tempfile.TemporaryDirectory() as plugin_templates:
plugin1 = env.plugins.DictPlugin( plugin1 = env.plugins.DictPlugin(
{"name": "plugin1", "version": "0", "templates": plugin_templates} {"name": "plugin1", "version": "0", "templates": plugin_templates}
@ -166,10 +162,10 @@ class EnvTests(unittest.TestCase):
f.write("some content") f.write("some content")
# Load env once # Load env once
config = {"PLUGINS": []} config: Dict[str, Any] = {"PLUGINS": []}
env1 = env.Renderer.instance(config).environment env1 = env.Renderer.instance(config).environment
with unittest.mock.patch.object( with patch.object(
env.plugins, env.plugins,
"iter_enabled", "iter_enabled",
return_value=[plugin1], return_value=[plugin1],
@ -178,5 +174,5 @@ class EnvTests(unittest.TestCase):
config["PLUGINS"].append("myplugin") config["PLUGINS"].append("myplugin")
env2 = env.Renderer.instance(config).environment env2 = env.Renderer.instance(config).environment
self.assertNotIn("plugin1/myplugin.txt", env1.loader.list_templates()) self.assertNotIn("plugin1/myplugin.txt", env1.loader.list_templates()) # type: ignore
self.assertIn("plugin1/myplugin.txt", env2.loader.list_templates()) self.assertIn("plugin1/myplugin.txt", env2.loader.list_templates()) # type: ignore

View File

@ -3,7 +3,7 @@ from tutor import images
class ImagesTests(unittest.TestCase): class ImagesTests(unittest.TestCase):
def test_get_tag(self): def test_get_tag(self) -> None:
config = { config = {
"DOCKER_IMAGE_OPENEDX": "registry/openedx", "DOCKER_IMAGE_OPENEDX": "registry/openedx",
"DOCKER_IMAGE_OPENEDX_DEV": "registry/openedxdev", "DOCKER_IMAGE_OPENEDX_DEV": "registry/openedxdev",

View File

@ -1,5 +1,6 @@
from typing import Any, Dict
import unittest import unittest
from unittest.mock import patch from unittest.mock import Mock, patch
from tutor import config as tutor_config from tutor import config as tutor_config
from tutor import exceptions from tutor import exceptions
@ -8,22 +9,22 @@ from tutor import plugins
class PluginsTests(unittest.TestCase): class PluginsTests(unittest.TestCase):
def setUp(self): def setUp(self) -> None:
plugins.Plugins.clear() plugins.Plugins.clear()
@patch.object(plugins.DictPlugin, "iter_installed", return_value=[]) @patch.object(plugins.DictPlugin, "iter_installed", return_value=[])
def test_iter_installed(self, _dict_plugin_iter_installed): def test_iter_installed(self, _dict_plugin_iter_installed: Mock) -> None:
with patch.object(plugins.pkg_resources, "iter_entry_points", return_value=[]): with patch.object(plugins.pkg_resources, "iter_entry_points", return_value=[]): # type: ignore
self.assertEqual([], list(plugins.iter_installed())) self.assertEqual([], list(plugins.iter_installed()))
def test_is_installed(self): def test_is_installed(self) -> None:
self.assertFalse(plugins.is_installed("dummy")) self.assertFalse(plugins.is_installed("dummy"))
@patch.object(plugins.DictPlugin, "iter_installed", return_value=[]) @patch.object(plugins.DictPlugin, "iter_installed", return_value=[])
def test_official_plugins(self, _dict_plugin_iter_installed): def test_official_plugins(self, _dict_plugin_iter_installed: Mock) -> None:
with patch.object(plugins.importlib, "import_module", return_value=42): with patch.object(plugins.importlib, "import_module", return_value=42): # type: ignore
plugin1 = plugins.OfficialPlugin.load("plugin1") plugin1 = plugins.OfficialPlugin.load("plugin1")
with patch.object(plugins.importlib, "import_module", return_value=43): with patch.object(plugins.importlib, "import_module", return_value=43): # type: ignore
plugin2 = plugins.OfficialPlugin.load("plugin2") plugin2 = plugins.OfficialPlugin.load("plugin2")
with patch.object( with patch.object(
plugins.EntrypointPlugin, plugins.EntrypointPlugin,
@ -35,32 +36,32 @@ class PluginsTests(unittest.TestCase):
list(plugins.iter_installed()), list(plugins.iter_installed()),
) )
def test_enable(self): def test_enable(self) -> None:
config = {plugins.CONFIG_KEY: []} config: Dict[str, Any] = {plugins.CONFIG_KEY: []}
with patch.object(plugins, "is_installed", return_value=True): with patch.object(plugins, "is_installed", return_value=True):
plugins.enable(config, "plugin2") plugins.enable(config, "plugin2")
plugins.enable(config, "plugin1") plugins.enable(config, "plugin1")
self.assertEqual(["plugin1", "plugin2"], config[plugins.CONFIG_KEY]) self.assertEqual(["plugin1", "plugin2"], config[plugins.CONFIG_KEY])
def test_enable_twice(self): def test_enable_twice(self) -> None:
config = {plugins.CONFIG_KEY: []} config: Dict[str, Any] = {plugins.CONFIG_KEY: []}
with patch.object(plugins, "is_installed", return_value=True): with patch.object(plugins, "is_installed", return_value=True):
plugins.enable(config, "plugin1") plugins.enable(config, "plugin1")
plugins.enable(config, "plugin1") plugins.enable(config, "plugin1")
self.assertEqual(["plugin1"], config[plugins.CONFIG_KEY]) self.assertEqual(["plugin1"], config[plugins.CONFIG_KEY])
def test_enable_not_installed_plugin(self): def test_enable_not_installed_plugin(self) -> None:
config = {"PLUGINS": []} config: Dict[str, Any] = {"PLUGINS": []}
with patch.object(plugins, "is_installed", return_value=False): with patch.object(plugins, "is_installed", return_value=False):
self.assertRaises(exceptions.TutorError, plugins.enable, config, "plugin1") self.assertRaises(exceptions.TutorError, plugins.enable, config, "plugin1")
def test_disable(self): def test_disable(self) -> None:
config = {"PLUGINS": ["plugin1", "plugin2"]} config: Dict[str, Any] = {"PLUGINS": ["plugin1", "plugin2"]}
with patch.object(fmt, "STDOUT"): with patch.object(fmt, "STDOUT"):
plugins.disable(config, "plugin1") plugins.disable(config, "plugin1")
self.assertEqual(["plugin2"], config["PLUGINS"]) self.assertEqual(["plugin2"], config["PLUGINS"])
def test_disable_removes_set_config(self): def test_disable_removes_set_config(self) -> None:
with patch.object( with patch.object(
plugins.Plugins, plugins.Plugins,
"iter_enabled", "iter_enabled",
@ -80,7 +81,7 @@ class PluginsTests(unittest.TestCase):
self.assertEqual([], config["PLUGINS"]) self.assertEqual([], config["PLUGINS"])
self.assertNotIn("KEY", config) self.assertNotIn("KEY", config)
def test_patches(self): def test_patches(self) -> None:
class plugin1: class plugin1:
patches = {"patch1": "Hello {{ ID }}"} patches = {"patch1": "Hello {{ ID }}"}
@ -92,7 +93,7 @@ class PluginsTests(unittest.TestCase):
patches = list(plugins.iter_patches({}, "patch1")) patches = list(plugins.iter_patches({}, "patch1"))
self.assertEqual([("plugin1", "Hello {{ ID }}")], patches) self.assertEqual([("plugin1", "Hello {{ ID }}")], patches)
def test_plugin_without_patches(self): def test_plugin_without_patches(self) -> None:
with patch.object( with patch.object(
plugins.Plugins, plugins.Plugins,
"iter_enabled", "iter_enabled",
@ -101,9 +102,9 @@ class PluginsTests(unittest.TestCase):
patches = list(plugins.iter_patches({}, "patch1")) patches = list(plugins.iter_patches({}, "patch1"))
self.assertEqual([], patches) self.assertEqual([], patches)
def test_configure(self): def test_configure(self) -> None:
config = {"ID": "id"} config = {"ID": "id"}
defaults = {} defaults: Dict[str, Any] = {}
class plugin1: class plugin1:
config = { config = {
@ -130,7 +131,7 @@ class PluginsTests(unittest.TestCase):
) )
self.assertEqual({"PLUGIN1_PARAM4": "value4"}, defaults) self.assertEqual({"PLUGIN1_PARAM4": "value4"}, defaults)
def test_configure_set_does_not_override(self): def test_configure_set_does_not_override(self) -> None:
config = {"ID": "oldid"} config = {"ID": "oldid"}
class plugin1: class plugin1:
@ -145,8 +146,8 @@ class PluginsTests(unittest.TestCase):
self.assertEqual({"ID": "oldid"}, config) self.assertEqual({"ID": "oldid"}, config)
def test_configure_set_random_string(self): def test_configure_set_random_string(self) -> None:
config = {} config: Dict[str, Any] = {}
class plugin1: class plugin1:
config = {"set": {"PARAM1": "{{ 128|random_string }}"}} config = {"set": {"PARAM1": "{{ 128|random_string }}"}}
@ -159,8 +160,8 @@ class PluginsTests(unittest.TestCase):
tutor_config.load_plugins(config, {}) tutor_config.load_plugins(config, {})
self.assertEqual(128, len(config["PARAM1"])) self.assertEqual(128, len(config["PARAM1"]))
def test_configure_default_value_with_previous_definition(self): def test_configure_default_value_with_previous_definition(self) -> None:
config = {} config: Dict[str, Any] = {}
defaults = {"PARAM1": "value"} defaults = {"PARAM1": "value"}
class plugin1: class plugin1:
@ -174,8 +175,8 @@ class PluginsTests(unittest.TestCase):
tutor_config.load_plugins(config, defaults) tutor_config.load_plugins(config, defaults)
self.assertEqual("{{ PARAM1 }}", defaults["PLUGIN1_PARAM2"]) self.assertEqual("{{ PARAM1 }}", defaults["PLUGIN1_PARAM2"])
def test_configure_add_twice(self): def test_configure_add_twice(self) -> None:
config = {} config: Dict[str, Any] = {}
class plugin1: class plugin1:
config = {"add": {"PARAM1": "{{ 10|random_string }}"}} config = {"add": {"PARAM1": "{{ 10|random_string }}"}}
@ -199,7 +200,7 @@ class PluginsTests(unittest.TestCase):
self.assertEqual(10, len(value2)) self.assertEqual(10, len(value2))
self.assertEqual(value1, value2) self.assertEqual(value1, value2)
def test_hooks(self): def test_hooks(self) -> None:
class plugin1: class plugin1:
hooks = {"init": ["myclient"]} hooks = {"init": ["myclient"]}
@ -212,8 +213,8 @@ class PluginsTests(unittest.TestCase):
[("plugin1", ["myclient"])], list(plugins.iter_hooks({}, "init")) [("plugin1", ["myclient"])], list(plugins.iter_hooks({}, "init"))
) )
def test_plugins_are_updated_on_config_change(self): def test_plugins_are_updated_on_config_change(self) -> None:
config = {"PLUGINS": []} config: Dict[str, Any] = {"PLUGINS": []}
plugins1 = plugins.Plugins(config) plugins1 = plugins.Plugins(config)
self.assertEqual(0, len(list(plugins1.iter_enabled()))) self.assertEqual(0, len(list(plugins1.iter_enabled())))
config["PLUGINS"].append("plugin1") config["PLUGINS"].append("plugin1")
@ -225,7 +226,7 @@ class PluginsTests(unittest.TestCase):
plugins2 = plugins.Plugins(config) plugins2 = plugins.Plugins(config)
self.assertEqual(1, len(list(plugins2.iter_enabled()))) self.assertEqual(1, len(list(plugins2.iter_enabled())))
def test_dict_plugin(self): def test_dict_plugin(self) -> None:
plugin = plugins.DictPlugin( plugin = plugins.DictPlugin(
{"name": "myplugin", "config": {"set": {"KEY": "value"}}, "version": "0.1"} {"name": "myplugin", "config": {"set": {"KEY": "value"}}, "version": "0.1"}
) )

View File

@ -6,32 +6,32 @@ from tutor import serialize
class SerializeTests(unittest.TestCase): class SerializeTests(unittest.TestCase):
def test_parse_str(self): def test_parse_str(self) -> None:
self.assertEqual("abcd", serialize.parse("abcd")) self.assertEqual("abcd", serialize.parse("abcd"))
def test_parse_int(self): def test_parse_int(self) -> None:
self.assertEqual(1, serialize.parse("1")) self.assertEqual(1, serialize.parse("1"))
def test_parse_bool(self): def test_parse_bool(self) -> None:
self.assertEqual(True, serialize.parse("true")) self.assertEqual(True, serialize.parse("true"))
self.assertEqual(False, serialize.parse("false")) self.assertEqual(False, serialize.parse("false"))
def test_parse_null(self): def test_parse_null(self) -> None:
self.assertIsNone(serialize.parse("null")) self.assertIsNone(serialize.parse("null"))
def test_parse_invalid_format(self): def test_parse_invalid_format(self) -> None:
self.assertEqual('["abcd"', serialize.parse('["abcd"')) self.assertEqual('["abcd"', serialize.parse('["abcd"'))
def test_parse_list(self): def test_parse_list(self) -> None:
self.assertEqual(["abcd"], serialize.parse('["abcd"]')) self.assertEqual(["abcd"], serialize.parse('["abcd"]'))
def test_parse_weird_chars(self): def test_parse_weird_chars(self) -> None:
self.assertEqual("*@google.com", serialize.parse("*@google.com")) self.assertEqual("*@google.com", serialize.parse("*@google.com"))
def test_parse_empty_string(self): def test_parse_empty_string(self) -> None:
self.assertEqual("", serialize.parse("''")) self.assertEqual("", serialize.parse("''"))
def test_yaml_param_type(self): def test_yaml_param_type(self) -> None:
param = serialize.YamlParamType() param = serialize.YamlParamType()
self.assertEqual(("name", True), param.convert("name=true", "param", {})) self.assertEqual(("name", True), param.convert("name=true", "param", {}))
self.assertEqual(("name", "abcd"), param.convert("name=abcd", "param", {})) self.assertEqual(("name", "abcd"), param.convert("name=abcd", "param", {}))

View File

@ -5,7 +5,7 @@ from tutor import utils
class UtilsTests(unittest.TestCase): class UtilsTests(unittest.TestCase):
def test_common_domain(self): def test_common_domain(self) -> None:
self.assertEqual( self.assertEqual(
"domain.com", utils.common_domain("sub1.domain.com", "sub2.domain.com") "domain.com", utils.common_domain("sub1.domain.com", "sub2.domain.com")
) )
@ -18,13 +18,13 @@ class UtilsTests(unittest.TestCase):
"domain.com", utils.common_domain("sub.domain.com", "ub.domain.com") "domain.com", utils.common_domain("sub.domain.com", "ub.domain.com")
) )
def test_reverse_host(self): def test_reverse_host(self) -> None:
self.assertEqual("com.google.www", utils.reverse_host("www.google.com")) self.assertEqual("com.google.www", utils.reverse_host("www.google.com"))
def test_list_if(self): def test_list_if(self) -> None:
self.assertEqual('["cms"]', utils.list_if([("lms", False), ("cms", True)])) self.assertEqual('["cms"]', utils.list_if([("lms", False), ("cms", True)]))
def test_encrypt_decrypt(self): def test_encrypt_decrypt(self) -> None:
password = "passw0rd" password = "passw0rd"
encrypted1 = utils.encrypt(password) encrypted1 = utils.encrypt(password)
encrypted2 = utils.encrypt(password) encrypted2 = utils.encrypt(password)
@ -32,7 +32,7 @@ class UtilsTests(unittest.TestCase):
self.assertTrue(utils.verify_encrypted(encrypted1, password)) self.assertTrue(utils.verify_encrypted(encrypted1, password))
self.assertTrue(utils.verify_encrypted(encrypted2, password)) self.assertTrue(utils.verify_encrypted(encrypted2, password))
def test_long_to_base64(self): def test_long_to_base64(self) -> None:
self.assertEqual( self.assertEqual(
b"\x00", base64.urlsafe_b64decode(utils.long_to_base64(0) + "==") b"\x00", base64.urlsafe_b64decode(utils.long_to_base64(0) + "==")
) )

View File

@ -1,12 +1,20 @@
import os import os
from typing import Any, Callable, Dict, List, Tuple
import click import click
from mypy_extensions import VarArg
from .exceptions import TutorError from .exceptions import TutorError
from .utils import get_user_id from .utils import get_user_id
def create(root, config, docker_compose_func, service, path): def create(
root: str,
config: Dict[str, Any],
docker_compose_func: Callable[[str, Dict[str, Any], VarArg(str)], int],
service: str,
path: str,
) -> str:
volumes_root_path = get_root_path(root) volumes_root_path = get_root_path(root)
volume_name = get_name(path) volume_name = get_name(path)
container_volumes_root_path = "/tmp/volumes" container_volumes_root_path = "/tmp/volumes"
@ -41,12 +49,12 @@ chown -R {user_id} {volumes_path}/{volume_name}""".format(
return os.path.join(volumes_root_path, volume_name) return os.path.join(volumes_root_path, volume_name)
def get_path(root, container_bind_path): def get_path(root: str, container_bind_path: str) -> str:
bind_basename = get_name(container_bind_path) bind_basename = get_name(container_bind_path)
return os.path.join(get_root_path(root), bind_basename) return os.path.join(get_root_path(root), bind_basename)
def get_name(container_bind_path): def get_name(container_bind_path: str) -> str:
# We rstrip slashes, otherwise os.path.basename returns an empty string # We rstrip slashes, otherwise os.path.basename returns an empty string
# We don't use basename here as it will not work on Windows # We don't use basename here as it will not work on Windows
name = container_bind_path.rstrip("/").split("/")[-1] name = container_bind_path.rstrip("/").split("/")[-1]
@ -55,11 +63,11 @@ def get_name(container_bind_path):
return name return name
def get_root_path(root): def get_root_path(root: str) -> str:
return os.path.join(root, "volumes") return os.path.join(root, "volumes")
def parse_volumes(docker_compose_args): def parse_volumes(docker_compose_args: List[str]) -> Tuple[List[str], List[str]]:
""" """
Parse `-v/--volume` options from an arbitrary list of arguments. Parse `-v/--volume` options from an arbitrary list of arguments.
""" """
@ -67,7 +75,9 @@ def parse_volumes(docker_compose_args):
@click.command(context_settings={"ignore_unknown_options": True}) @click.command(context_settings={"ignore_unknown_options": True})
@click.option("-v", "--volume", "volumes", multiple=True) @click.option("-v", "--volume", "volumes", multiple=True)
@click.argument("args", nargs=-1, required=True) @click.argument("args", nargs=-1, required=True)
def custom_docker_compose(volumes, args): # pylint: disable=unused-argument def custom_docker_compose(
volumes: List[str], args: List[str]
) -> None: # pylint: disable=unused-argument
pass pass
if isinstance(docker_compose_args, tuple): if isinstance(docker_compose_args, tuple):

View File

@ -1,3 +1,5 @@
from typing import Dict
import click import click
from .compose import ComposeJobRunner from .compose import ComposeJobRunner
@ -5,17 +7,18 @@ from .local import docker_compose as local_docker_compose
from .. import config as tutor_config from .. import config as tutor_config
from .. import env as tutor_env from .. import env as tutor_env
from .. import fmt from .. import fmt
from .context import Context
@click.group(help="Build an Android app for your Open edX platform [BETA FEATURE]") @click.group(help="Build an Android app for your Open edX platform [BETA FEATURE]")
def android(): def android() -> None:
pass pass
@click.command(help="Build the application") @click.command(help="Build the application")
@click.argument("mode", type=click.Choice(["debug", "release"])) @click.argument("mode", type=click.Choice(["debug", "release"]))
@click.pass_obj @click.pass_obj
def build(context, mode): def build(context: Context, mode: str) -> None:
config = tutor_config.load(context.root) config = tutor_config.load(context.root)
docker_run(context.root, build_command(config, mode)) docker_run(context.root, build_command(config, mode))
fmt.echo_info( fmt.echo_info(
@ -25,7 +28,7 @@ def build(context, mode):
) )
def build_command(config, target): def build_command(config: Dict[str, str], target: str) -> str:
gradle_target = { gradle_target = {
"debug": "assembleProdDebuggable", "debug": "assembleProdDebuggable",
"release": "assembleProdRelease", "release": "assembleProdRelease",
@ -41,7 +44,7 @@ cp OpenEdXMobile/build/outputs/apk/prod/{apk_folder}/*.apk /openedx/data/"""
return command return command
def docker_run(root, command): def docker_run(root: str, command: str) -> None:
config = tutor_config.load(root) config = tutor_config.load(root)
runner = ComposeJobRunner(root, config, local_docker_compose) runner = ComposeJobRunner(root, config, local_docker_compose)
runner.run_job("android", command) runner.run_job("android", command)

View File

@ -21,7 +21,7 @@ from .. import fmt
from .. import utils from .. import utils
def main(): def main() -> None:
try: try:
click_repl.register_repl(cli, name="ui") click_repl.register_repl(cli, name="ui")
cli.add_command(images_command) cli.add_command(images_command)
@ -55,7 +55,7 @@ def main():
help="Root project directory (environment variable: TUTOR_ROOT)", help="Root project directory (environment variable: TUTOR_ROOT)",
) )
@click.pass_context @click.pass_context
def cli(context, root): def cli(context: click.Context, root: str) -> None:
if utils.is_root(): if utils.is_root():
fmt.echo_alert( fmt.echo_alert(
"You are running Tutor as root. This is strongly not recommended. If you are doing this in order to access" "You are running Tutor as root. This is strongly not recommended. If you are doing this in order to access"
@ -66,9 +66,9 @@ def cli(context, root):
@click.command(help="Print this help", name="help") @click.command(help="Print this help", name="help")
def print_help(): def print_help() -> None:
with click.Context(cli) as context: context = click.Context(cli)
click.echo(cli.get_help(context)) click.echo(cli.get_help(context))
if __name__ == "__main__": if __name__ == "__main__":

View File

@ -1,6 +1,8 @@
import os import os
from typing import Any, Callable, Dict, List
import click import click
from mypy_extensions import VarArg
from .. import bindmounts from .. import bindmounts
from .. import config as tutor_config from .. import config as tutor_config
@ -10,14 +12,20 @@ from .. import fmt
from .. import jobs from .. import jobs
from .. import serialize from .. import serialize
from .. import utils from .. import utils
from .context import Context
class ComposeJobRunner(jobs.BaseJobRunner): class ComposeJobRunner(jobs.BaseJobRunner):
def __init__(self, root, config, docker_compose_func): def __init__(
self,
root: str,
config: Dict[str, Any],
docker_compose_func: Callable[[str, Dict[str, Any], VarArg(str)], int],
):
super().__init__(root, config) super().__init__(root, config)
self.docker_compose_func = docker_compose_func self.docker_compose_func = docker_compose_func
def run_job(self, service, command): def run_job(self, service: str, command: str) -> int:
""" """
Run the "{{ service }}-job" service from local/docker-compose.jobs.yml with the Run the "{{ service }}-job" service from local/docker-compose.jobs.yml with the
specified command. For backward-compatibility reasons, if the corresponding specified command. For backward-compatibility reasons, if the corresponding
@ -28,7 +36,7 @@ class ComposeJobRunner(jobs.BaseJobRunner):
job_service_name = "{}-job".format(service) job_service_name = "{}-job".format(service)
opts = [] if utils.is_a_tty() else ["-T"] opts = [] if utils.is_a_tty() else ["-T"]
if job_service_name in serialize.load(open(jobs_path).read())["services"]: if job_service_name in serialize.load(open(jobs_path).read())["services"]:
self.docker_compose_func( return self.docker_compose_func(
self.root, self.root,
self.config, self.config,
"-f", "-f",
@ -42,44 +50,43 @@ class ComposeJobRunner(jobs.BaseJobRunner):
"-c", "-c",
command, command,
) )
else: fmt.echo_alert(
fmt.echo_alert( (
( "The '{job_service_name}' service does not exist in {jobs_path}. "
"The '{job_service_name}' service does not exist in {jobs_path}. " "This might be caused by an older plugin. Tutor switched to a job "
"This might be caused by an older plugin. Tutor switched to a job " "runner model for running one-time commands, such as database"
"runner model for running one-time commands, such as database" " initialisation. For the record, this is the command that we are "
" initialisation. For the record, this is the command that we are " "running:\n"
"running:\n" "\n"
"\n" " {command}\n"
" {command}\n" "\n"
"\n" "Old-style job running will be deprecated soon. Please inform "
"Old-style job running will be deprecated soon. Please inform " "your plugin maintainer!"
"your plugin maintainer!" ).format(
).format( job_service_name=job_service_name,
job_service_name=job_service_name, jobs_path=jobs_path,
jobs_path=jobs_path, command=command.replace("\n", "\n "),
command=command.replace("\n", "\n "),
)
)
self.docker_compose_func(
self.root,
self.config,
"run",
*opts,
"--rm",
service,
"sh",
"-e",
"-c",
command,
) )
)
return self.docker_compose_func(
self.root,
self.config,
"run",
*opts,
"--rm",
service,
"sh",
"-e",
"-c",
command,
)
@click.command(help="Run all or a selection of configured Open edX services") @click.command(help="Run all or a selection of configured Open edX services")
@click.option("-d", "--detach", is_flag=True, help="Start in daemon mode") @click.option("-d", "--detach", is_flag=True, help="Start in daemon mode")
@click.argument("services", metavar="service", nargs=-1) @click.argument("services", metavar="service", nargs=-1)
@click.pass_obj @click.pass_obj
def start(context, detach, services): def start(context: Context, detach: bool, services: List[str]) -> None:
command = ["up", "--remove-orphans"] command = ["up", "--remove-orphans"]
if detach: if detach:
command.append("-d") command.append("-d")
@ -91,7 +98,7 @@ def start(context, detach, services):
@click.command(help="Stop a running platform") @click.command(help="Stop a running platform")
@click.argument("services", metavar="service", nargs=-1) @click.argument("services", metavar="service", nargs=-1)
@click.pass_obj @click.pass_obj
def stop(context, services): def stop(context: Context, services: List[str]) -> None:
config = tutor_config.load(context.root) config = tutor_config.load(context.root)
context.docker_compose(context.root, config, "stop", *services) context.docker_compose(context.root, config, "stop", *services)
@ -102,9 +109,10 @@ def stop(context, services):
) )
@click.option("-d", "--detach", is_flag=True, help="Start in daemon mode") @click.option("-d", "--detach", is_flag=True, help="Start in daemon mode")
@click.argument("services", metavar="service", nargs=-1) @click.argument("services", metavar="service", nargs=-1)
def reboot(detach, services): @click.pass_context
stop.callback(services) def reboot(context: click.Context, detach: bool, services: List[str]) -> None:
start.callback(detach, services) context.invoke(stop, services=services)
context.invoke(start, detach=detach, services=services)
@click.command( @click.command(
@ -116,7 +124,7 @@ fully stop the platform, use the 'reboot' command.""",
) )
@click.argument("services", metavar="service", nargs=-1) @click.argument("services", metavar="service", nargs=-1)
@click.pass_obj @click.pass_obj
def restart(context, services): def restart(context: Context, services: List[str]) -> None:
config = tutor_config.load(context.root) config = tutor_config.load(context.root)
command = ["restart"] command = ["restart"]
if "all" in services: if "all" in services:
@ -136,7 +144,7 @@ def restart(context, services):
@click.command(help="Initialise all applications") @click.command(help="Initialise all applications")
@click.option("-l", "--limit", help="Limit initialisation to this service or plugin") @click.option("-l", "--limit", help="Limit initialisation to this service or plugin")
@click.pass_obj @click.pass_obj
def init(context, limit): def init(context: Context, limit: str) -> None:
config = tutor_config.load(context.root) config = tutor_config.load(context.root)
runner = ComposeJobRunner(context.root, config, context.docker_compose) runner = ComposeJobRunner(context.root, config, context.docker_compose)
jobs.initialise(runner, limit_to=limit) jobs.initialise(runner, limit_to=limit)
@ -153,7 +161,9 @@ def init(context, limit):
@click.argument("name") @click.argument("name")
@click.argument("email") @click.argument("email")
@click.pass_obj @click.pass_obj
def createuser(context, superuser, staff, password, name, email): def createuser(
context: Context, superuser: str, staff: bool, password: str, name: str, email: str
) -> None:
config = tutor_config.load(context.root) config = tutor_config.load(context.root)
runner = ComposeJobRunner(context.root, config, context.docker_compose) runner = ComposeJobRunner(context.root, config, context.docker_compose)
command = jobs.create_user_command(superuser, staff, name, email, password=password) command = jobs.create_user_command(superuser, staff, name, email, password=password)
@ -166,7 +176,7 @@ def createuser(context, superuser, staff, password, name, email):
@click.argument("theme_name") @click.argument("theme_name")
@click.argument("domain_names", metavar="domain_name", nargs=-1) @click.argument("domain_names", metavar="domain_name", nargs=-1)
@click.pass_obj @click.pass_obj
def settheme(context, theme_name, domain_names): def settheme(context: Context, theme_name: str, domain_names: List[str]) -> None:
config = tutor_config.load(context.root) config = tutor_config.load(context.root)
runner = ComposeJobRunner(context.root, config, context.docker_compose) runner = ComposeJobRunner(context.root, config, context.docker_compose)
for domain_name in domain_names: for domain_name in domain_names:
@ -175,7 +185,7 @@ def settheme(context, theme_name, domain_names):
@click.command(help="Import the demo course") @click.command(help="Import the demo course")
@click.pass_obj @click.pass_obj
def importdemocourse(context): def importdemocourse(context: Context) -> None:
config = tutor_config.load(context.root) config = tutor_config.load(context.root)
runner = ComposeJobRunner(context.root, config, context.docker_compose) runner = ComposeJobRunner(context.root, config, context.docker_compose)
fmt.echo_info("Importing demo course") fmt.echo_info("Importing demo course")
@ -192,11 +202,12 @@ def importdemocourse(context):
context_settings={"ignore_unknown_options": True}, context_settings={"ignore_unknown_options": True},
) )
@click.argument("args", nargs=-1, required=True) @click.argument("args", nargs=-1, required=True)
def run(args): @click.pass_context
def run(context: click.Context, args: List[str]) -> None:
extra_args = ["--rm"] extra_args = ["--rm"]
if not utils.is_a_tty(): if not utils.is_a_tty():
extra_args.append("-T") extra_args.append("-T")
dc_command.callback("run", [*extra_args, *args]) context.invoke(dc_command, command="run", args=[*extra_args, *args])
@click.command( @click.command(
@ -208,7 +219,7 @@ def run(args):
) )
@click.argument("path") @click.argument("path")
@click.pass_obj @click.pass_obj
def bindmount_command(context, service, path): def bindmount_command(context: Context, service: str, path: str) -> None:
config = tutor_config.load(context.root) config = tutor_config.load(context.root)
host_path = bindmounts.create( host_path = bindmounts.create(
context.root, config, context.docker_compose, service, path context.root, config, context.docker_compose, service, path
@ -231,8 +242,9 @@ def bindmount_command(context, service, path):
name="exec", name="exec",
) )
@click.argument("args", nargs=-1, required=True) @click.argument("args", nargs=-1, required=True)
def execute(args): @click.pass_context
dc_command.callback("exec", args) def execute(context: click.Context, args: List[str]) -> None:
context.invoke(dc_command, command="exec", args=args)
@click.command( @click.command(
@ -242,14 +254,15 @@ def execute(args):
@click.option("-f", "--follow", is_flag=True, help="Follow log output") @click.option("-f", "--follow", is_flag=True, help="Follow log output")
@click.option("--tail", type=int, help="Number of lines to show from each container") @click.option("--tail", type=int, help="Number of lines to show from each container")
@click.argument("service", nargs=-1) @click.argument("service", nargs=-1)
def logs(follow, tail, service): @click.pass_context
def logs(context: click.Context, follow: bool, tail: bool, service: str) -> None:
args = [] args = []
if follow: if follow:
args.append("--follow") args.append("--follow")
if tail is not None: if tail is not None:
args += ["--tail", str(tail)] args += ["--tail", str(tail)]
args += service args += service
dc_command.callback("logs", args) context.invoke(dc_command, command="logs", args=args)
@click.command( @click.command(
@ -264,7 +277,7 @@ def logs(follow, tail, service):
@click.argument("command") @click.argument("command")
@click.argument("args", nargs=-1, required=True) @click.argument("args", nargs=-1, required=True)
@click.pass_obj @click.pass_obj
def dc_command(context, command, args): def dc_command(context: Context, command: str, args: List[str]) -> None:
config = tutor_config.load(context.root) config = tutor_config.load(context.root)
volumes, non_volume_args = bindmounts.parse_volumes(args) volumes, non_volume_args = bindmounts.parse_volumes(args)
volume_args = [] volume_args = []
@ -286,7 +299,7 @@ def dc_command(context, command, args):
) )
def add_commands(command_group): def add_commands(command_group: click.Group) -> None:
command_group.add_command(start) command_group.add_command(start)
command_group.add_command(stop) command_group.add_command(stop)
command_group.add_command(restart) command_group.add_command(restart)

View File

@ -1,3 +1,5 @@
from typing import Any, Dict, List
import click import click
from .. import config as tutor_config from .. import config as tutor_config
@ -6,6 +8,7 @@ from .. import exceptions
from .. import fmt from .. import fmt
from .. import interactive as interactive_config from .. import interactive as interactive_config
from .. import serialize from .. import serialize
from .context import Context
@click.group( @click.group(
@ -13,7 +16,7 @@ from .. import serialize
short_help="Configure Open edX", short_help="Configure Open edX",
help="""Configure Open edX and store configuration values in $TUTOR_ROOT/config.yml""", help="""Configure Open edX and store configuration values in $TUTOR_ROOT/config.yml""",
) )
def config_command(): def config_command() -> None:
pass pass
@ -36,7 +39,9 @@ def config_command():
help="Remove a configuration value (can be used multiple times)", help="Remove a configuration value (can be used multiple times)",
) )
@click.pass_obj @click.pass_obj
def save(context, interactive, set_vars, unset_vars): def save(
context: Context, interactive: bool, set_vars: Dict[str, Any], unset_vars: List[str]
) -> None:
config, defaults = interactive_config.load_all( config, defaults = interactive_config.load_all(
context.root, interactive=interactive context.root, interactive=interactive
) )
@ -61,7 +66,7 @@ def save(context, interactive, set_vars, unset_vars):
@click.argument("src", type=click.Path(exists=True, resolve_path=True)) @click.argument("src", type=click.Path(exists=True, resolve_path=True))
@click.argument("dst") @click.argument("dst")
@click.pass_obj @click.pass_obj
def render(context, extra_configs, src, dst): def render(context: Context, extra_configs: List[str], src: str, dst: str) -> None:
config = tutor_config.load(context.root) config = tutor_config.load(context.root)
for extra_config in extra_configs: for extra_config in extra_configs:
tutor_config.merge( tutor_config.merge(
@ -75,14 +80,14 @@ def render(context, extra_configs, src, dst):
@click.command(help="Print the project root") @click.command(help="Print the project root")
@click.pass_obj @click.pass_obj
def printroot(context): def printroot(context: Context) -> None:
click.echo(context.root) click.echo(context.root)
@click.command(help="Print a configuration value") @click.command(help="Print a configuration value")
@click.argument("key") @click.argument("key")
@click.pass_obj @click.pass_obj
def printvalue(context, key): def printvalue(context: Context, key: str) -> None:
config = tutor_config.load(context.root) config = tutor_config.load(context.root)
try: try:
# Note that this will incorrectly print None values # Note that this will incorrectly print None values

View File

@ -1,8 +1,17 @@
from typing import Any, Dict
def unimplemented_docker_compose(
root: str, config: Dict[str, Any], *command: str
) -> int:
raise NotImplementedError
# pylint: disable=too-few-public-methods # pylint: disable=too-few-public-methods
class Context: class Context:
def __init__(self, root): def __init__(self, root: str) -> None:
self.root = root self.root = root
self.docker_compose_func = unimplemented_docker_compose
@staticmethod def docker_compose(self, root: str, config: Dict[str, Any], *command: str) -> int:
def docker_compose(root, config, *command): return self.docker_compose_func(root, config, *command)
raise NotImplementedError

View File

@ -1,15 +1,17 @@
import os import os
from typing import Any, Dict, List
import click import click
from . import compose
from .. import config as tutor_config from .. import config as tutor_config
from .. import env as tutor_env from .. import env as tutor_env
from .. import fmt from .. import fmt
from .. import utils from .. import utils
from . import compose
from .context import Context
def docker_compose(root, config, *command): def docker_compose(root: str, config: Dict[str, Any], *command: str) -> int:
""" """
Run docker-compose with dev arguments. Run docker-compose with dev arguments.
""" """
@ -27,15 +29,15 @@ def docker_compose(root, config, *command):
return utils.docker_compose( return utils.docker_compose(
*args, *args,
"--project-name", "--project-name",
config["DEV_PROJECT_NAME"], str(config["DEV_PROJECT_NAME"]),
*command, *command,
) )
@click.group(help="Run Open edX locally with development settings") @click.group(help="Run Open edX locally with development settings")
@click.pass_obj @click.pass_obj
def dev(context): def dev(context: Context) -> None:
context.docker_compose = docker_compose context.docker_compose_func = docker_compose
@click.command( @click.command(
@ -44,9 +46,9 @@ def dev(context):
) )
@click.argument("options", nargs=-1, required=False) @click.argument("options", nargs=-1, required=False)
@click.argument("service") @click.argument("service")
@click.pass_obj @click.pass_context
def runserver(context, options, service): def runserver(context: click.Context, options: List[str], service: str) -> None:
config = tutor_config.load(context.root) config = tutor_config.load(context.obj.root)
if service in ["lms", "cms"]: if service in ["lms", "cms"]:
port = 8000 if service == "lms" else 8001 port = 8000 if service == "lms" else 8001
host = config["LMS_HOST"] if service == "lms" else config["CMS_HOST"] host = config["LMS_HOST"] if service == "lms" else config["CMS_HOST"]
@ -56,7 +58,7 @@ def runserver(context, options, service):
) )
) )
args = ["--service-ports", *options, service] args = ["--service-ports", *options, service]
compose.run.callback(args) context.invoke(compose.run, args=args)
dev.add_command(runserver) dev.add_command(runserver)

View File

@ -1,3 +1,5 @@
from typing import cast, Any, Dict, Iterator, List, Tuple
import click import click
from .. import config as tutor_config from .. import config as tutor_config
@ -5,6 +7,7 @@ from .. import env as tutor_env
from .. import images from .. import images
from .. import plugins from .. import plugins
from .. import utils from .. import utils
from .context import Context
BASE_IMAGE_NAMES = ["openedx", "forum", "android"] BASE_IMAGE_NAMES = ["openedx", "forum", "android"]
DEV_IMAGE_NAMES = ["openedx-dev"] DEV_IMAGE_NAMES = ["openedx-dev"]
@ -20,7 +23,7 @@ VENDOR_IMAGES = [
@click.group(name="images", short_help="Manage docker images") @click.group(name="images", short_help="Manage docker images")
def images_command(): def images_command() -> None:
pass pass
@ -50,7 +53,14 @@ def images_command():
help="Set the target build stage to build.", help="Set the target build stage to build.",
) )
@click.pass_obj @click.pass_obj
def build(context, image_names, no_cache, build_args, add_hosts, target): def build(
context: Context,
image_names: List[str],
no_cache: bool,
build_args: List[str],
add_hosts: List[str],
target: str,
) -> None:
config = tutor_config.load(context.root) config = tutor_config.load(context.root)
command_args = [] command_args = []
if no_cache: if no_cache:
@ -68,7 +78,7 @@ def build(context, image_names, no_cache, build_args, add_hosts, target):
@click.command(short_help="Pull images from the Docker registry") @click.command(short_help="Pull images from the Docker registry")
@click.argument("image_names", metavar="image", nargs=-1) @click.argument("image_names", metavar="image", nargs=-1)
@click.pass_obj @click.pass_obj
def pull(context, image_names): def pull(context: Context, image_names: List[str]) -> None:
config = tutor_config.load(context.root) config = tutor_config.load(context.root)
for image in image_names: for image in image_names:
pull_image(config, image) pull_image(config, image)
@ -77,7 +87,7 @@ def pull(context, image_names):
@click.command(short_help="Push images to the Docker registry") @click.command(short_help="Push images to the Docker registry")
@click.argument("image_names", metavar="image", nargs=-1) @click.argument("image_names", metavar="image", nargs=-1)
@click.pass_obj @click.pass_obj
def push(context, image_names): def push(context: Context, image_names: List[str]) -> None:
config = tutor_config.load(context.root) config = tutor_config.load(context.root)
for image in image_names: for image in image_names:
push_image(config, image) push_image(config, image)
@ -86,16 +96,16 @@ def push(context, image_names):
@click.command(short_help="Print tag associated to a Docker image") @click.command(short_help="Print tag associated to a Docker image")
@click.argument("image_names", metavar="image", nargs=-1) @click.argument("image_names", metavar="image", nargs=-1)
@click.pass_obj @click.pass_obj
def printtag(context, image_names): def printtag(context: Context, image_names: List[str]) -> None:
config = tutor_config.load(context.root) config = tutor_config.load(context.root)
for image in image_names: for image in image_names:
for _img, tag in iter_images(config, image, BASE_IMAGE_NAMES): for _img, tag in iter_images(config, image, BASE_IMAGE_NAMES):
print(tag) print(tag)
for _img, tag in iter_plugin_images(config, image, "build-image"): for _plugin, _img, tag in iter_plugin_images(config, image, "build-image"):
print(tag) print(tag)
def build_image(root, config, image, *args): def build_image(root: str, config: Dict[str, Any], image: str, *args: str) -> None:
# Build base images # Build base images
for img, tag in iter_images(config, image, BASE_IMAGE_NAMES): for img, tag in iter_images(config, image, BASE_IMAGE_NAMES):
images.build(tutor_env.pathjoin(root, "build", img), tag, *args) images.build(tutor_env.pathjoin(root, "build", img), tag, *args)
@ -112,40 +122,45 @@ def build_image(root, config, image, *args):
images.build(tutor_env.pathjoin(root, "build", img), tag, *dev_build_arg, *args) images.build(tutor_env.pathjoin(root, "build", img), tag, *dev_build_arg, *args)
def pull_image(config, image): def pull_image(config: Dict[str, Any], image: str) -> None:
for _img, tag in iter_images(config, image, all_image_names(config)): for _img, tag in iter_images(config, image, all_image_names(config)):
images.pull(tag) images.pull(tag)
for _plugin, _img, tag in iter_plugin_images(config, image, "remote-image"): for _plugin, _img, tag in iter_plugin_images(config, image, "remote-image"):
images.pull(tag) images.pull(tag)
def push_image(config, image): def push_image(config: Dict[str, Any], image: str) -> None:
for _img, tag in iter_images(config, image, BASE_IMAGE_NAMES): for _img, tag in iter_images(config, image, BASE_IMAGE_NAMES):
images.push(tag) images.push(tag)
for _plugin, _img, tag in iter_plugin_images(config, image, "remote-image"): for _plugin, _img, tag in iter_plugin_images(config, image, "remote-image"):
images.push(tag) images.push(tag)
def iter_images(config, image, image_list): def iter_images(
config: Dict[str, Any], image: str, image_list: List[str]
) -> Iterator[Tuple[str, str]]:
for img in image_list: for img in image_list:
if image in [img, "all"]: if image in [img, "all"]:
tag = images.get_tag(config, img) tag = images.get_tag(config, img)
yield img, tag yield img, tag
def iter_plugin_images(config, image, hook_name): def iter_plugin_images(
config: Dict[str, Any], image: str, hook_name: str
) -> Iterator[Tuple[str, str, str]]:
for plugin, hook in plugins.iter_hooks(config, hook_name): for plugin, hook in plugins.iter_hooks(config, hook_name):
hook = cast(Dict[str, str], hook)
for img, tag in hook.items(): for img, tag in hook.items():
if image in [img, "all"]: if image in [img, "all"]:
tag = tutor_env.render_str(config, tag) tag = tutor_env.render_str(config, tag)
yield plugin, img, tag yield plugin, img, tag
def all_image_names(config): def all_image_names(config: Dict[str, Any]) -> List[str]:
return BASE_IMAGE_NAMES + vendor_image_names(config) return BASE_IMAGE_NAMES + vendor_image_names(config)
def vendor_image_names(config): def vendor_image_names(config: Dict[str, Any]) -> List[str]:
vendor_images = VENDOR_IMAGES[:] vendor_images = VENDOR_IMAGES[:]
for image in VENDOR_IMAGES: for image in VENDOR_IMAGES:
if not config.get("RUN_" + image.upper(), True): if not config.get("RUN_" + image.upper(), True):

View File

@ -1,5 +1,6 @@
from datetime import datetime from datetime import datetime
from time import sleep from time import sleep
from typing import cast, Any, Dict, List, Optional, Type
import click import click
@ -11,286 +12,13 @@ from .. import interactive as interactive_config
from .. import jobs from .. import jobs
from .. import serialize from .. import serialize
from .. import utils from .. import utils
from .context import Context
@click.group(help="Run Open edX on Kubernetes")
def k8s():
pass
@click.command(help="Configure and run Open edX from scratch")
@click.option("-I", "--non-interactive", is_flag=True, help="Run non-interactively")
@click.pass_obj
def quickstart(context, non_interactive):
click.echo(fmt.title("Interactive platform configuration"))
config = interactive_config.update(context.root, interactive=(not non_interactive))
if not config["RUN_CADDY"]:
fmt.echo_alert(
"Potentially invalid configuration: RUN_CADDY=false\n"
"This setting might have been defined because you previously set WEB_PROXY=true. This is no longer"
" necessary in order to get Tutor to work on Kubernetes. In Tutor v11+ a Caddy-based load balancer is"
" provided out of the box to handle SSL/TLS certificate generation at runtime. If you disable this"
" service, you will have to configure an Ingress resource and a certificate manager yourself to redirect"
" traffic to the nginx service. See the Kubernetes section in the Tutor documentation for more"
" information."
)
click.echo(fmt.title("Updating the current environment"))
tutor_env.save(context.root, config)
click.echo(fmt.title("Starting the platform"))
start.callback()
click.echo(fmt.title("Database creation and migrations"))
init.callback(limit=None)
fmt.echo_info(
"""Your Open edX platform is ready and can be accessed at the following urls:
{http}://{lms_host}
{http}://{cms_host}
""".format(
http="https" if config["ENABLE_HTTPS"] else "http",
lms_host=config["LMS_HOST"],
cms_host=config["CMS_HOST"],
)
)
@click.command(help="Run all configured Open edX services")
@click.pass_obj
def start(context):
# Create namespace
utils.kubectl(
"apply",
"--kustomize",
tutor_env.pathjoin(context.root),
"--wait",
"--selector",
"app.kubernetes.io/component=namespace",
)
# Create volumes
utils.kubectl(
"apply",
"--kustomize",
tutor_env.pathjoin(context.root),
"--wait",
"--selector",
"app.kubernetes.io/component=volume",
)
# Create everything else except jobs
utils.kubectl(
"apply",
"--kustomize",
tutor_env.pathjoin(context.root),
"--selector",
# Here use `notin (job, xxx)` when there are other components to ignore
"app.kubernetes.io/component!=job",
)
@click.command(help="Stop a running platform")
@click.pass_obj
def stop(context):
config = tutor_config.load(context.root)
utils.kubectl(
"delete",
*resource_selector(config),
"deployments,services,configmaps,jobs",
)
@click.command(help="Reboot an existing platform")
def reboot():
stop.callback()
start.callback()
def resource_selector(config, *selectors):
"""
Convenient utility for filtering only the resources that belong to this project.
"""
selector = ",".join(
["app.kubernetes.io/instance=openedx-" + config["ID"]] + list(selectors)
)
return ["--namespace", config["K8S_NAMESPACE"], "--selector=" + selector]
@click.command(help="Completely delete an existing platform")
@click.option("-y", "--yes", is_flag=True, help="Do not ask for confirmation")
@click.pass_obj
def delete(context, yes):
if not yes:
click.confirm(
"Are you sure you want to delete the platform? All data will be removed.",
abort=True,
)
utils.kubectl(
"delete",
"-k",
tutor_env.pathjoin(context.root),
"--ignore-not-found=true",
"--wait",
)
@click.command(help="Initialise all applications")
@click.option("-l", "--limit", help="Limit initialisation to this service or plugin")
@click.pass_obj
def init(context, limit):
config = tutor_config.load(context.root)
runner = K8sJobRunner(context.root, config)
for service in ["mysql", "elasticsearch", "mongodb"]:
if tutor_config.is_service_activated(config, service):
wait_for_pod_ready(config, service)
jobs.initialise(runner, limit_to=limit)
@click.command(help="Create an Open edX user and interactively set their password")
@click.option("--superuser", is_flag=True, help="Make superuser")
@click.option("--staff", is_flag=True, help="Make staff user")
@click.option(
"-p",
"--password",
help="Specify password from the command line. If undefined, you will be prompted to input a password",
)
@click.argument("name")
@click.argument("email")
@click.pass_obj
def createuser(context, superuser, staff, password, name, email):
config = tutor_config.load(context.root)
command = jobs.create_user_command(superuser, staff, name, email, password=password)
# This needs to be interactive in case the user needs to type a password
kubectl_exec(config, "lms", command, attach=True)
@click.command(help="Import the demo course")
@click.pass_obj
def importdemocourse(context):
fmt.echo_info("Importing demo course")
config = tutor_config.load(context.root)
runner = K8sJobRunner(context.root, config)
jobs.import_demo_course(runner)
@click.command(
help="Set a theme for a given domain name. To reset to the default theme , use 'default' as the theme name."
)
@click.argument("theme_name")
@click.argument("domain_names", metavar="domain_name", nargs=-1)
@click.pass_obj
def settheme(context, theme_name, domain_names):
config = tutor_config.load(context.root)
runner = K8sJobRunner(context.root, config)
for domain_name in domain_names:
jobs.set_theme(theme_name, domain_name, runner)
@click.command(name="exec", help="Execute a command in a pod of the given application")
@click.argument("service")
@click.argument("command")
@click.pass_obj
def exec_command(context, service, command):
config = tutor_config.load(context.root)
kubectl_exec(config, service, command, attach=True)
@click.command(help="View output from containers")
@click.option("-c", "--container", help="Print the logs of this specific container")
@click.option("-f", "--follow", is_flag=True, help="Follow log output")
@click.option("--tail", type=int, help="Number of lines to show from each container")
@click.argument("service")
@click.pass_obj
def logs(context, container, follow, tail, service):
config = tutor_config.load(context.root)
command = ["logs"]
selectors = ["app.kubernetes.io/name=" + service] if service else []
command += resource_selector(config, *selectors)
if container:
command += ["-c", container]
if follow:
command += ["--follow"]
if tail is not None:
command += ["--tail", str(tail)]
utils.kubectl(*command)
@click.command(help="Wait for a pod to become ready")
@click.argument("name")
@click.pass_obj
def wait(context, name):
config = tutor_config.load(context.root)
wait_for_pod_ready(config, name)
@click.command(help="Upgrade from a previous Open edX named release")
@click.option(
"--from", "from_version", default="ironwood", type=click.Choice(["ironwood"])
)
@click.pass_obj
def upgrade(context, from_version):
config = tutor_config.load(context.root)
running_version = from_version
if running_version == "ironwood":
upgrade_from_ironwood(config)
running_version = "juniper"
if running_version == "juniper":
running_version = "koa"
def upgrade_from_ironwood(config):
if not config["RUN_MONGODB"]:
fmt.echo_info(
"You are not running MongDB (RUN_MONGODB=false). It is your "
"responsibility to upgrade your MongoDb instance to v3.6. There is "
"nothing left to do to upgrade from Ironwood."
)
return
message = """Automatic release upgrade is unsupported in Kubernetes. To upgrade from Ironwood, you should upgrade
your MongoDb cluster from v3.2 to v3.6. You should run something similar to:
# Upgrade from v3.2 to v3.4
tutor k8s stop
tutor config save --set DOCKER_IMAGE_MONGODB=mongo:3.4.24
tutor k8s start
tutor k8s exec mongodb mongo --eval 'db.adminCommand({ setFeatureCompatibilityVersion: "3.4" })'
# Upgrade from v3.4 to v3.6
tutor k8s stop
tutor config save --set DOCKER_IMAGE_MONGODB=mongo:3.6.18
tutor k8s start
tutor k8s exec mongodb mongo --eval 'db.adminCommand({ setFeatureCompatibilityVersion: "3.6" })'
tutor config save --unset DOCKER_IMAGE_MONGODB"""
fmt.echo_info(message)
def upgrade_from_juniper(config):
if not config["RUN_MYSQL"]:
fmt.echo_info(
"You are not running MySQL (RUN_MYSQL=false). It is your "
"responsibility to upgrade your MySQL instance to v5.7. There is "
"nothing left to do to upgrade from Juniper."
)
return
message = """Automatic release upgrade is unsupported in Kubernetes. To upgrade from Juniper, you should upgrade
your MySQL database from v5.6 to v5.7. You should run something similar to:
tutor k8s start
tutor k8s exec mysql bash -e -c "mysql_upgrade \
-u $(tutor config printvalue MYSQL_ROOT_USERNAME) \
--password='$(tutor config printvalue MYSQL_ROOT_PASSWORD)'
"""
fmt.echo_info(message)
class K8sClients: class K8sClients:
_instance = None _instance = None
def __init__(self): def __init__(self) -> None:
# Loading the kubernetes module here to avoid import overhead # Loading the kubernetes module here to avoid import overhead
from kubernetes import client, config # pylint: disable=import-outside-toplevel from kubernetes import client, config # pylint: disable=import-outside-toplevel
@ -300,33 +28,34 @@ class K8sClients:
self._client = client self._client = client
@classmethod @classmethod
def instance(cls): def instance(cls: Type["K8sClients"]) -> "K8sClients":
if cls._instance is None: if cls._instance is None:
cls._instance = cls() cls._instance = cls()
return cls._instance return cls._instance
@property @property
def batch_api(self): def batch_api(self): # type: ignore
if self._batch_api is None: if self._batch_api is None:
self._batch_api = self._client.BatchV1Api() self._batch_api = self._client.BatchV1Api()
return self._batch_api return self._batch_api
@property @property
def core_api(self): def core_api(self): # type: ignore
if self._core_api is None: if self._core_api is None:
self._core_api = self._client.CoreV1Api() self._core_api = self._client.CoreV1Api()
return self._core_api return self._core_api
class K8sJobRunner(jobs.BaseJobRunner): class K8sJobRunner(jobs.BaseJobRunner):
def load_job(self, name): def load_job(self, name: str) -> Any:
jobs = self.render("k8s", "jobs.yml") all_jobs = self.render("k8s", "jobs.yml")
for job in serialize.load_all(jobs): for job in serialize.load_all(all_jobs):
if job["metadata"]["name"] == name: job_name = cast(str, job["metadata"]["name"])
if job_name == name:
return job return job
raise ValueError("Could not find job '{}'".format(name)) raise ValueError("Could not find job '{}'".format(name))
def active_job_names(self): def active_job_names(self) -> List[str]:
""" """
Return a list of active job names Return a list of active job names
Docs: Docs:
@ -339,7 +68,7 @@ class K8sJobRunner(jobs.BaseJobRunner):
if job.status.active if job.status.active
] ]
def run_job(self, service, command): def run_job(self, service: str, command: str) -> int:
job_name = "{}-job".format(service) job_name = "{}-job".format(service)
try: try:
job = self.load_job(job_name) job = self.load_job(job_name)
@ -361,8 +90,7 @@ class K8sJobRunner(jobs.BaseJobRunner):
) )
fmt.echo_alert(message) fmt.echo_alert(message)
wait_for_pod_ready(self.config, service) wait_for_pod_ready(self.config, service)
kubectl_exec(self.config, service, command) return kubectl_exec(self.config, service, command)
return
# Create a unique job name to make it deduplicate jobs and make it easier to # Create a unique job name to make it deduplicate jobs and make it easier to
# find later. Logs of older jobs will remain available for some time. # find later. Logs of older jobs will remain available for some time.
job_name += "-" + datetime.now().strftime("%Y%m%d%H%M%S") job_name += "-" + datetime.now().strftime("%Y%m%d%H%M%S")
@ -417,12 +145,12 @@ class K8sJobRunner(jobs.BaseJobRunner):
# Wait for completion # Wait for completion
field_selector = "metadata.name={}".format(job_name) field_selector = "metadata.name={}".format(job_name)
while True: while True:
jobs = K8sClients.instance().batch_api.list_namespaced_job( namespaced_jobs = K8sClients.instance().batch_api.list_namespaced_job(
self.config["K8S_NAMESPACE"], field_selector=field_selector self.config["K8S_NAMESPACE"], field_selector=field_selector
) )
if not jobs.items: if not namespaced_jobs.items:
continue continue
job = jobs.items[0] job = namespaced_jobs.items[0]
if not job.status.active: if not job.status.active:
if job.status.succeeded: if job.status.succeeded:
fmt.echo_info("Job {} successful.".format(job_name)) fmt.echo_info("Job {} successful.".format(job_name))
@ -434,9 +162,292 @@ class K8sJobRunner(jobs.BaseJobRunner):
) )
) )
sleep(5) sleep(5)
return 0
def kubectl_exec(config, service, command, attach=False): @click.group(help="Run Open edX on Kubernetes")
def k8s() -> None:
pass
@click.command(help="Configure and run Open edX from scratch")
@click.option("-I", "--non-interactive", is_flag=True, help="Run non-interactively")
@click.pass_context
def quickstart(context: click.Context, non_interactive: bool) -> None:
click.echo(fmt.title("Interactive platform configuration"))
config = interactive_config.update(
context.obj.root, interactive=(not non_interactive)
)
if not config["RUN_CADDY"]:
fmt.echo_alert(
"Potentially invalid configuration: RUN_CADDY=false\n"
"This setting might have been defined because you previously set WEB_PROXY=true. This is no longer"
" necessary in order to get Tutor to work on Kubernetes. In Tutor v11+ a Caddy-based load balancer is"
" provided out of the box to handle SSL/TLS certificate generation at runtime. If you disable this"
" service, you will have to configure an Ingress resource and a certificate manager yourself to redirect"
" traffic to the nginx service. See the Kubernetes section in the Tutor documentation for more"
" information."
)
click.echo(fmt.title("Updating the current environment"))
tutor_env.save(context.obj.root, config)
click.echo(fmt.title("Starting the platform"))
context.invoke(start)
click.echo(fmt.title("Database creation and migrations"))
context.invoke(init, limit=None)
fmt.echo_info(
"""Your Open edX platform is ready and can be accessed at the following urls:
{http}://{lms_host}
{http}://{cms_host}
""".format(
http="https" if config["ENABLE_HTTPS"] else "http",
lms_host=config["LMS_HOST"],
cms_host=config["CMS_HOST"],
)
)
@click.command(help="Run all configured Open edX services")
@click.pass_obj
def start(context: Context) -> None:
# Create namespace
utils.kubectl(
"apply",
"--kustomize",
tutor_env.pathjoin(context.root),
"--wait",
"--selector",
"app.kubernetes.io/component=namespace",
)
# Create volumes
utils.kubectl(
"apply",
"--kustomize",
tutor_env.pathjoin(context.root),
"--wait",
"--selector",
"app.kubernetes.io/component=volume",
)
# Create everything else except jobs
utils.kubectl(
"apply",
"--kustomize",
tutor_env.pathjoin(context.root),
"--selector",
# Here use `notin (job, xxx)` when there are other components to ignore
"app.kubernetes.io/component!=job",
)
@click.command(help="Stop a running platform")
@click.pass_obj
def stop(context: Context) -> None:
config = tutor_config.load(context.root)
utils.kubectl(
"delete",
*resource_selector(config),
"deployments,services,configmaps,jobs",
)
@click.command(help="Reboot an existing platform")
@click.pass_context
def reboot(context: click.Context) -> None:
context.invoke(stop)
context.invoke(start)
def resource_selector(config: Dict[str, str], *selectors: str) -> List[str]:
"""
Convenient utility for filtering only the resources that belong to this project.
"""
selector = ",".join(
["app.kubernetes.io/instance=openedx-" + config["ID"]] + list(selectors)
)
return ["--namespace", config["K8S_NAMESPACE"], "--selector=" + selector]
@click.command(help="Completely delete an existing platform")
@click.option("-y", "--yes", is_flag=True, help="Do not ask for confirmation")
@click.pass_obj
def delete(context: Context, yes: bool) -> None:
if not yes:
click.confirm(
"Are you sure you want to delete the platform? All data will be removed.",
abort=True,
)
utils.kubectl(
"delete",
"-k",
tutor_env.pathjoin(context.root),
"--ignore-not-found=true",
"--wait",
)
@click.command(help="Initialise all applications")
@click.option("-l", "--limit", help="Limit initialisation to this service or plugin")
@click.pass_obj
def init(context: Context, limit: Optional[str]) -> None:
config = tutor_config.load(context.root)
runner = K8sJobRunner(context.root, config)
for service in ["mysql", "elasticsearch", "mongodb"]:
if tutor_config.is_service_activated(config, service):
wait_for_pod_ready(config, service)
jobs.initialise(runner, limit_to=limit)
@click.command(help="Create an Open edX user and interactively set their password")
@click.option("--superuser", is_flag=True, help="Make superuser")
@click.option("--staff", is_flag=True, help="Make staff user")
@click.option(
"-p",
"--password",
help="Specify password from the command line. If undefined, you will be prompted to input a password",
)
@click.argument("name")
@click.argument("email")
@click.pass_obj
def createuser(
context: Context, superuser: str, staff: bool, password: str, name: str, email: str
) -> None:
config = tutor_config.load(context.root)
command = jobs.create_user_command(superuser, staff, name, email, password=password)
# This needs to be interactive in case the user needs to type a password
kubectl_exec(config, "lms", command, attach=True)
@click.command(help="Import the demo course")
@click.pass_obj
def importdemocourse(context: Context) -> None:
fmt.echo_info("Importing demo course")
config = tutor_config.load(context.root)
runner = K8sJobRunner(context.root, config)
jobs.import_demo_course(runner)
@click.command(
help="Set a theme for a given domain name. To reset to the default theme , use 'default' as the theme name."
)
@click.argument("theme_name")
@click.argument("domain_names", metavar="domain_name", nargs=-1)
@click.pass_obj
def settheme(context: Context, theme_name: str, domain_names: List[str]) -> None:
config = tutor_config.load(context.root)
runner = K8sJobRunner(context.root, config)
for domain_name in domain_names:
jobs.set_theme(theme_name, domain_name, runner)
@click.command(name="exec", help="Execute a command in a pod of the given application")
@click.argument("service")
@click.argument("command")
@click.pass_obj
def exec_command(context: Context, service: str, command: str) -> None:
config = tutor_config.load(context.root)
kubectl_exec(config, service, command, attach=True)
@click.command(help="View output from containers")
@click.option("-c", "--container", help="Print the logs of this specific container")
@click.option("-f", "--follow", is_flag=True, help="Follow log output")
@click.option("--tail", type=int, help="Number of lines to show from each container")
@click.argument("service")
@click.pass_obj
def logs(
context: Context, container: str, follow: bool, tail: bool, service: str
) -> None:
config = tutor_config.load(context.root)
command = ["logs"]
selectors = ["app.kubernetes.io/name=" + service] if service else []
command += resource_selector(config, *selectors)
if container:
command += ["-c", container]
if follow:
command += ["--follow"]
if tail is not None:
command += ["--tail", str(tail)]
utils.kubectl(*command)
@click.command(help="Wait for a pod to become ready")
@click.argument("name")
@click.pass_obj
def wait(context: Context, name: str) -> None:
config = tutor_config.load(context.root)
wait_for_pod_ready(config, name)
@click.command(help="Upgrade from a previous Open edX named release")
@click.option(
"--from", "from_version", default="ironwood", type=click.Choice(["ironwood"])
)
@click.pass_obj
def upgrade(context: Context, from_version: str) -> None:
config = tutor_config.load(context.root)
running_version = from_version
if running_version == "ironwood":
upgrade_from_ironwood(config)
running_version = "juniper"
if running_version == "juniper":
running_version = "koa"
def upgrade_from_ironwood(config: Dict[str, Any]) -> None:
if not config["RUN_MONGODB"]:
fmt.echo_info(
"You are not running MongDB (RUN_MONGODB=false). It is your "
"responsibility to upgrade your MongoDb instance to v3.6. There is "
"nothing left to do to upgrade from Ironwood."
)
return
message = """Automatic release upgrade is unsupported in Kubernetes. To upgrade from Ironwood, you should upgrade
your MongoDb cluster from v3.2 to v3.6. You should run something similar to:
# Upgrade from v3.2 to v3.4
tutor k8s stop
tutor config save --set DOCKER_IMAGE_MONGODB=mongo:3.4.24
tutor k8s start
tutor k8s exec mongodb mongo --eval 'db.adminCommand({ setFeatureCompatibilityVersion: "3.4" })'
# Upgrade from v3.4 to v3.6
tutor k8s stop
tutor config save --set DOCKER_IMAGE_MONGODB=mongo:3.6.18
tutor k8s start
tutor k8s exec mongodb mongo --eval 'db.adminCommand({ setFeatureCompatibilityVersion: "3.6" })'
tutor config save --unset DOCKER_IMAGE_MONGODB"""
fmt.echo_info(message)
def upgrade_from_juniper(config: Dict[str, Any]) -> None:
if not config["RUN_MYSQL"]:
fmt.echo_info(
"You are not running MySQL (RUN_MYSQL=false). It is your "
"responsibility to upgrade your MySQL instance to v5.7. There is "
"nothing left to do to upgrade from Juniper."
)
return
message = """Automatic release upgrade is unsupported in Kubernetes. To upgrade from Juniper, you should upgrade
your MySQL database from v5.6 to v5.7. You should run something similar to:
tutor k8s start
tutor k8s exec mysql bash -e -c "mysql_upgrade \
-u $(tutor config printvalue MYSQL_ROOT_USERNAME) \
--password='$(tutor config printvalue MYSQL_ROOT_PASSWORD)'
"""
fmt.echo_info(message)
def kubectl_exec(
config: Dict[str, Any], service: str, command: str, attach: bool = False
) -> int:
selector = "app.kubernetes.io/name={}".format(service) selector = "app.kubernetes.io/name={}".format(service)
pods = K8sClients.instance().core_api.list_namespaced_pod( pods = K8sClients.instance().core_api.list_namespaced_pod(
namespace=config["K8S_NAMESPACE"], label_selector=selector namespace=config["K8S_NAMESPACE"], label_selector=selector
@ -449,7 +460,7 @@ def kubectl_exec(config, service, command, attach=False):
# Run command # Run command
attach_opts = ["-i", "-t"] if attach else [] attach_opts = ["-i", "-t"] if attach else []
utils.kubectl( return utils.kubectl(
"exec", "exec",
*attach_opts, *attach_opts,
"--namespace", "--namespace",
@ -463,7 +474,7 @@ def kubectl_exec(config, service, command, attach=False):
) )
def wait_for_pod_ready(config, service): def wait_for_pod_ready(config: Dict[str, str], service: str) -> None:
fmt.echo_info("Waiting for a {} pod to be ready...".format(service)) fmt.echo_info("Waiting for a {} pod to be ready...".format(service))
utils.kubectl( utils.kubectl(
"wait", "wait",

View File

@ -1,15 +1,18 @@
import os import os
from typing import Dict, Any
import click import click
from .. import config as tutor_config from .. import config as tutor_config
from .. import env as tutor_env from .. import env as tutor_env
from .. import fmt, utils from .. import fmt
from .. import utils
from . import compose from . import compose
from .config import save as config_save_command from .config import save as config_save_command
from .context import Context
def docker_compose(root, config, *command): def docker_compose(root: str, config: Dict[str, Any], *command: str) -> int:
""" """
Run docker-compose with local and production yml files. Run docker-compose with local and production yml files.
""" """
@ -31,39 +34,41 @@ def docker_compose(root, config, *command):
@click.group(help="Run Open edX locally with docker-compose") @click.group(help="Run Open edX locally with docker-compose")
@click.pass_obj @click.pass_obj
def local(context): def local(context: Context) -> None:
context.docker_compose = docker_compose context.docker_compose_func = docker_compose
@click.command(help="Configure and run Open edX from scratch") @click.command(help="Configure and run Open edX from scratch")
@click.option("-I", "--non-interactive", is_flag=True, help="Run non-interactively") @click.option("-I", "--non-interactive", is_flag=True, help="Run non-interactively")
@click.option( @click.option("-p", "--pullimages", is_flag=True, help="Update docker images")
"-p", "--pullimages", "pullimages_", is_flag=True, help="Update docker images" @click.pass_context
) def quickstart(context: click.Context, non_interactive: bool, pullimages: bool) -> None:
@click.pass_obj if tutor_env.needs_major_upgrade(context.obj.root):
def quickstart(context, non_interactive, pullimages_):
if tutor_env.needs_major_upgrade(context.root):
click.echo(fmt.title("Upgrading from an older release")) click.echo(fmt.title("Upgrading from an older release"))
upgrade.callback( context.invoke(
from_version=tutor_env.current_release(context.root), upgrade,
from_version=tutor_env.current_release(context.obj.root),
non_interactive=non_interactive, non_interactive=non_interactive,
) )
click.echo(fmt.title("Interactive platform configuration")) click.echo(fmt.title("Interactive platform configuration"))
config_save_command.callback( context.invoke(
interactive=(not non_interactive), set_vars=[], unset_vars=[] config_save_command,
interactive=(not non_interactive),
set_vars=[],
unset_vars=[],
) )
click.echo(fmt.title("Stopping any existing platform")) click.echo(fmt.title("Stopping any existing platform"))
compose.stop.callback([]) context.invoke(compose.stop)
if pullimages_: if pullimages:
click.echo(fmt.title("Docker image updates")) click.echo(fmt.title("Docker image updates"))
compose.dc_command.callback(["pull"]) context.invoke(compose.dc_command, command="pull")
click.echo(fmt.title("Starting the platform in detached mode")) click.echo(fmt.title("Starting the platform in detached mode"))
compose.start.callback(True, []) context.invoke(compose.start, detach=True)
click.echo(fmt.title("Database creation and migrations")) click.echo(fmt.title("Database creation and migrations"))
compose.init.callback(limit=None) context.invoke(compose.init)
config = tutor_config.load(context.root) config = tutor_config.load(context.obj.root)
fmt.echo_info( fmt.echo_info(
"""The Open edX platform is now running in detached mode """The Open edX platform is now running in detached mode
Your Open edX platform is ready and can be accessed at the following urls: Your Open edX platform is ready and can be accessed at the following urls:
@ -86,9 +91,9 @@ Your Open edX platform is ready and can be accessed at the following urls:
type=click.Choice(["ironwood", "juniper"]), type=click.Choice(["ironwood", "juniper"]),
) )
@click.option("-I", "--non-interactive", is_flag=True, help="Run non-interactively") @click.option("-I", "--non-interactive", is_flag=True, help="Run non-interactively")
@click.pass_obj @click.pass_context
def upgrade(context, from_version, non_interactive): def upgrade(context: click.Context, from_version: str, non_interactive: bool) -> None:
config = tutor_config.load_no_check(context.root) config = tutor_config.load_no_check(context.obj.root)
if not non_interactive: if not non_interactive:
question = """You are about to upgrade your Open edX platform. It is strongly recommended to make a backup before upgrading. To do so, run: question = """You are about to upgrade your Open edX platform. It is strongly recommended to make a backup before upgrading. To do so, run:
@ -113,12 +118,12 @@ Are you sure you want to continue?"""
running_version = "koa" running_version = "koa"
def upgrade_from_ironwood(context, config): def upgrade_from_ironwood(context: click.Context, config: Dict[str, Any]) -> None:
click.echo(fmt.title("Upgrading from Ironwood")) click.echo(fmt.title("Upgrading from Ironwood"))
tutor_env.save(context.root, config) tutor_env.save(context.obj.root, config)
click.echo(fmt.title("Stopping any existing platform")) click.echo(fmt.title("Stopping any existing platform"))
compose.stop.callback([]) context.invoke(compose.stop)
if not config["RUN_MONGODB"]: if not config["RUN_MONGODB"]:
fmt.echo_info( fmt.echo_info(
@ -132,39 +137,41 @@ def upgrade_from_ironwood(context, config):
# environment, not the configuration. # environment, not the configuration.
click.echo(fmt.title("Upgrading MongoDb from v3.2 to v3.4")) click.echo(fmt.title("Upgrading MongoDb from v3.2 to v3.4"))
config["DOCKER_IMAGE_MONGODB"] = "mongo:3.4.24" config["DOCKER_IMAGE_MONGODB"] = "mongo:3.4.24"
tutor_env.save(context.root, config) tutor_env.save(context.obj.root, config)
compose.start.callback(detach=True, services=["mongodb"]) context.invoke(compose.start, detach=True, services=["mongodb"])
compose.execute.callback( context.invoke(
[ compose.execute,
args=[
"mongodb", "mongodb",
"mongo", "mongo",
"--eval", "--eval",
'db.adminCommand({ setFeatureCompatibilityVersion: "3.4" })', 'db.adminCommand({ setFeatureCompatibilityVersion: "3.4" })',
] ],
) )
compose.stop.callback([]) context.invoke(compose.stop)
click.echo(fmt.title("Upgrading MongoDb from v3.4 to v3.6")) click.echo(fmt.title("Upgrading MongoDb from v3.4 to v3.6"))
config["DOCKER_IMAGE_MONGODB"] = "mongo:3.6.18" config["DOCKER_IMAGE_MONGODB"] = "mongo:3.6.18"
tutor_env.save(context.root, config) tutor_env.save(context.obj.root, config)
compose.start.callback(detach=True, services=["mongodb"]) context.invoke(compose.start, detach=True, services=["mongodb"])
compose.execute.callback( context.invoke(
[ compose.execute,
args=[
"mongodb", "mongodb",
"mongo", "mongo",
"--eval", "--eval",
'db.adminCommand({ setFeatureCompatibilityVersion: "3.6" })', 'db.adminCommand({ setFeatureCompatibilityVersion: "3.6" })',
] ],
) )
compose.stop.callback([]) context.invoke(compose.stop)
def upgrade_from_juniper(context, config): def upgrade_from_juniper(context: click.Context, config: Dict[str, Any]) -> None:
click.echo(fmt.title("Upgrading from Juniper")) click.echo(fmt.title("Upgrading from Juniper"))
tutor_env.save(context.root, config) tutor_env.save(context.obj.root, config)
click.echo(fmt.title("Stopping any existing platform")) click.echo(fmt.title("Stopping any existing platform"))
compose.stop.callback([]) context.invoke(compose.stop)
if not config["RUN_MYSQL"]: if not config["RUN_MYSQL"]:
fmt.echo_info( fmt.echo_info(
@ -175,9 +182,10 @@ def upgrade_from_juniper(context, config):
return return
click.echo(fmt.title("Upgrading MySQL from v5.6 to v5.7")) click.echo(fmt.title("Upgrading MySQL from v5.6 to v5.7"))
compose.start.callback(detach=True, services=["mysql"]) context.invoke(compose.start, detach=True, services=["mysql"])
compose.execute.callback( context.invoke(
[ compose.execute,
args=[
"mysql", "mysql",
"bash", "bash",
"-e", "-e",
@ -185,9 +193,9 @@ def upgrade_from_juniper(context, config):
"mysql_upgrade -u {} --password='{}'".format( "mysql_upgrade -u {} --password='{}'".format(
config["MYSQL_ROOT_USERNAME"], config["MYSQL_ROOT_PASSWORD"] config["MYSQL_ROOT_USERNAME"], config["MYSQL_ROOT_PASSWORD"]
), ),
] ],
) )
compose.stop.callback([]) context.invoke(compose.stop)
local.add_command(quickstart) local.add_command(quickstart)

View File

@ -1,5 +1,6 @@
import os import os
import shutil import shutil
from typing import List
import urllib.request import urllib.request
import click import click
@ -9,6 +10,7 @@ from .. import env as tutor_env
from .. import exceptions from .. import exceptions
from .. import fmt from .. import fmt
from .. import plugins from .. import plugins
from .context import Context
@click.group( @click.group(
@ -16,7 +18,7 @@ from .. import plugins
short_help="Manage Tutor plugins", short_help="Manage Tutor plugins",
help="Manage Tutor plugins to add new features and customize your Open edX platform", help="Manage Tutor plugins to add new features and customize your Open edX platform",
) )
def plugins_command(): def plugins_command() -> None:
""" """
All plugin commands should work even if there is no existing config file. This is All plugin commands should work even if there is no existing config file. This is
because users might enable plugins prior to configuration or environment generation. because users might enable plugins prior to configuration or environment generation.
@ -25,7 +27,7 @@ def plugins_command():
@click.command(name="list", help="List installed plugins") @click.command(name="list", help="List installed plugins")
@click.pass_obj @click.pass_obj
def list_command(context): def list_command(context: Context) -> None:
config = tutor_config.load_user(context.root) config = tutor_config.load_user(context.root)
for plugin in plugins.iter_installed(): for plugin in plugins.iter_installed():
status = "" if plugins.is_enabled(config, plugin.name) else " (disabled)" status = "" if plugins.is_enabled(config, plugin.name) else " (disabled)"
@ -39,7 +41,7 @@ def list_command(context):
@click.command(help="Enable a plugin") @click.command(help="Enable a plugin")
@click.argument("plugin_names", metavar="plugin", nargs=-1) @click.argument("plugin_names", metavar="plugin", nargs=-1)
@click.pass_obj @click.pass_obj
def enable(context, plugin_names): def enable(context: Context, plugin_names: List[str]) -> None:
config = tutor_config.load_user(context.root) config = tutor_config.load_user(context.root)
for plugin in plugin_names: for plugin in plugin_names:
plugins.enable(config, plugin) plugins.enable(config, plugin)
@ -56,7 +58,7 @@ def enable(context, plugin_names):
) )
@click.argument("plugin_names", metavar="plugin", nargs=-1) @click.argument("plugin_names", metavar="plugin", nargs=-1)
@click.pass_obj @click.pass_obj
def disable(context, plugin_names): def disable(context: Context, plugin_names: List[str]) -> None:
config = tutor_config.load_user(context.root) config = tutor_config.load_user(context.root)
if "all" in plugin_names: if "all" in plugin_names:
plugin_names = [plugin.name for plugin in plugins.iter_enabled(config)] plugin_names = [plugin.name for plugin in plugins.iter_enabled(config)]
@ -70,7 +72,7 @@ def disable(context, plugin_names):
) )
def delete_plugin(root, name): def delete_plugin(root: str, name: str) -> None:
plugin_dir = tutor_env.pathjoin(root, "plugins", name) plugin_dir = tutor_env.pathjoin(root, "plugins", name)
if os.path.exists(plugin_dir): if os.path.exists(plugin_dir):
try: try:
@ -90,7 +92,7 @@ defined by setting the {} environment variable""".format(
plugins.DictPlugin.ROOT_ENV_VAR_NAME plugins.DictPlugin.ROOT_ENV_VAR_NAME
), ),
) )
def printroot(): def printroot() -> None:
fmt.echo(plugins.DictPlugin.ROOT) fmt.echo(plugins.DictPlugin.ROOT)
@ -102,7 +104,7 @@ location. The plugin will be installed to {}.""".format(
), ),
) )
@click.argument("location") @click.argument("location")
def install(location): def install(location: str) -> None:
basename = os.path.basename(location) basename = os.path.basename(location)
if not basename.endswith(".yml"): if not basename.endswith(".yml"):
basename += ".yml" basename += ".yml"
@ -127,7 +129,7 @@ def install(location):
fmt.echo_info("Plugin installed at {}".format(plugin_path)) fmt.echo_info("Plugin installed at {}".format(plugin_path))
def add_plugin_commands(command_group): def add_plugin_commands(command_group: click.Group) -> None:
""" """
Add commands provided by all plugins to the given command group. Each command is Add commands provided by all plugins to the given command group. Each command is
added with a name that is equal to the plugin name. added with a name that is equal to the plugin name.

View File

@ -6,7 +6,7 @@ import click_repl
short_help="Interactive shell", short_help="Interactive shell",
help="Launch an interactive shell for launching Tutor commands", help="Launch an interactive shell for launching Tutor commands",
) )
def ui(): def ui() -> None:
click.echo( click.echo(
"""Welcome to the Tutor interactive shell UI! """Welcome to the Tutor interactive shell UI!
Type "help" to view all available commands. Type "help" to view all available commands.

View File

@ -4,6 +4,7 @@ import platform
import subprocess import subprocess
import sys import sys
import tarfile import tarfile
from typing import Any, Dict
from urllib.request import urlopen from urllib.request import urlopen
import click import click
@ -13,12 +14,13 @@ import click
from .. import fmt from .. import fmt
from .. import env as tutor_env from .. import env as tutor_env
from .. import serialize from .. import serialize
from .context import Context
@click.group( @click.group(
short_help="Web user interface", help="""Run Tutor commands from a web terminal""" short_help="Web user interface", help="""Run Tutor commands from a web terminal"""
) )
def webui(): def webui() -> None:
pass pass
@ -35,7 +37,7 @@ def webui():
"-h", "--host", default="0.0.0.0", show_default=True, help="Host address to listen" "-h", "--host", default="0.0.0.0", show_default=True, help="Host address to listen"
) )
@click.pass_obj @click.pass_obj
def start(context, port, host): def start(context: Context, port: int, host: str) -> None:
check_gotty_binary(context.root) check_gotty_binary(context.root)
fmt.echo_info("Access the Tutor web UI at http://{}:{}".format(host, port)) fmt.echo_info("Access the Tutor web UI at http://{}:{}".format(host, port))
while True: while True:
@ -86,7 +88,7 @@ def start(context, port, host):
help="Authentication password", help="Authentication password",
) )
@click.pass_obj @click.pass_obj
def configure(context, user, password): def configure(context: Context, user: str, password: str) -> None:
save_webui_config_file(context.root, {"user": user, "password": password}) save_webui_config_file(context.root, {"user": user, "password": password})
fmt.echo_info( fmt.echo_info(
"The web UI configuration has been updated. " "The web UI configuration has been updated. "
@ -95,7 +97,7 @@ def configure(context, user, password):
) )
def check_gotty_binary(root): def check_gotty_binary(root: str) -> None:
path = gotty_path(root) path = gotty_path(root)
if os.path.exists(path): if os.path.exists(path):
return return
@ -119,7 +121,7 @@ def check_gotty_binary(root):
compressed.extract("./gotty", dirname) compressed.extract("./gotty", dirname)
def load_config(root): def load_config(root: str) -> Dict[str, Any]:
path = config_path(root) path = config_path(root)
if not os.path.exists(path): if not os.path.exists(path):
save_webui_config_file(root, {"user": None, "password": None}) save_webui_config_file(root, {"user": None, "password": None})
@ -127,7 +129,7 @@ def load_config(root):
return serialize.load(f) return serialize.load(f)
def save_webui_config_file(root, config): def save_webui_config_file(root: str, config: Dict[str, Any]) -> None:
path = config_path(root) path = config_path(root)
directory = os.path.dirname(path) directory = os.path.dirname(path)
if not os.path.exists(directory): if not os.path.exists(directory):
@ -136,15 +138,15 @@ def save_webui_config_file(root, config):
serialize.dump(config, of) serialize.dump(config, of)
def gotty_path(root): def gotty_path(root: str) -> str:
return get_path(root, "gotty") return get_path(root, "gotty")
def config_path(root): def config_path(root: str) -> str:
return get_path(root, "config.yml") return get_path(root, "config.yml")
def get_path(root, filename): def get_path(root: str, filename: str) -> str:
return tutor_env.pathjoin(root, "webui", filename) return tutor_env.pathjoin(root, "webui", filename)

View File

@ -1,4 +1,5 @@
import os import os
from typing import Dict, Any, Tuple
from . import exceptions from . import exceptions
from . import env from . import env
@ -8,7 +9,7 @@ from . import serialize
from . import utils from . import utils
def update(root): def update(root: str) -> Dict[str, Any]:
""" """
Load and save the configuration. Load and save the configuration.
""" """
@ -18,7 +19,7 @@ def update(root):
return config return config
def load(root): def load(root: str) -> Dict[str, Any]:
""" """
Load full configuration. This will raise an exception if there is no current Load full configuration. This will raise an exception if there is no current
configuration in the project root. configuration in the project root.
@ -27,13 +28,13 @@ def load(root):
return load_no_check(root) return load_no_check(root)
def load_no_check(root): def load_no_check(root: str) -> Dict[str, Any]:
config, defaults = load_all(root) config, defaults = load_all(root)
merge(config, defaults) merge(config, defaults)
return config return config
def load_all(root): def load_all(root: str) -> Tuple[Dict[str, Any], Dict[str, Any]]:
""" """
Return: Return:
current (dict): params currently saved in config.yml current (dict): params currently saved in config.yml
@ -45,7 +46,9 @@ def load_all(root):
return current, defaults return current, defaults
def merge(config, defaults, force=False): def merge(
config: Dict[str, str], defaults: Dict[str, str], force: bool = False
) -> None:
""" """
Merge default values with user configuration and perform rendering of "{{...}}" Merge default values with user configuration and perform rendering of "{{...}}"
values. values.
@ -55,16 +58,16 @@ def merge(config, defaults, force=False):
config[key] = env.render_unknown(config, value) config[key] = env.render_unknown(config, value)
def load_defaults(): def load_defaults() -> Dict[str, Any]:
return serialize.load(env.read_template_file("config.yml")) return serialize.load(env.read_template_file("config.yml"))
def load_config_file(path): def load_config_file(path: str) -> Dict[str, Any]:
with open(path) as f: with open(path) as f:
return serialize.load(f.read()) return serialize.load(f.read())
def load_current(root, defaults): def load_current(root: str, defaults: Dict[str, str]) -> Dict[str, Any]:
""" """
Load the configuration currently stored on disk. Load the configuration currently stored on disk.
Note: this modifies the defaults with the plugin default values. Note: this modifies the defaults with the plugin default values.
@ -77,7 +80,7 @@ def load_current(root, defaults):
return config return config
def load_user(root): def load_user(root: str) -> Dict[str, Any]:
path = config_path(root) path = config_path(root)
if not os.path.exists(path): if not os.path.exists(path):
return {} return {}
@ -87,14 +90,14 @@ def load_user(root):
return config return config
def load_env(config, defaults): def load_env(config: Dict[str, str], defaults: Dict[str, str]) -> None:
for k in defaults.keys(): for k in defaults.keys():
env_var = "TUTOR_" + k env_var = "TUTOR_" + k
if env_var in os.environ: if env_var in os.environ:
config[k] = serialize.parse(os.environ[env_var]) config[k] = serialize.parse(os.environ[env_var])
def load_required(config, defaults): def load_required(config: Dict[str, str], defaults: Dict[str, str]) -> None:
""" """
All these keys must be present in the user's config.yml. This includes all values All these keys must be present in the user's config.yml. This includes all values
that are generated once and must be kept after that, such as passwords. that are generated once and must be kept after that, such as passwords.
@ -111,7 +114,7 @@ def load_required(config, defaults):
config[key] = env.render_unknown(config, defaults[key]) config[key] = env.render_unknown(config, defaults[key])
def load_plugins(config, defaults): def load_plugins(config: Dict[str, str], defaults: Dict[str, str]) -> None:
""" """
Add, override and set new defaults from plugins. Add, override and set new defaults from plugins.
""" """
@ -133,11 +136,11 @@ def load_plugins(config, defaults):
config[key] = env.render_unknown(config, value) config[key] = env.render_unknown(config, value)
def is_service_activated(config, service): def is_service_activated(config: Dict[str, Any], service: str) -> bool:
return config["RUN_" + service.upper()] return config["RUN_" + service.upper()] is not False
def upgrade_obsolete(config): def upgrade_obsolete(config: Dict[str, Any]) -> None:
# Openedx-specific mysql passwords # Openedx-specific mysql passwords
if "MYSQL_PASSWORD" in config: if "MYSQL_PASSWORD" in config:
config["MYSQL_ROOT_PASSWORD"] = config["MYSQL_PASSWORD"] config["MYSQL_ROOT_PASSWORD"] = config["MYSQL_PASSWORD"]
@ -178,7 +181,7 @@ def upgrade_obsolete(config):
config[name.replace("ACTIVATE_", "RUN_")] = config.pop(name) config[name.replace("ACTIVATE_", "RUN_")] = config.pop(name)
def convert_json2yml(root): def convert_json2yml(root: str) -> None:
""" """
Older versions of tutor used to have json config files. Older versions of tutor used to have json config files.
""" """
@ -199,7 +202,7 @@ def convert_json2yml(root):
) )
def save_config_file(root, config): def save_config_file(root: str, config: Dict[str, str]) -> None:
path = config_path(root) path = config_path(root)
utils.ensure_file_directory_exists(path) utils.ensure_file_directory_exists(path)
with open(path, "w") as of: with open(path, "w") as of:
@ -207,7 +210,7 @@ def save_config_file(root, config):
fmt.echo_info("Configuration saved to {}".format(path)) fmt.echo_info("Configuration saved to {}".format(path))
def check_existing_config(root): def check_existing_config(root: str) -> None:
""" """
Check there is a configuration on disk and the current environment is up-to-date. Check there is a configuration on disk and the current environment is up-to-date.
""" """
@ -220,5 +223,5 @@ def check_existing_config(root):
env.check_is_up_to_date(root) env.check_is_up_to_date(root)
def config_path(root): def config_path(root: str) -> str:
return os.path.join(root, "config.yml") return os.path.join(root, "config.yml")

View File

@ -1,6 +1,7 @@
import codecs import codecs
from copy import deepcopy from copy import deepcopy
import os import os
from typing import Dict, Any, Iterable, List, Optional, Type, Union
import jinja2 import jinja2
import pkg_resources import pkg_resources
@ -19,7 +20,7 @@ BIN_FILE_EXTENSIONS = [".ico", ".jpg", ".png", ".ttf", ".woff", ".woff2"]
class Renderer: class Renderer:
@classmethod @classmethod
def instance(cls, config): def instance(cls: Type["Renderer"], config: Dict[str, Any]) -> "Renderer":
# Load template roots: these are required to be able to use # Load template roots: these are required to be able to use
# {% include .. %} directives # {% include .. %} directives
template_roots = [TEMPLATES_ROOT] template_roots = [TEMPLATES_ROOT]
@ -29,11 +30,12 @@ class Renderer:
return cls(config, template_roots, ignore_folders=["partials"]) return cls(config, template_roots, ignore_folders=["partials"])
@classmethod def __init__(
def reset(cls): self,
cls.INSTANCE = None config: Dict[str, Any],
template_roots: List[str],
def __init__(self, config, template_roots, ignore_folders=None): ignore_folders: Optional[List[str]] = None,
):
self.config = deepcopy(config) self.config = deepcopy(config)
self.template_roots = template_roots self.template_roots = template_roots
self.ignore_folders = ignore_folders or [] self.ignore_folders = ignore_folders or []
@ -57,16 +59,16 @@ class Renderer:
environment.globals["TUTOR_VERSION"] = __version__ environment.globals["TUTOR_VERSION"] = __version__
self.environment = environment self.environment = environment
def iter_templates_in(self, *prefix): def iter_templates_in(self, *prefix: str) -> Iterable[str]:
""" """
The elements of `prefix` must contain only "/", and not os.sep. The elements of `prefix` must contain only "/", and not os.sep.
""" """
prefix = "/".join(prefix) full_prefix = "/".join(prefix)
for template in self.environment.loader.list_templates(): for template in self.environment.loader.list_templates(): # type: ignore
if template.startswith(prefix) and self.is_part_of_env(template): if template.startswith(full_prefix) and self.is_part_of_env(template):
yield template yield template
def walk_templates(self, subdir): def walk_templates(self, subdir: str) -> Iterable[str]:
""" """
Iterate on the template files from `templates/<subdir>`. Iterate on the template files from `templates/<subdir>`.
@ -75,7 +77,7 @@ class Renderer:
""" """
yield from self.iter_templates_in(subdir + "/") yield from self.iter_templates_in(subdir + "/")
def is_part_of_env(self, path): def is_part_of_env(self, path: str) -> bool:
""" """
Determines whether a template should be rendered or not. Note that here we don't Determines whether a template should be rendered or not. Note that here we don't
rely on the OS separator, as we are handling templates rely on the OS separator, as we are handling templates
@ -91,7 +93,7 @@ class Renderer:
is_excluded = is_excluded or ignore_folder in parts is_excluded = is_excluded or ignore_folder in parts
return not is_excluded return not is_excluded
def find_os_path(self, template_name): def find_os_path(self, template_name: str) -> str:
path = template_name.replace("/", os.sep) path = template_name.replace("/", os.sep)
for templates_root in self.template_roots: for templates_root in self.template_roots:
full_path = os.path.join(templates_root, path) full_path = os.path.join(templates_root, path)
@ -99,7 +101,7 @@ class Renderer:
return full_path return full_path
raise ValueError("Template path does not exist") raise ValueError("Template path does not exist")
def patch(self, name, separator="\n", suffix=""): def patch(self, name: str, separator: str = "\n", suffix: str = "") -> str:
""" """
Render calls to {{ patch("...") }} in environment templates from plugin patches. Render calls to {{ patch("...") }} in environment templates from plugin patches.
""" """
@ -119,11 +121,11 @@ class Renderer:
rendered += suffix rendered += suffix
return rendered return rendered
def render_str(self, text): def render_str(self, text: str) -> str:
template = self.environment.from_string(text) template = self.environment.from_string(text)
return self.__render(template) return self.__render(template)
def render_template(self, template_name): def render_template(self, template_name: str) -> Union[str, bytes]:
""" """
Render a template file. Return the corresponding string. If it's a binary file Render a template file. Return the corresponding string. If it's a binary file
(as indicated by its path), return bytes. (as indicated by its path), return bytes.
@ -151,7 +153,7 @@ class Renderer:
fmt.echo_error("Unknown error rendering template " + template_name) fmt.echo_error("Unknown error rendering template " + template_name)
raise raise
def render_all_to(self, root, *prefix): def render_all_to(self, root: str, *prefix: str) -> None:
""" """
`prefix` can be used to limit the templates to render. `prefix` can be used to limit the templates to render.
""" """
@ -160,7 +162,7 @@ class Renderer:
dst = os.path.join(root, template_name.replace("/", os.sep)) dst = os.path.join(root, template_name.replace("/", os.sep))
write_to(rendered, dst) write_to(rendered, dst)
def __render(self, template): def __render(self, template: jinja2.Template) -> str:
try: try:
return template.render(**self.config) return template.render(**self.config)
except jinja2.exceptions.UndefinedError as e: except jinja2.exceptions.UndefinedError as e:
@ -169,7 +171,7 @@ class Renderer:
) )
def save(root, config): def save(root: str, config: Dict[str, Any]) -> None:
""" """
Save the full environment, including version information. Save the full environment, including version information.
""" """
@ -195,7 +197,7 @@ def save(root, config):
fmt.echo_info("Environment generated in {}".format(base_dir(root))) fmt.echo_info("Environment generated in {}".format(base_dir(root)))
def upgrade_obsolete(root): def upgrade_obsolete(root: str) -> None:
# tutor.conf was renamed to _tutor.conf in order to be the first config file loaded # tutor.conf was renamed to _tutor.conf in order to be the first config file loaded
# by nginx # by nginx
nginx_tutor_conf = pathjoin(root, "apps", "nginx", "tutor.conf") nginx_tutor_conf = pathjoin(root, "apps", "nginx", "tutor.conf")
@ -203,7 +205,9 @@ def upgrade_obsolete(root):
os.remove(nginx_tutor_conf) os.remove(nginx_tutor_conf)
def save_plugin_templates(plugin, root, config): def save_plugin_templates(
plugin: plugins.BasePlugin, root: str, config: Dict[str, Any]
) -> None:
""" """
Save plugin templates to plugins/<plugin name>/*. Save plugin templates to plugins/<plugin name>/*.
Only the "apps" and "build" subfolders are rendered. Only the "apps" and "build" subfolders are rendered.
@ -214,7 +218,7 @@ def save_plugin_templates(plugin, root, config):
save_all_from(subdir_path, plugins_root, config) save_all_from(subdir_path, plugins_root, config)
def save_all_from(prefix, root, config): def save_all_from(prefix: str, root: str, config: Dict[str, Any]) -> None:
""" """
Render the templates that start with `prefix` and store them with the same Render the templates that start with `prefix` and store them with the same
hierarchy at `root`. Here, `prefix` can be the result of os.path.join(...). hierarchy at `root`. Here, `prefix` can be the result of os.path.join(...).
@ -223,23 +227,20 @@ def save_all_from(prefix, root, config):
renderer.render_all_to(root, prefix.replace(os.sep, "/")) renderer.render_all_to(root, prefix.replace(os.sep, "/"))
def write_to(content, path): def write_to(content: Union[str, bytes], path: str) -> None:
""" """
Write some content to a path. Content can be either str or bytes. Write some content to a path. Content can be either str or bytes.
""" """
open_kwargs = {"mode": "w"}
if isinstance(content, bytes):
open_kwargs["mode"] += "b"
else:
# Make files readable by Docker on Windows
open_kwargs["encoding"] = "utf8"
open_kwargs["newline"] = "\n"
utils.ensure_file_directory_exists(path) utils.ensure_file_directory_exists(path)
with open(path, **open_kwargs) as of: if isinstance(content, bytes):
of.write(content) with open(path, mode="wb") as of_binary:
of_binary.write(content)
else:
with open(path, mode="w", encoding="utf8", newline="\n") as of_text:
of_text.write(content)
def render_file(config, *path): def render_file(config: Dict[str, Any], *path: str) -> Union[str, bytes]:
""" """
Return the rendered contents of a template. Return the rendered contents of a template.
""" """
@ -248,7 +249,7 @@ def render_file(config, *path):
return renderer.render_template(template_name) return renderer.render_template(template_name)
def render_dict(config): def render_dict(config: Dict[str, Any]) -> None:
""" """
Render the values from the dict. This is useful for rendering the default Render the values from the dict. This is useful for rendering the default
values from config.yml. values from config.yml.
@ -266,13 +267,13 @@ def render_dict(config):
config[k] = v config[k] = v
def render_unknown(config, value): def render_unknown(config: Dict[str, Any], value: Any) -> Any:
if isinstance(value, str): if isinstance(value, str):
return render_str(config, value) return render_str(config, value)
return value return value
def render_str(config, text): def render_str(config: Dict[str, Any], text: str) -> str:
""" """
Args: Args:
text (str) text (str)
@ -284,7 +285,7 @@ def render_str(config, text):
return Renderer.instance(config).render_str(text) return Renderer.instance(config).render_str(text)
def check_is_up_to_date(root): def check_is_up_to_date(root: str) -> None:
if not is_up_to_date(root): if not is_up_to_date(root):
message = ( message = (
"The current environment stored at {} is not up-to-date: it is at " "The current environment stored at {} is not up-to-date: it is at "
@ -298,14 +299,14 @@ def check_is_up_to_date(root):
) )
def is_up_to_date(root): def is_up_to_date(root: str) -> bool:
""" """
Check if the currently rendered version is equal to the current tutor version. Check if the currently rendered version is equal to the current tutor version.
""" """
return current_version(root) == __version__ return current_version(root) == __version__
def needs_major_upgrade(root): def needs_major_upgrade(root: str) -> bool:
""" """
Return the current version as a tuple of int. E.g: (1, 0, 2). Return the current version as a tuple of int. E.g: (1, 0, 2).
""" """
@ -314,7 +315,7 @@ def needs_major_upgrade(root):
return 0 < current < required return 0 < current < required
def current_release(root): def current_release(root: str) -> str:
""" """
Return the name of the current Open edX release. Return the name of the current Open edX release.
""" """
@ -323,7 +324,7 @@ def current_release(root):
] ]
def current_version(root): def current_version(root: str) -> str:
""" """
Return the current environment version. If the current environment has no version, Return the current environment version. If the current environment has no version,
return "0.0.0". return "0.0.0".
@ -334,7 +335,7 @@ def current_version(root):
return open(path).read().strip() return open(path).read().strip()
def read_template_file(*path): def read_template_file(*path: str) -> str:
""" """
Read raw content of template located at `path`. Read raw content of template located at `path`.
""" """
@ -343,40 +344,40 @@ def read_template_file(*path):
return fi.read() return fi.read()
def is_binary_file(path): def is_binary_file(path: str) -> bool:
ext = os.path.splitext(path)[1] ext = os.path.splitext(path)[1]
return ext in BIN_FILE_EXTENSIONS return ext in BIN_FILE_EXTENSIONS
def template_path(*path, templates_root=TEMPLATES_ROOT): def template_path(*path: str, templates_root: str = TEMPLATES_ROOT) -> str:
""" """
Return the template file's absolute path. Return the template file's absolute path.
""" """
return os.path.join(templates_root, *path) return os.path.join(templates_root, *path)
def data_path(root, *path): def data_path(root: str, *path: str) -> str:
""" """
Return the file's absolute path inside the data directory. Return the file's absolute path inside the data directory.
""" """
return os.path.join(root_dir(root), "data", *path) return os.path.join(root_dir(root), "data", *path)
def pathjoin(root, *path): def pathjoin(root: str, *path: str) -> str:
""" """
Return the file's absolute path inside the environment. Return the file's absolute path inside the environment.
""" """
return os.path.join(base_dir(root), *path) return os.path.join(base_dir(root), *path)
def base_dir(root): def base_dir(root: str) -> str:
""" """
Return the environment base directory. Return the environment base directory.
""" """
return os.path.join(root_dir(root), "env") return os.path.join(root_dir(root), "env")
def root_dir(root): def root_dir(root: str) -> str:
""" """
Return the project root directory. Return the project root directory.
""" """

View File

@ -3,7 +3,7 @@ import click
STDOUT = None STDOUT = None
def title(text): def title(text: str) -> str:
indent = 8 indent = 8
separator = "=" * (len(text) + 2 * indent) separator = "=" * (len(text) + 2 * indent)
message = "{separator}\n{indent}{text}\n{separator}".format( message = "{separator}\n{indent}{text}\n{separator}".format(
@ -12,37 +12,37 @@ def title(text):
return click.style(message, fg="green") return click.style(message, fg="green")
def echo_info(text): def echo_info(text: str) -> None:
echo(info(text)) echo(info(text))
def info(text): def info(text: str) -> str:
return click.style(text, fg="blue") return click.style(text, fg="blue")
def error(text): def error(text: str) -> str:
return click.style(text, fg="red") return click.style(text, fg="red")
def echo_error(text): def echo_error(text: str) -> None:
echo(error(text), err=True) echo(error(text), err=True)
def command(text): def command(text: str) -> str:
return click.style(text, fg="magenta") return click.style(text, fg="magenta")
def question(text): def question(text: str) -> str:
return click.style(text, fg="yellow") return click.style(text, fg="yellow")
def echo_alert(text): def echo_alert(text: str) -> None:
echo_error(alert(text)) echo_error(alert(text))
def alert(text): def alert(text: str) -> str:
return click.style("⚠️ " + text, fg="yellow", bold=True) return click.style("⚠️ " + text, fg="yellow", bold=True)
def echo(text, err=False): def echo(text: str, err: bool = False) -> None:
click.echo(text, file=STDOUT, err=err) click.echo(text, file=STDOUT, err=err)

View File

@ -1,21 +1,23 @@
from typing import Any, Dict
from . import fmt from . import fmt
from . import utils from . import utils
def get_tag(config, name): def get_tag(config: Dict[str, Any], name: str) -> Any:
return config["DOCKER_IMAGE_" + name.upper().replace("-", "_")] return config["DOCKER_IMAGE_" + name.upper().replace("-", "_")]
def build(path, tag, *args): def build(path: str, tag: str, *args: str) -> None:
fmt.echo_info("Building image {}".format(tag)) fmt.echo_info("Building image {}".format(tag))
utils.docker("build", "-t", tag, *args, path) utils.docker("build", "-t", tag, *args, path)
def pull(tag): def pull(tag: str) -> None:
fmt.echo_info("Pulling image {}".format(tag)) fmt.echo_info("Pulling image {}".format(tag))
utils.docker("pull", tag) utils.docker("pull", tag)
def push(tag): def push(tag: str) -> None:
fmt.echo_info("Pushing image {}".format(tag)) fmt.echo_info("Pushing image {}".format(tag))
utils.docker("push", tag) utils.docker("push", tag)

View File

@ -1,3 +1,4 @@
from typing import Any, Dict, List, Tuple
import click import click
from . import config as tutor_config from . import config as tutor_config
@ -7,7 +8,7 @@ from . import fmt
from .__about__ import __version__ from .__about__ import __version__
def update(root, interactive=True): def update(root: str, interactive: bool = True) -> Dict[str, Any]:
""" """
Load and save the configuration. Load and save the configuration.
""" """
@ -17,7 +18,9 @@ def update(root, interactive=True):
return config return config
def load_all(root, interactive=True): def load_all(
root: str, interactive: bool = True
) -> Tuple[Dict[str, Any], Dict[str, Any]]:
""" """
Load configuration and interactively ask questions to collect param values from the user. Load configuration and interactively ask questions to collect param values from the user.
""" """
@ -27,7 +30,7 @@ def load_all(root, interactive=True):
return config, defaults return config, defaults
def ask_questions(config, defaults): def ask_questions(config: Dict[str, Any], defaults: Dict[str, Any]) -> None:
run_for_prod = config.get("LMS_HOST") != "local.overhang.io" run_for_prod = config.get("LMS_HOST") != "local.overhang.io"
run_for_prod = click.confirm( run_for_prod = click.confirm(
fmt.question( fmt.question(
@ -156,21 +159,31 @@ def ask_questions(config, defaults):
) )
def ask(question, key, config, defaults): def ask(
question: str, key: str, config: Dict[str, Any], defaults: Dict[str, Any]
) -> None:
default = env.render_str(config, config.get(key, defaults[key])) default = env.render_str(config, config.get(key, defaults[key]))
config[key] = click.prompt( config[key] = click.prompt(
fmt.question(question), prompt_suffix=" ", default=default, show_default=True fmt.question(question), prompt_suffix=" ", default=default, show_default=True
) )
def ask_bool(question, key, config, defaults): def ask_bool(
question: str, key: str, config: Dict[str, Any], defaults: Dict[str, Any]
) -> None:
default = config.get(key, defaults[key]) default = config.get(key, defaults[key])
config[key] = click.confirm( config[key] = click.confirm(
fmt.question(question), prompt_suffix=" ", default=default fmt.question(question), prompt_suffix=" ", default=default
) )
def ask_choice(question, key, config, defaults, choices): def ask_choice(
question: str,
key: str,
config: Dict[str, Any],
defaults: Dict[str, Any],
choices: List[str],
) -> None:
default = config.get(key, defaults[key]) default = config.get(key, defaults[key])
answer = click.prompt( answer = click.prompt(
fmt.question(question), fmt.question(question),

View File

@ -1,3 +1,5 @@
from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
from . import env from . import env
from . import fmt from . import fmt
from . import plugins from . import plugins
@ -9,25 +11,30 @@ echo "Loading settings $DJANGO_SETTINGS_MODULE"
class BaseJobRunner: class BaseJobRunner:
def __init__(self, root, config): def __init__(self, root: str, config: Dict[str, Any]):
self.root = root self.root = root
self.config = config self.config = config
def run_job_from_template(self, service, *path): def run_job_from_template(self, service: str, *path: str) -> None:
command = self.render(*path) command = self.render(*path)
self.run_job(service, command) self.run_job(service, command)
def render(self, *path): def render(self, *path: str) -> str:
return env.render_file(self.config, *path).strip() rendered = env.render_file(self.config, *path).strip()
if isinstance(rendered, bytes):
raise TypeError("Cannot load job from binary file")
return rendered
def run_job(self, service, command): def run_job(self, service: str, command: str) -> int:
raise NotImplementedError raise NotImplementedError
def iter_plugin_hooks(self, hook): def iter_plugin_hooks(
self, hook: str
) -> Iterator[Tuple[str, Union[Dict[str, str], List[str]]]]:
yield from plugins.iter_hooks(self.config, hook) yield from plugins.iter_hooks(self.config, hook)
def initialise(runner, limit_to=None): def initialise(runner: BaseJobRunner, limit_to: Optional[str] = None) -> None:
fmt.echo_info("Initialising all services...") fmt.echo_info("Initialising all services...")
if limit_to is None or limit_to == "mysql": if limit_to is None or limit_to == "mysql":
runner.run_job_from_template("mysql", "hooks", "mysql", "init") runner.run_job_from_template("mysql", "hooks", "mysql", "init")
@ -60,7 +67,13 @@ def initialise(runner, limit_to=None):
fmt.echo_info("All services initialised.") fmt.echo_info("All services initialised.")
def create_user_command(superuser, staff, username, email, password=None): def create_user_command(
superuser: str,
staff: bool,
username: str,
email: str,
password: Optional[str] = None,
) -> str:
command = BASE_OPENEDX_COMMAND command = BASE_OPENEDX_COMMAND
opts = "" opts = ""
@ -86,11 +99,11 @@ u.save()"
return command.format(opts=opts, username=username, email=email, password=password) return command.format(opts=opts, username=username, email=email, password=password)
def import_demo_course(runner): def import_demo_course(runner: BaseJobRunner) -> None:
runner.run_job_from_template("cms", "hooks", "cms", "importdemocourse") runner.run_job_from_template("cms", "hooks", "cms", "importdemocourse")
def set_theme(theme_name, domain_name, runner): def set_theme(theme_name: str, domain_name: str, runner: BaseJobRunner) -> None:
command = BASE_OPENEDX_COMMAND command = BASE_OPENEDX_COMMAND
command += """ command += """
echo "Assigning theme {theme_name} to {domain_name}..." echo "Assigning theme {theme_name} to {domain_name}..."

View File

@ -3,9 +3,10 @@ from copy import deepcopy
from glob import glob from glob import glob
import importlib import importlib
import os import os
import pkg_resources from typing import cast, Any, Dict, Iterator, List, Optional, Tuple, Type, Union
import appdirs import appdirs
import pkg_resources
from . import exceptions from . import exceptions
from . import fmt from . import fmt
@ -47,41 +48,50 @@ class BasePlugin:
`command` (click.Command): if a plugin exposes a `command` attribute, users will be able to run it from the command line as `tutor pluginname`. `command` (click.Command): if a plugin exposes a `command` attribute, users will be able to run it from the command line as `tutor pluginname`.
""" """
INSTALLED = [] INSTALLED: List["BasePlugin"] = []
_IS_LOADED = False _IS_LOADED = False
def __init__(self, name, obj): def __init__(self, name: str, obj: Any) -> None:
self.name = name self.name = name
self.config = get_callable_attr(obj, "config", {}) self.config = cast(
self.patches = get_callable_attr(obj, "patches", default={}) Dict[str, Dict[str, Any]], get_callable_attr(obj, "config", {})
self.hooks = get_callable_attr(obj, "hooks", default={}) )
self.templates_root = get_callable_attr(obj, "templates", default=None) self.patches = cast(
Dict[str, str], get_callable_attr(obj, "patches", default={})
)
self.hooks = cast(
Dict[str, Union[Dict[str, str], List[str]]],
get_callable_attr(obj, "hooks", default={}),
)
self.templates_root = cast(
Optional[str], get_callable_attr(obj, "templates", default=None)
)
self.command = getattr(obj, "command", None) self.command = getattr(obj, "command", None)
def config_key(self, key): def config_key(self, key: str) -> str:
""" """
Config keys in the "add" and "defaults" dicts should be prefixed by the plugin name, in uppercase. Config keys in the "add" and "defaults" dicts should be prefixed by the plugin name, in uppercase.
""" """
return self.name.upper() + "_" + key return self.name.upper() + "_" + key
@property @property
def config_add(self): def config_add(self) -> Dict[str, Any]:
return self.config.get("add", {}) return self.config.get("add", {})
@property @property
def config_set(self): def config_set(self) -> Dict[str, Any]:
return self.config.get("set", {}) return self.config.get("set", {})
@property @property
def config_defaults(self): def config_defaults(self) -> Dict[str, Any]:
return self.config.get("defaults", {}) return self.config.get("defaults", {})
@property @property
def version(self): def version(self) -> str:
raise NotImplementedError raise NotImplementedError
@classmethod @classmethod
def iter_installed(cls): def iter_installed(cls) -> Iterator["BasePlugin"]:
if not cls._IS_LOADED: if not cls._IS_LOADED:
for plugin in cls.iter_load(): for plugin in cls.iter_load():
cls.INSTALLED.append(plugin) cls.INSTALLED.append(plugin)
@ -89,7 +99,7 @@ class BasePlugin:
yield from cls.INSTALLED yield from cls.INSTALLED
@classmethod @classmethod
def iter_load(cls): def iter_load(cls) -> Iterator["BasePlugin"]:
raise NotImplementedError raise NotImplementedError
@ -103,16 +113,18 @@ class EntrypointPlugin(BasePlugin):
ENTRYPOINT = "tutor.plugin.v0" ENTRYPOINT = "tutor.plugin.v0"
def __init__(self, entrypoint): def __init__(self, entrypoint: pkg_resources.EntryPoint) -> None:
super().__init__(entrypoint.name, entrypoint.load()) super().__init__(entrypoint.name, entrypoint.load())
self.entrypoint = entrypoint self.entrypoint = entrypoint
@property @property
def version(self): def version(self) -> str:
if not self.entrypoint.dist:
return "0.0.0"
return self.entrypoint.dist.version return self.entrypoint.dist.version
@classmethod @classmethod
def iter_load(cls): def iter_load(cls) -> Iterator["EntrypointPlugin"]:
for entrypoint in pkg_resources.iter_entry_points(cls.ENTRYPOINT): for entrypoint in pkg_resources.iter_entry_points(cls.ENTRYPOINT):
yield cls(entrypoint) yield cls(entrypoint)
@ -124,21 +136,24 @@ class OfficialPlugin(BasePlugin):
""" """
@classmethod @classmethod
def load(cls, name): def load(cls, name: str) -> BasePlugin:
plugin = cls(name) plugin = cls(name)
cls.INSTALLED.append(plugin) cls.INSTALLED.append(plugin)
return plugin return plugin
def __init__(self, name): def __init__(self, name: str):
self.module = importlib.import_module("tutor{}.plugin".format(name)) self.module = importlib.import_module("tutor{}.plugin".format(name))
super().__init__(name, self.module) super().__init__(name, self.module)
@property @property
def version(self): def version(self) -> str:
return self.module.__version__ version = getattr(self.module, "__version__")
if not isinstance(version, str):
raise TypeError("OfficialPlugin __version__ must be 'str'")
return version
@classmethod @classmethod
def iter_load(cls): def iter_load(cls) -> Iterator[BasePlugin]:
yield from [] yield from []
@ -148,18 +163,20 @@ class DictPlugin(BasePlugin):
os.environ.get(ROOT_ENV_VAR_NAME, "") os.environ.get(ROOT_ENV_VAR_NAME, "")
) or appdirs.user_data_dir(appname="tutor-plugins") ) or appdirs.user_data_dir(appname="tutor-plugins")
def __init__(self, data): def __init__(self, data: Dict[str, Any]):
Module = namedtuple("Module", data.keys()) Module = namedtuple("Module", data.keys()) # type: ignore
obj = Module(**data) obj = Module(**data) # type: ignore
super().__init__(data["name"], obj) super().__init__(data["name"], obj)
self._version = data["version"] self._version = data["version"]
@property @property
def version(self): def version(self) -> str:
if not isinstance(self._version, str):
raise TypeError("DictPlugin.__version__ must be str")
return self._version return self._version
@classmethod @classmethod
def iter_load(cls): def iter_load(cls) -> Iterator[BasePlugin]:
for path in glob(os.path.join(cls.ROOT, "*.yml")): for path in glob(os.path.join(cls.ROOT, "*.yml")):
with open(path) as f: with open(path) as f:
data = serialize.load(f) data = serialize.load(f)
@ -176,13 +193,17 @@ class DictPlugin(BasePlugin):
class Plugins: class Plugins:
PLUGIN_CLASSES = [OfficialPlugin, EntrypointPlugin, DictPlugin] PLUGIN_CLASSES: List[Type[BasePlugin]] = [
OfficialPlugin,
EntrypointPlugin,
DictPlugin,
]
def __init__(self, config): def __init__(self, config: Dict[str, Any]):
self.config = deepcopy(config) self.config = deepcopy(config)
self.patches = {} self.patches: Dict[str, Dict[str, str]] = {}
self.hooks = {} self.hooks: Dict[str, Dict[str, Union[Dict[str, str], List[str]]]] = {}
self.template_roots = {} self.template_roots: Dict[str, str] = {}
for plugin in self.iter_enabled(): for plugin in self.iter_enabled():
for patch_name, content in plugin.patches.items(): for patch_name, content in plugin.patches.items():
@ -196,12 +217,12 @@ class Plugins:
self.hooks[hook_name][plugin.name] = services self.hooks[hook_name][plugin.name] = services
@classmethod @classmethod
def clear(cls): def clear(cls) -> None:
for PluginClass in cls.PLUGIN_CLASSES: for PluginClass in cls.PLUGIN_CLASSES:
PluginClass.INSTALLED.clear() PluginClass.INSTALLED.clear()
@classmethod @classmethod
def iter_installed(cls): def iter_installed(cls) -> Iterator[BasePlugin]:
""" """
Iterate on all installed plugins. Plugins are deduplicated by name. The list of installed plugins is cached to Iterate on all installed plugins. Plugins are deduplicated by name. The list of installed plugins is cached to
prevent too many re-computations, which happens a lot. prevent too many re-computations, which happens a lot.
@ -213,40 +234,44 @@ class Plugins:
installed_plugin_names.add(plugin.name) installed_plugin_names.add(plugin.name)
yield plugin yield plugin
def iter_enabled(self): def iter_enabled(self) -> Iterator[BasePlugin]:
for plugin in self.iter_installed(): for plugin in self.iter_installed():
if is_enabled(self.config, plugin.name): if is_enabled(self.config, plugin.name):
yield plugin yield plugin
def iter_patches(self, name): def iter_patches(self, name: str) -> Iterator[Tuple[str, str]]:
plugin_patches = self.patches.get(name, {}) plugin_patches = self.patches.get(name, {})
plugins = sorted(plugin_patches.keys()) plugins = sorted(plugin_patches.keys())
for plugin in plugins: for plugin in plugins:
yield plugin, plugin_patches[plugin] yield plugin, plugin_patches[plugin]
def iter_hooks(self, hook_name): def iter_hooks(
self, hook_name: str
) -> Iterator[Tuple[str, Union[Dict[str, str], List[str]]]]:
yield from self.hooks.get(hook_name, {}).items() yield from self.hooks.get(hook_name, {}).items()
def get_callable_attr(plugin, attr_name, default=None): def get_callable_attr(
plugin: Any, attr_name: str, default: Optional[Any] = None
) -> Optional[Any]:
attr = getattr(plugin, attr_name, default) attr = getattr(plugin, attr_name, default)
if callable(attr): if callable(attr):
attr = attr() attr = attr()
return attr return attr
def is_installed(name): def is_installed(name: str) -> bool:
for plugin in iter_installed(): for plugin in iter_installed():
if name == plugin.name: if name == plugin.name:
return True return True
return False return False
def iter_installed(): def iter_installed() -> Iterator[BasePlugin]:
yield from Plugins.iter_installed() yield from Plugins.iter_installed()
def enable(config, name): def enable(config: Dict[str, Any], name: str) -> None:
if not is_installed(name): if not is_installed(name):
raise exceptions.TutorError("plugin '{}' is not installed.".format(name)) raise exceptions.TutorError("plugin '{}' is not installed.".format(name))
if is_enabled(config, name): if is_enabled(config, name):
@ -257,7 +282,7 @@ def enable(config, name):
config[CONFIG_KEY].sort() config[CONFIG_KEY].sort()
def disable(config, name): def disable(config: Dict[str, Any], name: str) -> None:
fmt.echo_info("Disabling plugin {}...".format(name)) fmt.echo_info("Disabling plugin {}...".format(name))
for plugin in Plugins(config).iter_enabled(): for plugin in Plugins(config).iter_enabled():
if name == plugin.name: if name == plugin.name:
@ -271,17 +296,19 @@ def disable(config, name):
fmt.echo_info(" Plugin disabled") fmt.echo_info(" Plugin disabled")
def iter_enabled(config): def iter_enabled(config: Dict[str, Any]) -> Iterator[BasePlugin]:
yield from Plugins(config).iter_enabled() yield from Plugins(config).iter_enabled()
def is_enabled(config, name): def is_enabled(config: Dict[str, Any], name: str) -> bool:
return name in config.get(CONFIG_KEY, []) return name in config.get(CONFIG_KEY, [])
def iter_patches(config, name): def iter_patches(config: Dict[str, str], name: str) -> Iterator[Tuple[str, str]]:
yield from Plugins(config).iter_patches(name) yield from Plugins(config).iter_patches(name)
def iter_hooks(config, hook_name): def iter_hooks(
config: Dict[str, Any], hook_name: str
) -> Iterator[Tuple[str, Union[Dict[str, str], List[str]]]]:
yield from Plugins(config).iter_hooks(hook_name) yield from Plugins(config).iter_hooks(hook_name)

View File

@ -1,28 +1,27 @@
import re import re
from typing import cast, Any, Dict, IO, Iterator, Tuple, Union
import yaml import yaml
from _io import TextIOWrapper
from yaml.parser import ParserError from yaml.parser import ParserError
from yaml.scanner import ScannerError from yaml.scanner import ScannerError
import click import click
def load(stream): def load(stream: Union[str, IO[str]]) -> Dict[str, str]:
return yaml.load(stream, Loader=yaml.SafeLoader) return cast(Dict[str, str], yaml.load(stream, Loader=yaml.SafeLoader))
def load_all(stream): def load_all(stream: str) -> Iterator[Any]:
return yaml.load_all(stream, Loader=yaml.SafeLoader) return yaml.load_all(stream, Loader=yaml.SafeLoader)
def dump(content, fileobj): def dump(content: Dict[str, str], fileobj: TextIOWrapper) -> None:
yaml.dump(content, stream=fileobj, default_flow_style=False) yaml.dump(content, stream=fileobj, default_flow_style=False)
def dumps(content): def parse(v: Union[str, IO[str]]) -> Any:
return yaml.dump(content, stream=None, default_flow_style=False)
def parse(v):
""" """
Parse a yaml-formatted string. Parse a yaml-formatted string.
""" """
@ -37,7 +36,7 @@ class YamlParamType(click.ParamType):
name = "yaml" name = "yaml"
PARAM_REGEXP = r"(?P<key>[a-zA-Z0-9_-]+)=(?P<value>.*)" PARAM_REGEXP = r"(?P<key>[a-zA-Z0-9_-]+)=(?P<value>.*)"
def convert(self, value, param, ctx): def convert(self, value: str, param: Any, ctx: Any) -> Tuple[str, Any]:
match = re.match(self.PARAM_REGEXP, value) match = re.match(self.PARAM_REGEXP, value)
if not match: if not match:
self.fail("'{}' is not of the form 'key=value'.".format(value), param, ctx) self.fail("'{}' is not of the form 'key=value'.".format(value), param, ctx)

View File

@ -7,16 +7,17 @@ import string
import struct import struct
import subprocess import subprocess
import sys import sys
from typing import List, Tuple
import click import click
from Crypto.PublicKey import RSA
from Crypto.Protocol.KDF import bcrypt, bcrypt_check from Crypto.Protocol.KDF import bcrypt, bcrypt_check
from Crypto.PublicKey import RSA
from Crypto.PublicKey.RSA import RsaKey
from . import exceptions from . import exceptions, fmt
from . import fmt
def encrypt(text): def encrypt(text: str) -> str:
""" """
Encrypt some textual content with bcrypt. Encrypt some textual content with bcrypt.
https://pycryptodome.readthedocs.io/en/latest/src/protocol/kdf.html#bcrypt https://pycryptodome.readthedocs.io/en/latest/src/protocol/kdf.html#bcrypt
@ -26,7 +27,7 @@ def encrypt(text):
return bcrypt(text.encode(), 12).decode() return bcrypt(text.encode(), 12).decode()
def verify_encrypted(encrypted, text): def verify_encrypted(encrypted: str, text: str) -> bool:
""" """
Return True/False if the encrypted content corresponds to the unencrypted text. Return True/False if the encrypted content corresponds to the unencrypted text.
""" """
@ -37,7 +38,7 @@ def verify_encrypted(encrypted, text):
return False return False
def ensure_file_directory_exists(path): def ensure_file_directory_exists(path: str) -> None:
""" """
Create file's base directory if it does not exist. Create file's base directory if it does not exist.
""" """
@ -46,17 +47,17 @@ def ensure_file_directory_exists(path):
os.makedirs(directory) os.makedirs(directory)
def random_string(length): def random_string(length: int) -> str:
return "".join( return "".join(
[random.choice(string.ascii_letters + string.digits) for _ in range(length)] [random.choice(string.ascii_letters + string.digits) for _ in range(length)]
) )
def list_if(services): def list_if(services: List[Tuple[str, bool]]) -> str:
return json.dumps([service[0] for service in services if service[1]]) return json.dumps([service[0] for service in services if service[1]])
def common_domain(d1, d2): def common_domain(d1: str, d2: str) -> str:
""" """
Return the common domain between two domain names. Return the common domain between two domain names.
@ -73,7 +74,7 @@ def common_domain(d1, d2):
return ".".join(common[::-1]) return ".".join(common[::-1])
def reverse_host(domain): def reverse_host(domain: str) -> str:
""" """
Return the reverse domain name, java-style. Return the reverse domain name, java-style.
@ -82,7 +83,7 @@ def reverse_host(domain):
return ".".join(domain.split(".")[::-1]) return ".".join(domain.split(".")[::-1])
def rsa_private_key(bits=2048): def rsa_private_key(bits: int = 2048) -> str:
""" """
Export an RSA private key in PEM format. Export an RSA private key in PEM format.
""" """
@ -90,20 +91,20 @@ def rsa_private_key(bits=2048):
return key.export_key().decode() return key.export_key().decode()
def rsa_import_key(key): def rsa_import_key(key: str) -> RsaKey:
""" """
Import PEM-formatted RSA key and return the corresponding object. Import PEM-formatted RSA key and return the corresponding object.
""" """
return RSA.import_key(key.encode()) return RSA.import_key(key.encode())
def long_to_base64(n): def long_to_base64(n: int) -> str:
""" """
Borrowed from jwkest.__init__ Borrowed from jwkest.__init__
""" """
def long2intarr(long_int): def long2intarr(long_int: int) -> List[int]:
_bytes = [] _bytes: List[int] = []
while long_int: while long_int:
long_int, r = divmod(long_int, 256) long_int, r = divmod(long_int, 256)
_bytes.insert(0, r) _bytes.insert(0, r)
@ -117,16 +118,7 @@ def long_to_base64(n):
return s.decode("ascii") return s.decode("ascii")
def walk_files(path): def is_root() -> bool:
"""
Iterate on file paths located in directory.
"""
for dirpath, _, filenames in os.walk(path):
for filename in filenames:
yield os.path.join(dirpath, filename)
def is_root():
""" """
Check whether tutor is being run as root/sudo. Check whether tutor is being run as root/sudo.
""" """
@ -136,7 +128,7 @@ def is_root():
return get_user_id() == 0 return get_user_id() == 0
def get_user_id(): def get_user_id() -> int:
""" """
Portable way to get user ID. Note: I have no idea if it actually works on windows... Portable way to get user ID. Note: I have no idea if it actually works on windows...
""" """
@ -146,14 +138,14 @@ def get_user_id():
return os.getuid() return os.getuid()
def docker_run(*command): def docker_run(*command: str) -> int:
args = ["run", "--rm"] args = ["run", "--rm"]
if is_a_tty(): if is_a_tty():
args.append("-it") args.append("-it")
return docker(*args, *command) return docker(*args, *command)
def docker(*command): def docker(*command: str) -> int:
if shutil.which("docker") is None: if shutil.which("docker") is None:
raise exceptions.TutorError( raise exceptions.TutorError(
"docker is not installed. Please follow instructions from https://docs.docker.com/install/" "docker is not installed. Please follow instructions from https://docs.docker.com/install/"
@ -161,7 +153,7 @@ def docker(*command):
return execute("docker", *command) return execute("docker", *command)
def docker_compose(*command): def docker_compose(*command: str) -> int:
if shutil.which("docker-compose") is None: if shutil.which("docker-compose") is None:
raise exceptions.TutorError( raise exceptions.TutorError(
"docker-compose is not installed. Please follow instructions from https://docs.docker.com/compose/install/" "docker-compose is not installed. Please follow instructions from https://docs.docker.com/compose/install/"
@ -169,7 +161,7 @@ def docker_compose(*command):
return execute("docker-compose", *command) return execute("docker-compose", *command)
def kubectl(*command): def kubectl(*command: str) -> int:
if shutil.which("kubectl") is None: if shutil.which("kubectl") is None:
raise exceptions.TutorError( raise exceptions.TutorError(
"kubectl is not installed. Please follow instructions from https://kubernetes.io/docs/tasks/tools/install-kubectl/" "kubectl is not installed. Please follow instructions from https://kubernetes.io/docs/tasks/tools/install-kubectl/"
@ -177,7 +169,7 @@ def kubectl(*command):
return execute("kubectl", *command) return execute("kubectl", *command)
def is_a_tty(): def is_a_tty() -> bool:
""" """
Return True if stdin is able to allocate a tty. Tty allocation sometimes cannot be Return True if stdin is able to allocate a tty. Tty allocation sometimes cannot be
enabled, for instance in cron jobs enabled, for instance in cron jobs
@ -185,7 +177,7 @@ def is_a_tty():
return os.isatty(sys.stdin.fileno()) return os.isatty(sys.stdin.fileno())
def execute(*command): def execute(*command: str) -> int:
click.echo(fmt.command(" ".join(command))) click.echo(fmt.command(" ".join(command)))
with subprocess.Popen(command) as p: with subprocess.Popen(command) as p:
try: try:
@ -204,9 +196,10 @@ def execute(*command):
raise exceptions.TutorError( raise exceptions.TutorError(
"Command failed with status {}: {}".format(result, " ".join(command)) "Command failed with status {}: {}".format(result, " ".join(command))
) )
return result
def check_output(*command): def check_output(*command: str) -> bytes:
click.echo(fmt.command(" ".join(command))) click.echo(fmt.command(" ".join(command)))
try: try:
return subprocess.check_output(command) return subprocess.check_output(command)