diff --git a/changelog.d/20230412_100608_regis_palm.md b/changelog.d/20230412_100608_regis_palm.md index 13a2011..1641eeb 100644 --- a/changelog.d/20230412_100608_regis_palm.md +++ b/changelog.d/20230412_100608_regis_palm.md @@ -3,3 +3,9 @@ - 💥[Improvement] During registration, the honor code and terms of service links are no longer visible by default. For most platforms, these links did not work anyway. - 💥[Deprecation] Halt support for Python 3.7. The binary release of Tutor is also no longer compatible with macOS 10. - 💥[Deprecation] Drop support for `docker-compose`, also known as Compose V1. The `docker compose` (no hyphen) plugin must be installed. + - 💥[Refactor] We simplify the hooks API by getting rid of the `ContextTemplate`, `FilterTemplate` and `ActionTemplate` classes. As a consequences, the following changes occur: + - `APP` was previously a ContextTemplate, and is now a dictionary of contexts indexed by name. Developers who implemented this context should replace `Contexts.APP(...)` by `Contexts.app(...)`. + - Removed the `ENV_PATCH` filter, which was for internal use only anyway. + - The `PLUGIN_LOADED` ActionTemplate is now an Action which takes a single argument. (the plugin name) + - 💥[Refactor] We refactored the hooks API further by removing the static hook indexes and the hooks names. As a consequence: + - The syntactic sugar functions from the "filters" and "actions" modules were all removed: `get`, `add*`, `iterate*`, `apply*`, `do*`, etc. diff --git a/docs/reference/api/hooks/actions.rst b/docs/reference/api/hooks/actions.rst index b9f94bb..bf73ddb 100644 --- a/docs/reference/api/hooks/actions.rst +++ b/docs/reference/api/hooks/actions.rst @@ -9,9 +9,6 @@ Actions are one of the two types of hooks (the other being :ref:`filters`) that .. autoclass:: tutor.core.hooks.Action :members: -.. autoclass:: tutor.core.hooks.ActionTemplate - :members: - .. The following are only to ensure that the docs build without warnings .. class:: tutor.core.hooks.actions.T .. class:: tutor.types.Config diff --git a/docs/reference/api/hooks/filters.rst b/docs/reference/api/hooks/filters.rst index f14a582..81ec443 100644 --- a/docs/reference/api/hooks/filters.rst +++ b/docs/reference/api/hooks/filters.rst @@ -9,9 +9,6 @@ Filters are one of the two types of hooks (the other being :ref:`actions`) that .. autoclass:: tutor.core.hooks.Filter :members: -.. autoclass:: tutor.core.hooks.FilterTemplate - :members: - .. The following are only to ensure that the docs build without warnings .. class:: tutor.core.hooks.filters.T1 .. class:: tutor.core.hooks.filters.T2 diff --git a/tests/commands/test_config.py b/tests/commands/test_config.py index f344897..5c9dcea 100644 --- a/tests/commands/test_config.py +++ b/tests/commands/test_config.py @@ -1,7 +1,7 @@ import unittest -from tutor import config as tutor_config from tests.helpers import temporary_root +from tutor import config as tutor_config from .base import TestCommandMixin diff --git a/tests/core/hooks/test_actions.py b/tests/core/hooks/test_actions.py index 60654a6..e0c884b 100644 --- a/tests/core/hooks/test_actions.py +++ b/tests/core/hooks/test_actions.py @@ -8,16 +8,12 @@ class PluginActionsTests(unittest.TestCase): def setUp(self) -> None: self.side_effect_int = 0 - def tearDown(self) -> None: - super().tearDown() - actions.clear_all(context="tests") - def run(self, result: t.Any = None) -> t.Any: with contexts.enter("tests"): return super().run(result=result) def test_do(self) -> None: - action: actions.Action[int] = actions.get("test-action") + action: actions.Action[int] = actions.Action() @action.add() def _test_action_1(increment: int) -> None: @@ -31,29 +27,33 @@ class PluginActionsTests(unittest.TestCase): self.assertEqual(3, self.side_effect_int) def test_priority(self) -> None: - @actions.add("test-action", priority=2) + action: actions.Action[[]] = actions.Action() + + @action.add(priority=2) def _test_action_1() -> None: self.side_effect_int += 4 - @actions.add("test-action", priority=1) + @action.add(priority=1) def _test_action_2() -> None: self.side_effect_int = self.side_effect_int // 2 # Action 2 must be performed before action 1 self.side_effect_int = 4 - actions.do("test-action") + action.do() self.assertEqual(6, self.side_effect_int) def test_equal_priority(self) -> None: - @actions.add("test-action", priority=2) + action: actions.Action[[]] = actions.Action() + + @action.add(priority=2) def _test_action_1() -> None: self.side_effect_int += 4 - @actions.add("test-action", priority=2) + @action.add(priority=2) def _test_action_2() -> None: self.side_effect_int = self.side_effect_int // 2 # Action 2 must be performed after action 1 self.side_effect_int = 4 - actions.do("test-action") + action.do() self.assertEqual(4, self.side_effect_int) diff --git a/tests/core/hooks/test_filters.py b/tests/core/hooks/test_filters.py index 4d7b821..3443e91 100644 --- a/tests/core/hooks/test_filters.py +++ b/tests/core/hooks/test_filters.py @@ -7,32 +7,32 @@ from tutor.core.hooks import contexts, filters class PluginFiltersTests(unittest.TestCase): - def tearDown(self) -> None: - super().tearDown() - filters.clear_all(context="tests") - def run(self, result: t.Any = None) -> t.Any: with contexts.enter("tests"): return super().run(result=result) def test_add(self) -> None: - @filters.add("tests:count-sheeps") + filtre: filters.Filter[int, []] = filters.Filter() + + @filtre.add() def filter1(value: int) -> int: return value + 1 - value = filters.apply("tests:count-sheeps", 0) + value = filtre.apply(0) self.assertEqual(1, value) def test_add_items(self) -> None: - @filters.add("tests:add-sheeps") + filtre: filters.Filter[list[int], []] = filters.Filter() + + @filtre.add() def filter1(sheeps: list[int]) -> list[int]: return sheeps + [0] - filters.add_item("tests:add-sheeps", 1) - filters.add_item("tests:add-sheeps", 2) - filters.add_items("tests:add-sheeps", [3, 4]) + filtre.add_item(1) + filtre.add_item(2) + filtre.add_items([3, 4]) - sheeps: list[int] = filters.apply("tests:add-sheeps", []) + sheeps: list[int] = filtre.apply([]) self.assertEqual([0, 1, 2, 3, 4], sheeps) def test_filter_callbacks(self) -> None: @@ -42,20 +42,20 @@ class PluginFiltersTests(unittest.TestCase): self.assertEqual(1, callback.apply(0)) def test_filter_context(self) -> None: + filtre: filters.Filter[list[int], []] = filters.Filter() with contexts.enter("testcontext"): - filters.add_item("test:sheeps", 1) - filters.add_item("test:sheeps", 2) + filtre.add_item(1) + filtre.add_item(2) - self.assertEqual([1, 2], filters.apply("test:sheeps", [])) - self.assertEqual( - [1], filters.apply_from_context("testcontext", "test:sheeps", []) - ) + self.assertEqual([1, 2], filtre.apply([])) + self.assertEqual([1], filtre.apply_from_context("testcontext", [])) def test_clear_context(self) -> None: + filtre: filters.Filter[list[int], []] = filters.Filter() with contexts.enter("testcontext"): - filters.add_item("test:sheeps", 1) - filters.add_item("test:sheeps", 2) + filtre.add_item(1) + filtre.add_item(2) - self.assertEqual([1, 2], filters.apply("test:sheeps", [])) - filters.clear("test:sheeps", context="testcontext") - self.assertEqual([2], filters.apply("test:sheeps", [])) + self.assertEqual([1, 2], filtre.apply([])) + filtre.clear(context="testcontext") + self.assertEqual([2], filtre.apply([])) diff --git a/tests/test_plugins_v0.py b/tests/test_plugins_v0.py index d343087..248cf80 100644 --- a/tests/test_plugins_v0.py +++ b/tests/test_plugins_v0.py @@ -92,7 +92,7 @@ class PluginsTests(PluginsTestCase): def test_plugin_without_patches(self) -> None: plugins_v0.DictPlugin({"name": "plugin1"}) - plugins.load("plugin1") + plugins.load_all(["plugin1"]) patches = list(plugins.iter_patches("patch1")) self.assertEqual([], patches) diff --git a/tutor/bindmount.py b/tutor/bindmount.py index 0f2d206..fab7dae 100644 --- a/tutor/bindmount.py +++ b/tutor/bindmount.py @@ -1,9 +1,9 @@ from __future__ import annotations -from functools import lru_cache import os import re import typing as t +from functools import lru_cache from tutor import hooks diff --git a/tutor/commands/compose.py b/tutor/commands/compose.py index 53112d0..c7a4f6d 100644 --- a/tutor/commands/compose.py +++ b/tutor/commands/compose.py @@ -4,9 +4,10 @@ import os import click +from tutor import bindmount from tutor import config as tutor_config from tutor import env as tutor_env -from tutor import bindmount, hooks, utils +from tutor import hooks, utils from tutor.commands import jobs from tutor.commands.context import BaseTaskContext from tutor.core.hooks import Filter # pylint: disable=unused-import diff --git a/tutor/commands/jobs.py b/tutor/commands/jobs.py index a6648c7..0412fc1 100644 --- a/tutor/commands/jobs.py +++ b/tutor/commands/jobs.py @@ -37,11 +37,11 @@ def _add_core_init_tasks() -> None: The context is important, because it allows us to select the init scripts based on the --limit argument. """ - with hooks.Contexts.APP("mysql").enter(): + with hooks.Contexts.app("mysql").enter(): hooks.Filters.CLI_DO_INIT_TASKS.add_item( ("mysql", env.read_core_template_file("jobs", "init", "mysql.sh")) ) - with hooks.Contexts.APP("lms").enter(): + with hooks.Contexts.app("lms").enter(): hooks.Filters.CLI_DO_INIT_TASKS.add_item( ( "lms", @@ -54,7 +54,7 @@ def _add_core_init_tasks() -> None: hooks.Filters.CLI_DO_INIT_TASKS.add_item( ("lms", env.read_core_template_file("jobs", "init", "lms.sh")) ) - with hooks.Contexts.APP("cms").enter(): + with hooks.Contexts.app("cms").enter(): hooks.Filters.CLI_DO_INIT_TASKS.add_item( ("cms", env.read_core_template_file("jobs", "init", "cms.sh")) ) @@ -64,7 +64,7 @@ def _add_core_init_tasks() -> None: @click.option("-l", "--limit", help="Limit initialisation to this service or plugin") def initialise(limit: t.Optional[str]) -> t.Iterator[tuple[str, str]]: fmt.echo_info("Initialising all services...") - filter_context = hooks.Contexts.APP(limit).name if limit else None + filter_context = hooks.Contexts.app(limit).name if limit else None # Deprecated pre-init tasks for service, path in hooks.Filters.COMMANDS_PRE_INIT.iterate_from_context( diff --git a/tutor/config.py b/tutor/config.py index 3459a82..a5e1eca 100644 --- a/tutor/config.py +++ b/tutor/config.py @@ -304,7 +304,7 @@ def _remove_plugin_config_overrides_on_unload( # Find the configuration entries that were overridden by the plugin and # remove them from the current config for key, _value in hooks.Filters.CONFIG_OVERRIDES.iterate_from_context( - hooks.Contexts.APP(plugin).name + hooks.Contexts.app(plugin).name ): value = config.pop(key, None) value = env.render_unknown(config, value) diff --git a/tutor/core/hooks/__init__.py b/tutor/core/hooks/__init__.py index 1609444..fa1451a 100644 --- a/tutor/core/hooks/__init__.py +++ b/tutor/core/hooks/__init__.py @@ -1,15 +1,13 @@ import typing as t -from .actions import Action, ActionTemplate -from .actions import clear_all as _clear_all_actions -from .contexts import Context, ContextTemplate -from .filters import Filter, FilterTemplate -from .filters import clear_all as _clear_all_filters +from .actions import Action +from .contexts import Context +from .filters import Filter def clear_all(context: t.Optional[str] = None) -> None: """ Clear both actions and filters. """ - _clear_all_actions(context=context) - _clear_all_filters(context=context) + Action.clear_all(context=context) + Filter.clear_all(context=context) diff --git a/tutor/core/hooks/actions.py b/tutor/core/hooks/actions.py index e6050d9..924c5ae 100644 --- a/tutor/core/hooks/actions.py +++ b/tutor/core/hooks/actions.py @@ -5,9 +5,11 @@ __license__ = "Apache 2.0" import sys import typing as t +from weakref import WeakSet from typing_extensions import ParamSpec + from . import priorities from .contexts import Contextualized @@ -44,32 +46,25 @@ class Action(t.Generic[T]): This is the typical action lifecycle: - 1. Create an action with method :py:meth:`get`. - 2. Add callbacks with method :py:meth:`add`. - 3. Call the action callbacks with method :py:meth:`do`. + 1. Create an action with ``Action()``. + 2. Add callbacks with :py:meth:`add`. + 3. Call the action callbacks with :py:meth:`do`. - The ``P`` type parameter of the Action class corresponds to the expected signature of + The ``T`` type parameter of the Action class corresponds to the expected signature of the action callbacks. For instance, ``Action[[str, int]]`` means that the action callbacks are expected to take two arguments: one string and one integer. - This strong typing makes it easier for plugin developers to quickly check whether they are adding and calling action callbacks correctly. + This strong typing makes it easier for plugin developers to quickly check whether + they are adding and calling action callbacks correctly. """ - INDEX: dict[str, "Action[t.Any]"] = {} + # Keep a weak reference to all created filters. This allows us to clear them when + # necessary. + INSTANCES: WeakSet[Action[t.Any]] = WeakSet() - def __init__(self, name: str) -> None: - self.name = name + def __init__(self) -> None: self.callbacks: list[ActionCallback[T]] = [] - - def __repr__(self) -> str: - return f"{self.__class__.__name__}('{self.name}')" - - @classmethod - def get(cls, name: str) -> "Action[t.Any]": - """ - Get an existing action with the given name from the index, or create one. - """ - return cls.INDEX.setdefault(name, cls(name)) + self.INSTANCES.add(self) def add( self, priority: t.Optional[int] = None @@ -144,7 +139,7 @@ class Action(t.Generic[T]): ) except: sys.stderr.write( - f"Error applying action '{self.name}': func={callback.func} contexts={callback.contexts}'\n" + f"Error applying action: func={callback.func} contexts={callback.contexts}'\n" ) raise @@ -168,98 +163,10 @@ class Action(t.Generic[T]): if not callback.is_in_context(context) ] - -class ActionTemplate(t.Generic[T]): - """ - Action templates are for actions for which the name needs to be formatted - before the action can be applied. - - Action templates can generate different :py:class:`Action` objects for which the - name matches a certain template. - - Templated actions must be formatted with ``(*args)`` before being applied. For example:: - - action_template = ActionTemplate("namespace:{0}") - - # Return the action named "namespace:name" - my_action = action_template("name") - - @my_action.add() - def my_callback(): - ... - - my_action.do() - """ - - def __init__(self, name: str): - self.template = name - - def __repr__(self) -> str: - return f"{self.__class__.__name__}('{self.template}')" - - def __call__(self, *args: t.Any, **kwargs: t.Any) -> Action[T]: - name = self.template.format(*args, **kwargs) - action: Action[T] = Action.get(name) - return action - - -# Syntactic sugar -get = Action.get - - -def get_template(name: str) -> ActionTemplate[t.Any]: - """ - Create an action with a template name. - """ - return ActionTemplate(name) - - -def add( - name: str, priority: t.Optional[int] = None -) -> t.Callable[[ActionCallbackFunc[T]], ActionCallbackFunc[T]]: - """ - Decorator to add a callback action associated to a name. - """ - return get(name).add(priority=priority) - - -def do( - name: str, - *args: T.args, - **kwargs: T.kwargs, -) -> None: - """ - Run action callbacks associated to a name/context. - """ - action: Action[T] = Action.get(name) - action.do(*args, **kwargs) - - -def do_from_context( - context: str, - name: str, - *args: T.args, - **kwargs: T.kwargs, -) -> None: - """ - Same as :py:func:`do` but only run the callbacks that were created in a given context. - """ - action: Action[T] = Action.get(name) - action.do_from_context(context, *args, **kwargs) - - -def clear_all(context: t.Optional[str] = None) -> None: - """ - Clear any previously defined filter with the given context. - - This will call :py:func:`clear` with all action names. - """ - for name in Action.INDEX: - clear(name, context=context) - - -def clear(name: str, context: t.Optional[str] = None) -> None: - """ - Clear any previously defined action with the given name and context. - """ - Action.get(name).clear(context=context) + @classmethod + def clear_all(cls, context: t.Optional[str] = None) -> None: + """ + Clear any previously defined action with the given context. + """ + for action in cls.INSTANCES: + action.clear(context) diff --git a/tutor/core/hooks/contexts.py b/tutor/core/hooks/contexts.py index 1dc48a8..940d2ba 100644 --- a/tutor/core/hooks/contexts.py +++ b/tutor/core/hooks/contexts.py @@ -43,22 +43,6 @@ class Context: Context.CURRENT.pop() -class ContextTemplate: - """ - Context templates are for filters for which the name needs to be formatted - before the filter can be applied. - """ - - def __init__(self, name: str): - self.template = name - - def __repr__(self) -> str: - return f"{self.__class__.__name__}('{self.template}')" - - def __call__(self, *args: t.Any, **kwargs: t.Any) -> Context: - return Context(self.template.format(*args, **kwargs)) - - class Contextualized: """ This is a simple class to store the current context in hooks. diff --git a/tutor/core/hooks/filters.py b/tutor/core/hooks/filters.py index 9e9b70e..ef42325 100644 --- a/tutor/core/hooks/filters.py +++ b/tutor/core/hooks/filters.py @@ -5,6 +5,7 @@ __license__ = "Apache 2.0" import sys import typing as t +from weakref import WeakSet from typing_extensions import Concatenate, ParamSpec @@ -43,16 +44,16 @@ class Filter(t.Generic[T1, T2]): This is the typical filter lifecycle: - 1. Create an action with method :py:meth:`get`. - 2. Add callbacks with method :py:meth:`add`. + 1. Create a filter with ``Filter()``. + 2. Add callbacks with :py:meth:`add`. 3. Call the filter callbacks with method :py:meth:`apply`. The result of each callback is passed as the first argument to the next one. Thus, the type of the first argument must match the callback return type. - The `T` and `P` type parameters of the Filter class correspond to the expected - signature of the filter callbacks. `T` is the type of the first argument (and thus - the return value type as well) and `P` is the signature of the other arguments. + The ``T1`` and ``T2`` type parameters of the Filter class correspond to the expected + signature of the filter callbacks. ``T1`` is the type of the first argument (and thus + the return value type as well) and ``T2`` is the signature of the other arguments. For instance, `Filter[str, [int]]` means that the filter callbacks are expected to take two arguments: one string and one integer. Each callback must then return a @@ -62,21 +63,13 @@ class Filter(t.Generic[T1, T2]): they are adding and calling filter callbacks correctly. """ - INDEX: dict[str, "Filter[t.Any, t.Any]"] = {} + # Keep a weak reference to all created filters. This allows us to clear them when + # necessary. + INSTANCES: WeakSet[Filter[t.Any, t.Any]] = WeakSet() - def __init__(self, name: str) -> None: - self.name = name + def __init__(self) -> None: self.callbacks: list[FilterCallback[T1, T2]] = [] - - def __repr__(self) -> str: - return f"{self.__class__.__name__}('{self.name}')" - - @classmethod - def get(cls, name: str) -> "Filter[t.Any, t.Any]": - """ - Get an existing action with the given name from the index, or create one. - """ - return cls.INDEX.setdefault(name, cls(name)) + self.INSTANCES.add(self) def add( self, priority: t.Optional[int] = None @@ -156,7 +149,7 @@ class Filter(t.Generic[T1, T2]): ) except: sys.stderr.write( - f"Error applying filter '{self.name}': func={callback.func} contexts={callback.contexts}'\n" + f"Error applying filter: func={callback.func} contexts={callback.contexts}'\n" ) raise return value @@ -171,6 +164,14 @@ class Filter(t.Generic[T1, T2]): if not callback.is_in_context(context) ] + @classmethod + def clear_all(cls, context: t.Optional[str] = None) -> None: + """ + Clear any previously defined filter with the given context. + """ + for filtre in cls.INSTANCES: + filtre.clear(context) + # The methods below are specific to filters which take lists as first arguments def add_item( self: "Filter[list[L], T2]", item: L, priority: t.Optional[int] = None @@ -205,8 +206,8 @@ class Filter(t.Generic[T1, T2]): ``add_item`` multiple times on the same filter, then you probably want to use a single call to ``add_items`` instead. - :param name: filter name. :param list[object] items: items that will be appended to the resulting list. + :param int priority: optional priority. Usage:: @@ -261,114 +262,3 @@ class Filter(t.Generic[T1, T2]): Same as :py:func:`Filter.iterate` but apply only callbacks from a given context. """ yield from self.apply_from_context(context, [], *args, **kwargs) - - -class FilterTemplate(t.Generic[T1, T2]): - """ - Filter templates are for filters for which the name needs to be formatted - before the filter can be applied. - - Similar to :py:class:`tutor.core.hooks.ActionTemplate`, filter templates are used to generate - :py:class:`Filter` objects for which the name matches a certain template. - - Templated filters must be formatted with ``(*args)`` before being applied. For example:: - - filter_template = FilterTemplate("namespace:{0}") - named_filter = filter_template("name") - - @named_filter.add() - def my_callback(x: int) -> int: - ... - - named_filter.apply(42) - """ - - def __init__(self, name: str): - self.template = name - - def __repr__(self) -> str: - return f"{self.__class__.__name__}('{self.template}')" - - def __call__(self, *args: t.Any, **kwargs: t.Any) -> Filter[T1, T2]: - return get(self.template.format(*args, **kwargs)) - - -# Syntactic sugar -get = Filter.get - - -def get_template(name: str) -> FilterTemplate[t.Any, t.Any]: - """ - Create a filter with a template name. - """ - return FilterTemplate(name) - - -def add( - name: str, priority: t.Optional[int] = None -) -> t.Callable[[FilterCallbackFunc[T1, T2]], FilterCallbackFunc[T1, T2]]: - """ - Decorator for functions that will be applied to a single named filter. - """ - return Filter.get(name).add(priority=priority) - - -def add_item(name: str, item: T1, priority: t.Optional[int] = None) -> None: - """ - Convenience function to add a single item to a filter that returns a list of items. - """ - get(name).add_item(item, priority=priority) - - -def add_items(name: str, items: list[T1], priority: t.Optional[int] = None) -> None: - """ - Convenience decorator to add multiple item to a filter that returns a list of items. - """ - get(name).add_items(items, priority=priority) - - -def iterate(name: str, *args: t.Any, **kwargs: t.Any) -> t.Iterator[T1]: - """ - Convenient function to iterate over the results of a filter result list. - """ - yield from iterate_from_context(None, name, *args, **kwargs) - - -def iterate_from_context( - context: t.Optional[str], name: str, *args: t.Any, **kwargs: t.Any -) -> t.Iterator[T1]: - yield from Filter.get(name).iterate_from_context(context, *args, **kwargs) - - -def apply(name: str, value: T1, *args: t.Any, **kwargs: t.Any) -> T1: - """ - Apply all declared filters to a single value, passing along the additional arguments. - """ - return apply_from_context(None, name, value, *args, **kwargs) - - -def apply_from_context( - context: t.Optional[str], name: str, value: T1, *args: T2.args, **kwargs: T2.kwargs -) -> T1: - """ - Same as :py:func:`apply` but only run the callbacks that were created in a given context. - """ - filtre: Filter[T1, T2] = Filter.get(name) - return filtre.apply_from_context(context, value, *args, **kwargs) - - -def clear_all(context: t.Optional[str] = None) -> None: - """ - Clear any previously defined filter with the given context. - """ - for name in Filter.INDEX: - clear(name, context=context) - - -def clear(name: str, context: t.Optional[str] = None) -> None: - """ - Clear any previously defined filter with the given name and context. - """ - filtre = Filter.INDEX.get(name) - if filtre: - filtre.clear(context=context) diff --git a/tutor/env.py b/tutor/env.py index 2460ec8..980bb77 100644 --- a/tutor/env.py +++ b/tutor/env.py @@ -526,7 +526,7 @@ def _delete_plugin_templates(plugin: str, root: str, _config: Config) -> None: Delete plugin env files on unload. """ targets = hooks.Filters.ENV_TEMPLATE_TARGETS.iterate_from_context( - hooks.Contexts.APP(plugin).name + hooks.Contexts.app(plugin).name ) for src, dst in targets: path = pathjoin(root, dst.replace("/", os.sep), src.replace("/", os.sep)) diff --git a/tutor/hooks/catalog.py b/tutor/hooks/catalog.py index 8f54007..23cb78d 100644 --- a/tutor/hooks/catalog.py +++ b/tutor/hooks/catalog.py @@ -11,16 +11,7 @@ from typing import Any, Callable, Iterable import click -from tutor.core.hooks import ( - Action, - ActionTemplate, - Context, - ContextTemplate, - Filter, - FilterTemplate, - actions, - filters, -) +from tutor.core.hooks import Action, Context, Filter from tutor.types import Config __all__ = ["Actions", "Filters", "Contexts"] @@ -64,9 +55,7 @@ class Actions: #: :parameter str root: project root. #: :parameter dict config: project configuration. #: :parameter str name: docker-compose project name. - COMPOSE_PROJECT_STARTED: Action[[str, Config, str]] = actions.get( - "compose:project:started" - ) + COMPOSE_PROJECT_STARTED: Action[[str, Config, str]] = Action() #: Called whenever the core project is ready to run. This action is called as soon #: as possible. This is the right time to discover plugins, for instance. In @@ -81,14 +70,14 @@ class Actions: #: developers probably don't have to implement this action themselves. #: #: This action does not have any parameter. - CORE_READY: Action[[]] = actions.get("core:ready") + CORE_READY: Action[[]] = Action() #: Called just before triggering the job tasks of any ``... do `` command. #: #: :parameter str job: job name. #: :parameter args: job positional arguments. #: :parameter kwargs: job named arguments. - DO_JOB: Action[[str, Any]] = actions.get("do:job") + DO_JOB: Action[[str, Any]] = Action() #: Triggered when a single plugin needs to be loaded. Only plugins that have previously been #: discovered can be loaded (see :py:data:`CORE_READY`). @@ -100,14 +89,14 @@ class Actions: #: Most plugin developers will not have to implement this action themselves, unless #: they want to perform a specific action at the moment the plugin is enabled. #: - #: This action does not have any parameter. - PLUGIN_LOADED: ActionTemplate[[]] = actions.get_template("plugins:loaded:{0}") + #: :parameter str plugin: plugin name. + PLUGIN_LOADED: Action[[str]] = Action() #: Triggered after all plugins have been loaded. At this point the list of loaded #: plugins may be obtained from the :py:data:`Filters.PLUGINS_LOADED` filter. #: #: This action does not have any parameter. - PLUGINS_LOADED: Action[[]] = actions.get("plugins:loaded") + PLUGINS_LOADED: Action[[]] = Action() #: Triggered when a single plugin is unloaded. Only plugins that have previously been #: loaded can be unloaded (see :py:data:`PLUGIN_LOADED`). @@ -120,12 +109,12 @@ class Actions: #: :parameter str plugin: plugin name. #: :parameter str root: absolute path to the project root. #: :parameter config: full project configuration - PLUGIN_UNLOADED: Action[str, str, Config] = actions.get("plugins:unloaded") + PLUGIN_UNLOADED: Action[str, str, Config] = Action() #: Called as soon as we have access to the Tutor project root. #: #: :parameter str root: absolute path to the project root. - PROJECT_ROOT_READY: Action[str] = actions.get("project:root:ready") + PROJECT_ROOT_READY: Action[str] = Action() class Filters: @@ -178,7 +167,7 @@ class Filters: #: #: :parameter list commands: commands are instances of ``click.Command``. They will #: all be added as subcommands of the main ``tutor`` command. - CLI_COMMANDS: Filter[list[click.Command], []] = filters.get("cli:commands") + CLI_COMMANDS: Filter[list[click.Command], []] = Filter() #: List of `do ...` commands. #: @@ -188,7 +177,7 @@ class Filters: #: in the "service" container, both in local, dev and k8s mode. CLI_DO_COMMANDS: Filter[ list[Callable[[Any], Iterable[tuple[str, str]]]], [] - ] = filters.get("cli:commands:do") + ] = Filter() #: List of initialization tasks (scripts) to be run in the `init` job. This job #: includes all database migrations, setting up, etc. To run some tasks before or @@ -197,9 +186,7 @@ class Filters: #: :parameter list[tuple[str, str]] tasks: list of ``(service, task)`` tuples. Each #: task is essentially a bash script to be run in the "service" container. Scripts #: may contain Jinja markup, similar to templates. - CLI_DO_INIT_TASKS: Filter[list[tuple[str, str]], []] = filters.get( - "cli:commands:do:init" - ) + CLI_DO_INIT_TASKS: Filter[list[tuple[str, str]], []] = Filter() #: DEPRECATED use :py:data:`CLI_DO_INIT_TASKS` instead. #: @@ -212,9 +199,7 @@ class Filters: #: - ``path`` is a tuple that corresponds to a template relative path. #: Example: ``("myplugin", "hooks", "myservice", "pre-init")`` (see :py:data:`IMAGES_BUILD`). #: The command to execute will be read from that template, after it is rendered. - COMMANDS_INIT: Filter[list[tuple[str, tuple[str, ...]]], []] = filters.get( - "commands:init" - ) + COMMANDS_INIT: Filter[list[tuple[str, tuple[str, ...]]], []] = Filter() #: DEPRECATED use :py:data:`CLI_DO_INIT_TASKS` instead with a lower priority score. #: @@ -223,9 +208,7 @@ class Filters: #: #: :parameter list[tuple[str, tuple[str, ...]]] tasks: list of ``(service, path)`` #: tasks. (see :py:data:`COMMANDS_INIT`). - COMMANDS_PRE_INIT: Filter[list[tuple[str, tuple[str, ...]]], []] = filters.get( - "commands:pre-init" - ) + COMMANDS_PRE_INIT: Filter[list[tuple[str, tuple[str, ...]]], []] = Filter() #: List of folders to bind-mount in docker-compose containers, either in ``tutor local`` or ``tutor dev``. #: @@ -250,7 +233,7 @@ class Filters: #: :parameter str name: basename of the host-mounted folder. In the example above, #: this is "edx-platform". When implementing this filter you should check this name to #: conditionally add mounts. - COMPOSE_MOUNTS: Filter[list[tuple[str, str]], [str]] = filters.get("compose:mounts") + COMPOSE_MOUNTS: Filter[list[tuple[str, str]], [str]] = Filter() #: Declare new default configuration settings that don't necessarily have to be saved in the user #: ``config.yml`` file. Default settings may be overridden with ``tutor config save --set=...``, in which @@ -258,44 +241,34 @@ class Filters: #: #: :parameter list[tuple[str, ...]] items: list of (name, value) new settings. All #: new entries must be prefixed with the plugin name in all-caps. - CONFIG_DEFAULTS: Filter[list[tuple[str, Any]], []] = filters.get("config:defaults") + CONFIG_DEFAULTS: Filter[list[tuple[str, Any]], []] = Filter() #: Modify existing settings, either from Tutor core or from other plugins. Beware not to override any #: important setting, such as passwords! Overridden setting values will be printed to stdout when the plugin #: is disabled, such that users have a chance to back them up. #: #: :parameter list[tuple[str, ...]] items: list of (name, value) settings. - CONFIG_OVERRIDES: Filter[list[tuple[str, Any]], []] = filters.get( - "config:overrides" - ) + CONFIG_OVERRIDES: Filter[list[tuple[str, Any]], []] = Filter() #: Declare unique configuration settings that must be saved in the user ``config.yml`` file. This is where #: you should declare passwords and randomly-generated values that are different from one environment to the next. #: #: :parameter list[tuple[str, ...]] items: list of (name, value) new settings. All #: names must be prefixed with the plugin name in all-caps. - CONFIG_UNIQUE: Filter[list[tuple[str, Any]], []] = filters.get("config:unique") + CONFIG_UNIQUE: Filter[list[tuple[str, Any]], []] = Filter() #: Use this filter to modify the ``docker build`` command. For instance, to replace #: the ``build`` subcommand by ``buildx build``. #: #: :parameter list[str] command: the full build command, including options and #: arguments. Note that these arguments do not include the leading ``docker`` command. - DOCKER_BUILD_COMMAND: Filter[list[str], []] = filters.get("docker:build:command") + DOCKER_BUILD_COMMAND: Filter[list[str], []] = Filter() - #: List of patches that should be inserted in a given location of the templates. The - #: filter name must be formatted with the patch name. - #: This filter is not so convenient and plugin developers will probably - #: prefer :py:data:`ENV_PATCHES`. - #: - #: :parameter list[str] patches: each item is the unrendered patch content. - ENV_PATCH: FilterTemplate[list[str], []] = filters.get_template("env:patches:{0}") - - #: List of patches that should be inserted in a given location of the templates. This is very similar to :py:data:`ENV_PATCH`, except that the patch is added as a ``(name, content)`` tuple. + #: List of patches that should be inserted in a given location of the templates. #: #: :parameter list[tuple[str, str]] patches: pairs of (name, content) tuples. Use this #: filter to modify the Tutor templates. - ENV_PATCHES: Filter[list[tuple[str, str]], []] = filters.get("env:patches") + ENV_PATCHES: Filter[list[tuple[str, str]], []] = Filter() #: List of template path patterns to be ignored when rendering templates to the project root. By default, we ignore: #: @@ -306,13 +279,13 @@ class Filters: #: Ignored patterns are overridden by include patterns; see :py:data:`ENV_PATTERNS_INCLUDE`. #: #: :parameter list[str] patterns: list of regular expression patterns. E.g: ``r"(.*/)?ignored_file_name(/.*)?"``. - ENV_PATTERNS_IGNORE: Filter[list[str], []] = filters.get("env:patterns:ignore") + ENV_PATTERNS_IGNORE: Filter[list[str], []] = Filter() #: List of template path patterns to be included when rendering templates to the project root. #: Patterns from this list will take priority over the patterns from :py:data:`ENV_PATTERNS_IGNORE`. #: #: :parameter list[str] patterns: list of regular expression patterns. See :py:data:`ENV_PATTERNS_IGNORE`. - ENV_PATTERNS_INCLUDE: Filter[list[str], []] = filters.get("env:patterns:include") + ENV_PATTERNS_INCLUDE: Filter[list[str], []] = Filter() #: List of `Jinja2 filters `__ that will be #: available in templates. Jinja2 filters are basically functions that can be used @@ -343,16 +316,14 @@ class Filters: #: #: :parameter filters: list of (name, function) tuples. The function signature #: should correspond to its usage in templates. - ENV_TEMPLATE_FILTERS: Filter[ - list[tuple[str, Callable[..., Any]]], [] - ] = filters.get("env:templates:filters") + ENV_TEMPLATE_FILTERS: Filter[list[tuple[str, Callable[..., Any]]], []] = Filter() #: List of all template root folders. #: #: :parameter list[str] templates_root: absolute paths to folders which contain templates. #: The templates in these folders will then be accessible by the environment #: renderer using paths that are relative to their template root. - ENV_TEMPLATE_ROOTS: Filter[list[str], []] = filters.get("env:templates:roots") + ENV_TEMPLATE_ROOTS: Filter[list[str], []] = Filter() #: List of template source/destination targets. #: @@ -361,9 +332,7 @@ class Filters: #: is a path relative to the environment root. For instance: adding ``("c/d", #: "a/b")`` to the filter will cause all files from "c/d" to be rendered to the ``a/b/c/d`` #: subfolder. - ENV_TEMPLATE_TARGETS: Filter[list[tuple[str, str]], []] = filters.get( - "env:templates:targets" - ) + ENV_TEMPLATE_TARGETS: Filter[list[tuple[str, str]], []] = Filter() #: List of extra variables to be included in all templates. #: @@ -378,9 +347,7 @@ class Filters: #: - `patch`: a function to incorporate extra content into a template. #: #: :parameter filters: list of (name, value) tuples. - ENV_TEMPLATE_VARIABLES: Filter[list[tuple[str, Any]], []] = filters.get( - "env:templates:variables" - ) + ENV_TEMPLATE_VARIABLES: Filter[list[tuple[str, Any]], []] = Filter() #: List of images to be built when we run ``tutor images build ...``. #: @@ -397,7 +364,7 @@ class Filters: #: :parameter Config config: user configuration. IMAGES_BUILD: Filter[ list[tuple[str, tuple[str, ...], str, tuple[str, ...]]], [Config] - ] = filters.get("images:build") + ] = Filter() #: List of host directories to be automatically bind-mounted in Docker images at #: build time. For instance, this is useful to build Docker images using a custom @@ -415,9 +382,7 @@ class Filters: #: :py:data:`COMPOSE_MOUNTS`, this is not just the basename, but the full path. When #: implementing this filter you should check this path (for instance: with #: ``os.path.basename(path)``) to conditionally add mounts. - IMAGES_BUILD_MOUNTS: Filter[list[tuple[str, str]], [str]] = filters.get( - "images:build:mounts" - ) + IMAGES_BUILD_MOUNTS: Filter[list[tuple[str, str]], [str]] = Filter() #: List of images to be pulled when we run ``tutor images pull ...``. #: @@ -426,11 +391,11 @@ class Filters: #: - ``name`` is the name of the image, as in ``tutor images pull myimage``. #: - ``tag`` is the Docker tag that will be applied to the image. (see :py:data:`IMAGES_BUILD`). #: :parameter Config config: user configuration. - IMAGES_PULL: Filter[list[tuple[str, str]], [Config]] = filters.get("images:pull") + IMAGES_PULL: Filter[list[tuple[str, str]], [Config]] = Filter() #: List of images to be pushed when we run ``tutor images push ...``. #: Parameters are the same as for :py:data:`IMAGES_PULL`. - IMAGES_PUSH: Filter[list[tuple[str, str]], [Config]] = filters.get("images:push") + IMAGES_PUSH: Filter[list[tuple[str, str]], [Config]] = Filter() #: List of plugin indexes that are loaded when we run `tutor plugins update`. By #: default, the plugin indexes are stored in the user configuration. This filter makes @@ -438,13 +403,13 @@ class Filters: #: #: :parameter list[str] indexes: list of index URLs. Remember that entries further #: in the list have priority. - PLUGIN_INDEXES: Filter[list[str], []] = filters.get("plugins:indexes:entries") + PLUGIN_INDEXES: Filter[list[str], []] = Filter() #: Filter to modify the url of a plugin index url. This is convenient to alias #: plugin indexes with a simple name, such as "main" or "contrib". #: #: :parameter str url: value passed to the `index add/remove` commands. - PLUGIN_INDEX_URL: Filter[str, []] = filters.get("plugins:indexes:url") + PLUGIN_INDEX_URL: Filter[str, []] = Filter() #: When installing an entry from a plugin index, the plugin data from the index will #: go through this filter before it is passed along to `pip install`. Thus, this is a @@ -453,17 +418,13 @@ class Filters: #: #: :parameter dict[str, str] plugin: the dict entry from the plugin index. It #: includes an additional "index" key which contains the plugin index URL. - PLUGIN_INDEX_ENTRY_TO_INSTALL: Filter[dict[str, str], []] = filters.get( - "plugins:indexes:entries:install" - ) + PLUGIN_INDEX_ENTRY_TO_INSTALL: Filter[dict[str, str], []] = Filter() #: Information about each installed plugin, including its version. #: Keep this information to a single line for easier parsing by 3rd-party scripts. #: #: :param list[tuple[str, str]] versions: each pair is a ``(plugin, info)`` tuple. - PLUGINS_INFO: Filter[list[tuple[str, str]], []] = filters.get( - "plugins:installed:versions" - ) + PLUGINS_INFO: Filter[list[tuple[str, str]], []] = Filter() #: List of installed plugins. In order to be added to this list, a plugin must first #: be discovered (see :py:data:`Actions.CORE_READY`). @@ -471,13 +432,13 @@ class Filters: #: :param list[str] plugins: plugin developers probably don't have to implement this #: filter themselves, but they can apply it to check for the presence of other #: plugins. - PLUGINS_INSTALLED: Filter[list[str], []] = filters.get("plugins:installed") + PLUGINS_INSTALLED: Filter[list[str], []] = Filter() #: List of loaded plugins. #: #: :param list[str] plugins: plugin developers probably don't have to modify this #: filter themselves, but they can apply it to check whether other plugins are enabled. - PLUGINS_LOADED: Filter[list[str], []] = filters.get("plugins:loaded") + PLUGINS_LOADED: Filter[list[str], []] = Filter() class Contexts: @@ -497,10 +458,16 @@ class Contexts: hooks.Filters.MY_FILTER.apply_from_context(hooks.Contexts.SOME_CONTEXT.name) """ - #: We enter this context whenever we create hooks for a specific application or : - #: plugin. For instance, plugin "myplugin" will be enabled within the "app:myplugin" - #: context. - APP = ContextTemplate("app:{0}") + #: Dictionary of name/contexts. Each value is a context that we enter whenever we + #: create hooks for a specific application or : : plugin. For instance, plugin + #: "myplugin" will be enabled within the "app:myplugin" : context. + APP: dict[str, Context] = {} + + @classmethod + def app(cls, name: str) -> Context: + if name not in cls.APP: + cls.APP[name] = Context(f"app:{name}") + return cls.APP[name] #: Plugins will be installed and enabled within this context. PLUGINS = Context("plugins") diff --git a/tutor/plugins/__init__.py b/tutor/plugins/__init__.py index b8e5564..c5e88d2 100644 --- a/tutor/plugins/__init__.py +++ b/tutor/plugins/__init__.py @@ -12,19 +12,24 @@ from tutor.types import Config, get_typed # Import modules to trigger hook creation from . import v0, v1 +# Cache of plugin patches, for efficiency +ENV_PATCHES_DICT: dict[str, list[str]] = {} + @hooks.Actions.PLUGINS_LOADED.add() def _convert_plugin_patches() -> None: """ Some patches are added as (name, content) tuples with the ENV_PATCHES - filter. We convert these patches to add them to ENV_PATCH. This makes it + filter. We convert these patches to add them to ENV_PATCHES_DICT. This makes it easier for end-user to declare patches, and it's more performant. This action is run after plugins have been loaded. """ + ENV_PATCHES_DICT.clear() patches: t.Iterable[tuple[str, str]] = hooks.Filters.ENV_PATCHES.iterate() for name, content in patches: - hooks.Filters.ENV_PATCH(name).add_item(content) + ENV_PATCHES_DICT.setdefault(name, []) + ENV_PATCHES_DICT[name].append(content) def is_installed(name: str) -> bool: @@ -89,8 +94,8 @@ def load(name: str) -> None: if not is_installed(name): raise exceptions.TutorError(f"plugin '{name}' is not installed.") with hooks.Contexts.PLUGINS.enter(): - with hooks.Contexts.APP(name).enter(): - hooks.Actions.PLUGIN_LOADED(name).do() + with hooks.Contexts.app(name).enter(): + hooks.Actions.PLUGIN_LOADED.do(name) hooks.Filters.PLUGINS_LOADED.add_item(name) @@ -109,14 +114,14 @@ def iter_patches(name: str) -> t.Iterator[str]: """ Yields: patch (str) """ - yield from hooks.Filters.ENV_PATCH(name).iterate() + yield from ENV_PATCHES_DICT.get(name, []) def unload(plugin: str) -> None: """ Remove all filters and actions associated to a given plugin. """ - hooks.clear_all(context=hooks.Contexts.APP(plugin).name) + hooks.clear_all(context=hooks.Contexts.app(plugin).name) @hooks.Actions.PLUGIN_UNLOADED.add(priority=50) diff --git a/tutor/plugins/v0.py b/tutor/plugins/v0.py index 74c40c0..16e6770 100644 --- a/tutor/plugins/v0.py +++ b/tutor/plugins/v0.py @@ -60,7 +60,10 @@ class BasePlugin: hooks.Filters.PLUGINS_INFO.add_item((self.name, self._version() or "")) # Create actions and filters on load - hooks.Actions.PLUGIN_LOADED(self.name).add()(self.__load) + @hooks.Actions.PLUGIN_LOADED.add() + def _load_plugin(name: str) -> None: + if name == self.name: + self.__load() def __load(self) -> None: """ diff --git a/tutor/plugins/v1.py b/tutor/plugins/v1.py index ec0f9ac..cf24bf0 100644 --- a/tutor/plugins/v1.py +++ b/tutor/plugins/v1.py @@ -43,16 +43,17 @@ def discover_module(path: str) -> None: hooks.Filters.PLUGINS_INFO.add_item((name, path)) # Import module on enable - load_plugin_action = hooks.Actions.PLUGIN_LOADED(name) - - @load_plugin_action.add() - def load() -> None: - # https://docs.python.org/3/library/importlib.html#importing-a-source-file-directly - spec = importlib.util.spec_from_file_location("tutor.plugin.v1.{name}", path) - if spec is None or spec.loader is None: - raise ValueError("Plugin could not be found: {path}") - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) + @hooks.Actions.PLUGIN_LOADED.add() + def load(plugin_name: str) -> None: + if name == plugin_name: + # https://docs.python.org/3/library/importlib.html#importing-a-source-file-directly + spec = importlib.util.spec_from_file_location( + "tutor.plugin.v1.{name}", path + ) + if spec is None or spec.loader is None: + raise ValueError("Plugin could not be found: {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) def discover_package(entrypoint: pkg_resources.EntryPoint) -> None: @@ -70,8 +71,7 @@ def discover_package(entrypoint: pkg_resources.EntryPoint) -> None: hooks.Filters.PLUGINS_INFO.add_item((name, entrypoint.dist.version)) # Import module on enable - load_plugin_action = hooks.Actions.PLUGIN_LOADED(name) - - @load_plugin_action.add() - def load() -> None: - entrypoint.load() + @hooks.Actions.PLUGIN_LOADED.add() + def load(plugin_name: str) -> None: + if name == plugin_name: + entrypoint.load() diff --git a/tutor/utils.py b/tutor/utils.py index 06c0e3d..59adee4 100644 --- a/tutor/utils.py +++ b/tutor/utils.py @@ -258,6 +258,7 @@ Tutor may not work if Docker is configured with < 4 GB RAM. Please follow instru https://docs.tutor.overhang.io/install.html""" ) + def check_macos_docker_memory() -> None: """ Try to check that the RAM allocated to the Docker VM on macOS is at least 4 GB.