From 71ed7a8618f4794f06e8692e18e47b55aab48a6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Fri, 6 Jan 2023 19:02:17 +0100 Subject: [PATCH] feat: refactor hooks API for simplification The hooks API had several issues which are summarized in this comment: https://github.com/openedx/wg-developer-experience/issues/125#issuecomment-1313553526 1. "consts" was a bad name 2. "hooks.filters" and "hooks.Filters" could easily be confused 3. docs made it difficult to understand that plugin developers should use the catalog To address these issues, we: 1. move "consts.py" to "catalog.py" 2. Remove "hooks.actions", "hooks.filters", "hooks.contexts" from the API. 3. re-organize the docs and give better usage examples in the catalog. This change is a partial fix for https://github.com/openedx/wg-developer-experience/issues/125 --- .../20230106_190620_regis_hooks_api.md | 7 + docs/conf.py | 49 ++- docs/plugins/intro.rst | 2 + docs/reference/api/hooks/actions.rst | 17 +- docs/reference/api/hooks/catalog.rst | 18 ++ docs/reference/api/hooks/consts.rst | 23 -- docs/reference/api/hooks/contexts.rst | 5 +- docs/reference/api/hooks/filters.rst | 22 +- docs/reference/api/hooks/index.rst | 11 +- docs/reference/index.rst | 3 +- docs/reference/patches.rst | 6 +- docs/tutorials/plugin.rst | 6 +- requirements/dev.txt | 6 +- tests/commands/test_compose.py | 1 + tests/{hooks => core}/__init__.py | 0 tests/core/hooks/__init__.py | 0 tests/{ => core}/hooks/test_actions.py | 20 +- tests/core/hooks/test_filters.py | 61 ++++ tests/helpers.py | 6 +- tests/hooks/test_filters.py | 60 ---- tests/test_plugins_v0.py | 2 +- tests/test_utils.py | 1 - tutor/commands/cli.py | 1 + tutor/commands/compose.py | 4 +- tutor/commands/config.py | 1 + tutor/commands/images.py | 4 +- tutor/commands/jobs.py | 1 + tutor/commands/k8s.py | 2 +- tutor/commands/local.py | 3 +- tutor/commands/plugins.py | 2 +- tutor/config.py | 1 + tutor/core/hooks/__init__.py | 15 + tutor/{ => core}/hooks/actions.py | 176 ++++++----- tutor/{ => core}/hooks/contexts.py | 26 +- tutor/{ => core}/hooks/filters.py | 287 ++++++++++-------- tutor/{ => core}/hooks/priorities.py | 1 + tutor/env.py | 1 + tutor/hooks/__init__.py | 13 +- tutor/hooks/{consts.py => catalog.py} | 151 +++++---- tutor/plugins/__init__.py | 1 + tutor/serialize.py | 1 + tutor/types.py | 1 + 42 files changed, 575 insertions(+), 443 deletions(-) create mode 100644 changelog.d/20230106_190620_regis_hooks_api.md create mode 100644 docs/reference/api/hooks/catalog.rst delete mode 100644 docs/reference/api/hooks/consts.rst rename tests/{hooks => core}/__init__.py (100%) create mode 100644 tests/core/hooks/__init__.py rename tests/{ => core}/hooks/test_actions.py (73%) create mode 100644 tests/core/hooks/test_filters.py delete mode 100644 tests/hooks/test_filters.py create mode 100644 tutor/core/hooks/__init__.py rename tutor/{ => core}/hooks/actions.py (56%) rename tutor/{ => core}/hooks/contexts.py (77%) rename tutor/{ => core}/hooks/filters.py (55%) rename tutor/{ => core}/hooks/priorities.py (99%) rename tutor/hooks/{consts.py => catalog.py} (83%) diff --git a/changelog.d/20230106_190620_regis_hooks_api.md b/changelog.d/20230106_190620_regis_hooks_api.md new file mode 100644 index 0000000..b71d379 --- /dev/null +++ b/changelog.d/20230106_190620_regis_hooks_api.md @@ -0,0 +1,7 @@ +- 💥[Feature] Simplify the hooks API. Plugin developers who were previously using `hooks.actions`, `hooks.filters` or `hooks.contexts` should now import these modules explicitely. (by @regisb) +- 💥[Feature] Simplify the hooks API. The modules `tutor.hooks.actions`, `tutor.hooks.filters`, and `tutor.hooks.contexts` are no longer part of the API. This change should affect mosst developers, who only use the `Actions` and `Filters` classes (notice the plural) from `tutor.hooks`. (by @regisb) + - Instead of `tutor.hooks.actions.get("some:action")`, use `tutor.hooks.Actions.SOME_ACTION`. + - Instead of `tutor.hooks.filters.get("some:filter")`, use `tutor.hooks.Filters.SOME_FILTER`. + - Instead of `tutor.hooks.actions.add("some:action")`, use `tutor.hooks.Actions.SOME_ACTION.add()`. The same applies to the `do` method. + - Instead of `tutor.hooks.filters.add("some:filter")`, use `tutor.hooks.Filters.SOME_FILTER.add()`. The same applies to the `add_item`, `add_items`, `apply`, and `iterate` methods. + - Instead of `tutor.hooks.contexts.enter`, use `tutor.core.hooks.contexts.enter`. diff --git a/docs/conf.py b/docs/conf.py index 1f17b4a..d6539b8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import io import os import sys @@ -9,8 +11,8 @@ import docutils.parsers.rst # -- Project information ----------------------------------------------------- project = "Tutor" -copyright = "" -author = "Overhang.io" +copyright = "" # pylint: disable=redefined-builtin +author = "Overhang.IO" # The short X.Y version version = "" @@ -32,19 +34,36 @@ extensions.append("sphinx.ext.autodoc") autodoc_typehints = "description" # For the life of me I can't get the docs to compile in nitpicky mode without these # ignore statements. You are most welcome to try and remove them. +# To make matters worse, some ignores are only required for some versions of Python, +# from 3.7 to 3.10... nitpick_ignore = [ - ("py:class", "Config"), + # Sphinx does not handle ParamSpec arguments + ("py:class", "T.args"), + ("py:class", "T.kwargs"), + ("py:class", "T2.args"), + ("py:class", "T2.kwargs"), + # Sphinx doesn't know about the following classes ("py:class", "click.Command"), - ("py:class", "tutor.hooks.filters.P"), - ("py:class", "tutor.hooks.filters.T"), - ("py:class", "tutor.hooks.actions.P"), - ("py:class", "P"), - ("py:class", "P.args"), - ("py:class", "P.kwargs"), - ("py:class", "T"), ("py:class", "t.Any"), + ("py:class", "t.Callable"), + ("py:class", "t.Iterator"), ("py:class", "t.Optional"), + # python 3.7 + ("py:class", "Concatenate"), + # python 3.10 + ("py:class", "NoneType"), + ("py:class", "click.core.Command"), ] +# Resolve type aliases here +# https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html#confval-autodoc_type_aliases +autodoc_type_aliases: dict[str, str] = { + "T1": "tutor.core.hooks.filters.T1", + "L": "tutor.core.hooks.filters.L", + # python 3.10 + "T": "tutor.core.hooks.actions.T", + "T2": "tutor.core.hooks.filters.T2", +} + # -- Sphinx-Click configuration # https://sphinx-click.readthedocs.io/ @@ -87,13 +106,11 @@ about: Dict[str, str] = {} with io.open( os.path.join(here, "..", "tutor", "__about__.py"), "rt", encoding="utf-8" ) as f: + # pylint: disable=exec-used exec(f.read(), about) -rst_prolog = """ -.. |tutor_version| replace:: {} -""".format( - about["__version__"], -) - +rst_prolog = f""" +.. |tutor_version| replace:: {about["__version__"]} +""" # Custom directives def youtube( diff --git a/docs/plugins/intro.rst b/docs/plugins/intro.rst index 1001df5..78c282f 100644 --- a/docs/plugins/intro.rst +++ b/docs/plugins/intro.rst @@ -15,6 +15,8 @@ Tutor comes with a plugin system that allows anyone to customise the deployment For simple changes, it may be extremely easy to create a Tutor plugin: even non-technical users may get started with our :ref:`plugin_development_tutorial` tutorial. We also provide a list of :ref:`simple example plugins `. +To learn about the different ways in which plugins can extend Tutor, check out the :ref:`hooks catalog `. + Plugin commands cheatsheet ========================== diff --git a/docs/reference/api/hooks/actions.rst b/docs/reference/api/hooks/actions.rst index 3154ff3..b9f94bb 100644 --- a/docs/reference/api/hooks/actions.rst +++ b/docs/reference/api/hooks/actions.rst @@ -6,13 +6,12 @@ Actions Actions are one of the two types of hooks (the other being :ref:`filters`) that can be used to extend Tutor. Each action represents an event that can occur during the application life cycle. Each action has a name, and callback functions can be attached to it. When an action is triggered, these callback functions are called in sequence. Each callback function can trigger side effects, independently from one another. -.. autofunction:: tutor.hooks.actions::get -.. autofunction:: tutor.hooks.actions::get_template -.. autofunction:: tutor.hooks.actions::add -.. autofunction:: tutor.hooks.actions::do -.. autofunction:: tutor.hooks.actions::do_from_context -.. autofunction:: tutor.hooks.actions::clear -.. autofunction:: tutor.hooks.actions::clear_all +.. autoclass:: tutor.core.hooks.Action + :members: -.. autoclass:: tutor.hooks.actions.Action -.. autoclass:: tutor.hooks.actions.ActionTemplate +.. autoclass:: tutor.core.hooks.ActionTemplate + :members: + +.. The following are only to ensure that the docs build without warnings +.. class:: tutor.core.hooks.actions.T +.. class:: tutor.types.Config diff --git a/docs/reference/api/hooks/catalog.rst b/docs/reference/api/hooks/catalog.rst new file mode 100644 index 0000000..cddd3c6 --- /dev/null +++ b/docs/reference/api/hooks/catalog.rst @@ -0,0 +1,18 @@ +.. _hooks_catalog: + +============= +Hooks catalog +============= + +Tutor can be extended by making use of "hooks". Hooks are either "actions" or "filters". Here, we list all instances of actions and filters that are used across Tutor. Plugin developers can leverage these hooks to modify the behaviour of Tutor. + +The underlying Python hook classes and API are documented :ref:`here `. + +.. autoclass:: tutor.hooks.Actions + :members: + +.. autoclass:: tutor.hooks.Filters + :members: + +.. autoclass:: tutor.hooks.Contexts + :members: diff --git a/docs/reference/api/hooks/consts.rst b/docs/reference/api/hooks/consts.rst deleted file mode 100644 index 7742567..0000000 --- a/docs/reference/api/hooks/consts.rst +++ /dev/null @@ -1,23 +0,0 @@ -========= -Constants -========= - -Here we lists named :ref:`actions`, :ref:`filters` and :ref:`contexts` that are used across Tutor. These are simply hook variables that we can refer to across the Tutor codebase without having to hard-code string names. The API is slightly different and less verbose than "native" hooks. - -Actions -======= - -.. autoclass:: tutor.hooks.Actions - :members: - -Filters -======= - -.. autoclass:: tutor.hooks.Filters - :members: - -Contexts -======== - -.. autoclass:: tutor.hooks.Contexts - :members: diff --git a/docs/reference/api/hooks/contexts.rst b/docs/reference/api/hooks/contexts.rst index 8f4d91d..808de98 100644 --- a/docs/reference/api/hooks/contexts.rst +++ b/docs/reference/api/hooks/contexts.rst @@ -6,6 +6,5 @@ Contexts Contexts are a feature of the hook-based extension system in Tutor, which allows us to keep track of which components of the code created which callbacks. Contexts are very much an internal concept that most plugin developers should not have to worry about. -.. autofunction:: tutor.hooks.contexts::enter - -.. autoclass:: tutor.hooks.contexts.Context +.. autoclass:: tutor.core.hooks.Context +.. autofunction:: tutor.core.hooks.contexts::enter diff --git a/docs/reference/api/hooks/filters.rst b/docs/reference/api/hooks/filters.rst index ce9d9a2..f14a582 100644 --- a/docs/reference/api/hooks/filters.rst +++ b/docs/reference/api/hooks/filters.rst @@ -6,17 +6,13 @@ Filters Filters are one of the two types of hooks (the other being :ref:`actions`) that can be used to extend Tutor. Filters allow one to modify the application behavior by transforming data. Each filter has a name, and callback functions can be attached to it. When a filter is applied, these callback functions are called in sequence; the result of each callback function is passed as the first argument to the next callback function. The result of the final callback function is returned to the application as the filter's output. -.. autofunction:: tutor.hooks.filters::get -.. autofunction:: tutor.hooks.filters::get_template -.. autofunction:: tutor.hooks.filters::add -.. autofunction:: tutor.hooks.filters::add_item -.. autofunction:: tutor.hooks.filters::add_items -.. autofunction:: tutor.hooks.filters::apply -.. autofunction:: tutor.hooks.filters::apply_from_context -.. autofunction:: tutor.hooks.filters::iterate -.. autofunction:: tutor.hooks.filters::iterate_from_context -.. autofunction:: tutor.hooks.filters::clear -.. autofunction:: tutor.hooks.filters::clear_all +.. autoclass:: tutor.core.hooks.Filter + :members: -.. autoclass:: tutor.hooks.filters.Filter -.. autoclass:: tutor.hooks.filters.FilterTemplate +.. autoclass:: tutor.core.hooks.FilterTemplate + :members: + +.. The following are only to ensure that the docs build without warnings +.. class:: tutor.core.hooks.filters.T1 +.. class:: tutor.core.hooks.filters.T2 +.. class:: tutor.core.hooks.filters.L diff --git a/docs/reference/api/hooks/index.rst b/docs/reference/api/hooks/index.rst index ed62703..d668685 100644 --- a/docs/reference/api/hooks/index.rst +++ b/docs/reference/api/hooks/index.rst @@ -1,6 +1,10 @@ -========= -Hooks API -========= +.. _hooks_api: + +========== +Hook types +========== + +This is the Python documentation of the two types of hooks (actions and filters) as well as the contexts system which is used to instrument them. Understanding how Tutor hooks work is useful to create plugins that modify the behaviour of Tutor. However, plugin developers should almost certainly not import these hook types directly. Instead, use the reference :ref:`hooks catalog `. .. toctree:: :maxdepth: 1 @@ -8,4 +12,3 @@ Hooks API actions filters contexts - consts diff --git a/docs/reference/index.rst b/docs/reference/index.rst index ad2a648..5d49671 100644 --- a/docs/reference/index.rst +++ b/docs/reference/index.rst @@ -5,5 +5,6 @@ Reference :maxdepth: 2 api/hooks/index - cli/index + api/hooks/catalog patches + cli/index diff --git a/docs/reference/patches.rst b/docs/reference/patches.rst index 49bcf4f..efca194 100644 --- a/docs/reference/patches.rst +++ b/docs/reference/patches.rst @@ -1,8 +1,8 @@ .. _patches: -================ -Template patches -================ +====================== +Template patch catalog +====================== This is the list of all patches used across Tutor (outside of any plugin). Alternatively, you can search for patches in Tutor templates by grepping the source code:: diff --git a/docs/tutorials/plugin.rst b/docs/tutorials/plugin.rst index f522f61..3970370 100644 --- a/docs/tutorials/plugin.rst +++ b/docs/tutorials/plugin.rst @@ -10,6 +10,8 @@ You may be thinking that creating a plugin might be overkill for your use case. A plugin can be created either as a simple, single Python module (a ``*.py`` file) or as a full-blown Python package. Single Python modules are easier to write, while Python packages can be distributed more easily with ``pip install ...``. We'll start by writing our plugin as a single Python module. +Plugins work by making extensive use of the Tutor hooks API. The list of available hooks is available from the :ref:`hooks catalog `. Developers who want to understand how hooks work should check the :ref:`hooks API `. + Writing a plugin as a single Python module ========================================== @@ -45,7 +47,7 @@ Modifying existing files with patches We'll start by modifying some of our Open edX settings files. It's a frequent requirement to modify the ``FEATURES`` setting from the LMS or the CMS in edx-platform. In the legacy native installation, this was done by modifying the ``lms.env.yml`` and ``cms.env.yml`` files. Here we'll modify the Python setting files that define the edx-platform configuration. To achieve that we'll make use of two concepts from the Tutor API: :ref:`patches` and :ref:`filters`. -If you have not already read :ref:`how_does_tutor_work` now would be a good time :-) Tutor uses templates to generate various files, such as settings, Dockerfiles, etc. These templates include ``{{ patch("patch-name") }}`` statements that allow plugins to insert arbitrary content in there. These patches are located at strategic locations. See :ref:`patches` for more information. +If you have not already read :ref:`how_does_tutor_work` now would be a good time ☺️ Tutor uses templates to generate various files, such as settings, Dockerfiles, etc. These templates include ``{{ patch("patch-name") }}`` statements that allow plugins to insert arbitrary content in there. These patches are located at strategic locations. See :ref:`patches` for more information. Let's say that we would like to limit access to our brand new Open edX platform. It is not ready for prime-time yet, so we want to prevent users from registering new accounts. There is a feature flag for that in the LMS: `FEATURES['ALLOW_PUBLIC_ACCOUNT_CREATION'] `__. By default this flag is set to a true value, enabling anyone to create an account. In the following we'll set it to false. @@ -73,7 +75,7 @@ This imports the ``hooks`` module from Tutor, which grants us access to ``hooks. ) -This means "add ```` to the ``{{ patch("") }}`` statement, thanks to the ENV_PATCHES filter". In our case, we want to modify the LMS settings, both in production and development. The right patch for that is :patch:`openedx-lms-common-settings`. We add one item, which is a single Python-formatted line of code:: +This means "add ```` to the ``{{ patch("") }}`` statement, thanks to the :py:data:`tutor.hooks.Filters.ENV_PATCHES` filter". In our case, we want to modify the LMS settings, both in production and development. The right patch for that is :patch:`openedx-lms-common-settings`. We add one item, which is a single Python-formatted line of code:: "FEATURES['ALLOW_PUBLIC_ACCOUNT_CREATION'] = False" diff --git a/requirements/dev.txt b/requirements/dev.txt index 0cafd87..caa9d45 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with Python 3.8 -# by the following command: +# This file is autogenerated by pip-compile with python 3.8 +# To update, run: # # pip-compile requirements/dev.in # @@ -62,7 +62,7 @@ idna==3.4 # via # -r requirements/base.txt # requests -importlib-metadata==5.1.0 +importlib-metadata==6.0.0 # via # keyring # twine diff --git a/tests/commands/test_compose.py b/tests/commands/test_compose.py index 9eb505e..77ca602 100644 --- a/tests/commands/test_compose.py +++ b/tests/commands/test_compose.py @@ -1,4 +1,5 @@ from __future__ import annotations + import typing as t import unittest from io import StringIO diff --git a/tests/hooks/__init__.py b/tests/core/__init__.py similarity index 100% rename from tests/hooks/__init__.py rename to tests/core/__init__.py diff --git a/tests/core/hooks/__init__.py b/tests/core/hooks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/hooks/test_actions.py b/tests/core/hooks/test_actions.py similarity index 73% rename from tests/hooks/test_actions.py rename to tests/core/hooks/test_actions.py index f0cbff4..60654a6 100644 --- a/tests/hooks/test_actions.py +++ b/tests/core/hooks/test_actions.py @@ -1,7 +1,7 @@ import typing as t import unittest -from tutor import hooks +from tutor.core.hooks import actions, contexts class PluginActionsTests(unittest.TestCase): @@ -10,14 +10,14 @@ class PluginActionsTests(unittest.TestCase): def tearDown(self) -> None: super().tearDown() - hooks.actions.clear_all(context="tests") + actions.clear_all(context="tests") def run(self, result: t.Any = None) -> t.Any: - with hooks.contexts.enter("tests"): + with contexts.enter("tests"): return super().run(result=result) def test_do(self) -> None: - action: hooks.actions.Action[int] = hooks.actions.get("test-action") + action: actions.Action[int] = actions.get("test-action") @action.add() def _test_action_1(increment: int) -> None: @@ -31,29 +31,29 @@ class PluginActionsTests(unittest.TestCase): self.assertEqual(3, self.side_effect_int) def test_priority(self) -> None: - @hooks.actions.add("test-action", priority=2) + @actions.add("test-action", priority=2) def _test_action_1() -> None: self.side_effect_int += 4 - @hooks.actions.add("test-action", priority=1) + @actions.add("test-action", priority=1) def _test_action_2() -> None: self.side_effect_int = self.side_effect_int // 2 # Action 2 must be performed before action 1 self.side_effect_int = 4 - hooks.actions.do("test-action") + actions.do("test-action") self.assertEqual(6, self.side_effect_int) def test_equal_priority(self) -> None: - @hooks.actions.add("test-action", priority=2) + @actions.add("test-action", priority=2) def _test_action_1() -> None: self.side_effect_int += 4 - @hooks.actions.add("test-action", priority=2) + @actions.add("test-action", priority=2) def _test_action_2() -> None: self.side_effect_int = self.side_effect_int // 2 # Action 2 must be performed after action 1 self.side_effect_int = 4 - hooks.actions.do("test-action") + actions.do("test-action") self.assertEqual(4, self.side_effect_int) diff --git a/tests/core/hooks/test_filters.py b/tests/core/hooks/test_filters.py new file mode 100644 index 0000000..4d7b821 --- /dev/null +++ b/tests/core/hooks/test_filters.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +import typing as t +import unittest + +from tutor.core.hooks import contexts, filters + + +class PluginFiltersTests(unittest.TestCase): + def tearDown(self) -> None: + super().tearDown() + filters.clear_all(context="tests") + + def run(self, result: t.Any = None) -> t.Any: + with contexts.enter("tests"): + return super().run(result=result) + + def test_add(self) -> None: + @filters.add("tests:count-sheeps") + def filter1(value: int) -> int: + return value + 1 + + value = filters.apply("tests:count-sheeps", 0) + self.assertEqual(1, value) + + def test_add_items(self) -> None: + @filters.add("tests:add-sheeps") + def filter1(sheeps: list[int]) -> list[int]: + return sheeps + [0] + + filters.add_item("tests:add-sheeps", 1) + filters.add_item("tests:add-sheeps", 2) + filters.add_items("tests:add-sheeps", [3, 4]) + + sheeps: list[int] = filters.apply("tests:add-sheeps", []) + self.assertEqual([0, 1, 2, 3, 4], sheeps) + + def test_filter_callbacks(self) -> None: + callback = filters.FilterCallback(lambda _: 1) + self.assertTrue(callback.is_in_context(None)) + self.assertFalse(callback.is_in_context("customcontext")) + self.assertEqual(1, callback.apply(0)) + + def test_filter_context(self) -> None: + with contexts.enter("testcontext"): + filters.add_item("test:sheeps", 1) + filters.add_item("test:sheeps", 2) + + self.assertEqual([1, 2], filters.apply("test:sheeps", [])) + self.assertEqual( + [1], filters.apply_from_context("testcontext", "test:sheeps", []) + ) + + def test_clear_context(self) -> None: + with contexts.enter("testcontext"): + filters.add_item("test:sheeps", 1) + filters.add_item("test:sheeps", 2) + + self.assertEqual([1, 2], filters.apply("test:sheeps", [])) + filters.clear("test:sheeps", context="testcontext") + self.assertEqual([2], filters.apply("test:sheeps", [])) diff --git a/tests/helpers.py b/tests/helpers.py index fef3203..3ec9029 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -6,6 +6,7 @@ import unittest.result from tutor import hooks from tutor.commands.context import BaseTaskContext +from tutor.core.hooks.contexts import enter as enter_context from tutor.tasks import BaseTaskRunner from tutor.types import Config @@ -64,8 +65,7 @@ class PluginsTestCase(unittest.TestCase): hooks.Contexts.PLUGINS_V0_YAML.name, "unittests", ]: - hooks.filters.clear_all(context=context) - hooks.actions.clear_all(context=context) + hooks.clear_all(context=context) def run( self, result: t.Optional[unittest.result.TestResult] = None @@ -74,5 +74,5 @@ class PluginsTestCase(unittest.TestCase): Run all actions and filters with a test context, such that they can be cleared from one run to the next. """ - with hooks.contexts.enter("unittests"): + with enter_context("unittests"): return super().run(result=result) diff --git a/tests/hooks/test_filters.py b/tests/hooks/test_filters.py deleted file mode 100644 index 4c23310..0000000 --- a/tests/hooks/test_filters.py +++ /dev/null @@ -1,60 +0,0 @@ -from __future__ import annotations -import typing as t -import unittest - -from tutor import hooks - - -class PluginFiltersTests(unittest.TestCase): - def tearDown(self) -> None: - super().tearDown() - hooks.filters.clear_all(context="tests") - - def run(self, result: t.Any = None) -> t.Any: - with hooks.contexts.enter("tests"): - return super().run(result=result) - - def test_add(self) -> None: - @hooks.filters.add("tests:count-sheeps") - def filter1(value: int) -> int: - return value + 1 - - value = hooks.filters.apply("tests:count-sheeps", 0) - self.assertEqual(1, value) - - def test_add_items(self) -> None: - @hooks.filters.add("tests:add-sheeps") - def filter1(sheeps: list[int]) -> list[int]: - return sheeps + [0] - - hooks.filters.add_item("tests:add-sheeps", 1) - hooks.filters.add_item("tests:add-sheeps", 2) - hooks.filters.add_items("tests:add-sheeps", [3, 4]) - - sheeps: list[int] = hooks.filters.apply("tests:add-sheeps", []) - self.assertEqual([0, 1, 2, 3, 4], sheeps) - - def test_filter_callbacks(self) -> None: - callback = hooks.filters.FilterCallback(lambda _: 1) - self.assertTrue(callback.is_in_context(None)) - self.assertFalse(callback.is_in_context("customcontext")) - self.assertEqual(1, callback.apply(0)) - - def test_filter_context(self) -> None: - with hooks.contexts.enter("testcontext"): - hooks.filters.add_item("test:sheeps", 1) - hooks.filters.add_item("test:sheeps", 2) - - self.assertEqual([1, 2], hooks.filters.apply("test:sheeps", [])) - self.assertEqual( - [1], hooks.filters.apply_from_context("testcontext", "test:sheeps", []) - ) - - def test_clear_context(self) -> None: - with hooks.contexts.enter("testcontext"): - hooks.filters.add_item("test:sheeps", 1) - hooks.filters.add_item("test:sheeps", 2) - - self.assertEqual([1, 2], hooks.filters.apply("test:sheeps", [])) - hooks.filters.clear("test:sheeps", context="testcontext") - self.assertEqual([2], hooks.filters.apply("test:sheeps", [])) diff --git a/tests/test_plugins_v0.py b/tests/test_plugins_v0.py index 311cac6..176e903 100644 --- a/tests/test_plugins_v0.py +++ b/tests/test_plugins_v0.py @@ -1,5 +1,5 @@ from __future__ import annotations -import typing as t + from unittest.mock import patch from tests.helpers import PluginsTestCase, temporary_root diff --git a/tests/test_utils.py b/tests/test_utils.py index c6c55e1b..882b2b5 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,6 +1,5 @@ import base64 import os -import sys import tempfile import unittest from io import StringIO diff --git a/tutor/commands/cli.py b/tutor/commands/cli.py index 98da421..dcd8949 100755 --- a/tutor/commands/cli.py +++ b/tutor/commands/cli.py @@ -1,4 +1,5 @@ from __future__ import annotations + import sys import typing as t diff --git a/tutor/commands/compose.py b/tutor/commands/compose.py index 921ff3c..af561a8 100644 --- a/tutor/commands/compose.py +++ b/tutor/commands/compose.py @@ -1,4 +1,5 @@ from __future__ import annotations + import os import re import typing as t @@ -13,11 +14,12 @@ from tutor import env as tutor_env from tutor import fmt, hooks, serialize, utils from tutor.commands import jobs from tutor.commands.context import BaseTaskContext +from tutor.core.hooks import Filter # pylint: disable=unused-import from tutor.exceptions import TutorError from tutor.tasks import BaseComposeTaskRunner from tutor.types import Config -COMPOSE_FILTER_TYPE: TypeAlias = "hooks.filters.Filter[dict[str, t.Any], []]" +COMPOSE_FILTER_TYPE: TypeAlias = "Filter[dict[str, t.Any], []]" class ComposeTaskRunner(BaseComposeTaskRunner): diff --git a/tutor/commands/config.py b/tutor/commands/config.py index 3d65801..0c7eb49 100644 --- a/tutor/commands/config.py +++ b/tutor/commands/config.py @@ -1,4 +1,5 @@ from __future__ import annotations + import json import typing as t diff --git a/tutor/commands/images.py b/tutor/commands/images.py index abc9207..06d2a2b 100644 --- a/tutor/commands/images.py +++ b/tutor/commands/images.py @@ -1,4 +1,5 @@ from __future__ import annotations + import typing as t import click @@ -7,6 +8,7 @@ from tutor import config as tutor_config from tutor import env as tutor_env from tutor import exceptions, hooks, images from tutor.commands.context import Context +from tutor.core.hooks import Filter from tutor.types import Config BASE_IMAGE_NAMES = ["openedx", "permissions"] @@ -183,7 +185,7 @@ def find_images_to_build( def find_remote_image_tags( config: Config, - filtre: "hooks.filters.Filter[list[tuple[str, str]], [Config]]", + filtre: Filter[list[tuple[str, str]], [Config]], image: str, ) -> t.Iterator[str]: """ diff --git a/tutor/commands/jobs.py b/tutor/commands/jobs.py index de80725..f20b904 100644 --- a/tutor/commands/jobs.py +++ b/tutor/commands/jobs.py @@ -2,6 +2,7 @@ Common jobs that must be added both to local, dev and k8s commands. """ from __future__ import annotations + import functools import typing as t diff --git a/tutor/commands/k8s.py b/tutor/commands/k8s.py index 006974d..8df275c 100644 --- a/tutor/commands/k8s.py +++ b/tutor/commands/k8s.py @@ -12,8 +12,8 @@ from tutor import serialize, utils from tutor.commands import jobs from tutor.commands.config import save as config_save_command from tutor.commands.context import BaseTaskContext -from tutor.commands.upgrade.k8s import upgrade_from from tutor.commands.upgrade import OPENEDX_RELEASE_NAMES +from tutor.commands.upgrade.k8s import upgrade_from from tutor.tasks import BaseTaskRunner from tutor.types import Config, get_typed diff --git a/tutor/commands/local.py b/tutor/commands/local.py index 140edc0..c39da85 100644 --- a/tutor/commands/local.py +++ b/tutor/commands/local.py @@ -1,4 +1,5 @@ from __future__ import annotations + import typing as t import click @@ -10,8 +11,8 @@ from tutor import interactive as interactive_config from tutor import utils from tutor.commands import compose from tutor.commands.config import save as config_save_command -from tutor.commands.upgrade.local import upgrade_from from tutor.commands.upgrade import OPENEDX_RELEASE_NAMES +from tutor.commands.upgrade.local import upgrade_from from tutor.types import Config, get_typed diff --git a/tutor/commands/plugins.py b/tutor/commands/plugins.py index 225be11..7db9ed4 100644 --- a/tutor/commands/plugins.py +++ b/tutor/commands/plugins.py @@ -1,6 +1,6 @@ from __future__ import annotations + import os -import typing as t import urllib.request import click diff --git a/tutor/config.py b/tutor/config.py index 5512b84..3459a82 100644 --- a/tutor/config.py +++ b/tutor/config.py @@ -1,4 +1,5 @@ from __future__ import annotations + import os from tutor import env, exceptions, fmt, hooks, plugins, serialize, utils diff --git a/tutor/core/hooks/__init__.py b/tutor/core/hooks/__init__.py new file mode 100644 index 0000000..1609444 --- /dev/null +++ b/tutor/core/hooks/__init__.py @@ -0,0 +1,15 @@ +import typing as t + +from .actions import Action, ActionTemplate +from .actions import clear_all as _clear_all_actions +from .contexts import Context, ContextTemplate +from .filters import Filter, FilterTemplate +from .filters import clear_all as _clear_all_filters + + +def clear_all(context: t.Optional[str] = None) -> None: + """ + Clear both actions and filters. + """ + _clear_all_actions(context=context) + _clear_all_filters(context=context) diff --git a/tutor/hooks/actions.py b/tutor/core/hooks/actions.py similarity index 56% rename from tutor/hooks/actions.py rename to tutor/core/hooks/actions.py index ccaf467..7562181 100644 --- a/tutor/hooks/actions.py +++ b/tutor/core/hooks/actions.py @@ -11,16 +11,18 @@ from typing_extensions import ParamSpec from . import priorities from .contexts import Contextualized -P = ParamSpec("P") +#: Action generic signature. +T = ParamSpec("T") + # Similarly to CallableFilter, it should be possible to create a CallableAction alias in # the future. -# CallableAction = t.Callable[P, None] +# CallableAction = t.Callable[T, None] -class ActionCallback(Contextualized, t.Generic[P]): +class ActionCallback(Contextualized, t.Generic[T]): def __init__( self, - func: t.Callable[P, None], + func: t.Callable[T, None], priority: t.Optional[int] = None, ): super().__init__() @@ -29,13 +31,13 @@ class ActionCallback(Contextualized, t.Generic[P]): def do( self, - *args: P.args, - **kwargs: P.kwargs, + *args: T.args, + **kwargs: T.kwargs, ) -> None: self.func(*args, **kwargs) -class Action(t.Generic[P]): +class Action(t.Generic[T]): """ Action hooks have callbacks that are triggered independently from one another. @@ -44,12 +46,12 @@ class Action(t.Generic[P]): This is the typical action lifecycle: - 1. Create an action with method :py:meth:`get` (or function :py:func:`get`). - 2. Add callbacks with method :py:meth:`add` (or function :py:func:`add`). - 3. Call the action callbacks with method :py:meth:`do` (or function :py:func:`do`). + 1. Create an action with method :py:meth:`get`. + 2. Add callbacks with method :py:meth:`add`. + 3. Call the action callbacks with method :py:meth:`do`. - The `P` type parameter of the Action class corresponds to the expected signature of - the action callbacks. For instance, `Action[[str, int]]` means that the action + The ``P`` type parameter of the Action class corresponds to the expected signature of + the action callbacks. For instance, ``Action[[str, int]]`` means that the action callbacks are expected to take two arguments: one string and one integer. This strong typing makes it easier for plugin developers to quickly check whether they are adding and calling action callbacks correctly. @@ -59,7 +61,7 @@ class Action(t.Generic[P]): def __init__(self, name: str) -> None: self.name = name - self.callbacks: list[ActionCallback[P]] = [] + self.callbacks: list[ActionCallback[T]] = [] def __repr__(self) -> str: return f"{self.__class__.__name__}('{self.name}')" @@ -73,14 +75,29 @@ class Action(t.Generic[P]): def add( self, priority: t.Optional[int] = None - ) -> t.Callable[[t.Callable[P, None]], t.Callable[P, None]]: + ) -> t.Callable[[t.Callable[T, None]], t.Callable[T, None]]: """ - Add a callback to the action + Decorator to add a callback to an action. - This is similar to :py:func:`add`. + :param priority: optional order in which the action callbacks are performed. Higher + values mean that they will be performed later. The default value is + ``priorities.DEFAULT`` (10). Actions that should be performed last should have a + priority of 100. + + Usage:: + + @my_action.add("my-action") + def do_stuff(my_arg): + ... + + The ``do_stuff`` callback function will be called on ``my_action.do(some_arg)``. + + The signature of each callback action function must match the signature of the + corresponding :py:meth:`do` method. Callback action functions are not supposed + to return any value. Returned values will be ignored. """ - def inner(func: t.Callable[P, None]) -> t.Callable[P, None]: + def inner(func: t.Callable[T, None]) -> t.Callable[T, None]: callback = ActionCallback(func, priority=priority) priorities.insert_callback(callback, self.callbacks) return func @@ -89,24 +106,36 @@ class Action(t.Generic[P]): def do( self, - *args: P.args, - **kwargs: P.kwargs, + *args: T.args, + **kwargs: T.kwargs, ) -> None: """ - Run the action callbacks + Run the action callbacks in sequence. - This is similar to :py:func:`do`. + :param name: name of the action for which callbacks will be run. + + Extra ``*args`` and ``*kwargs`` arguments will be passed as-is to + callback functions. + + Callbacks are executed in order of priority, then FIFO. There is no error + management here: a single exception will cause all following callbacks + not to be run and the exception will be bubbled up. """ self.do_from_context(None, *args, **kwargs) def do_from_context( self, context: t.Optional[str], - *args: P.args, - **kwargs: P.kwargs, + *args: T.args, + **kwargs: T.kwargs, ) -> None: """ - Same as :py:func:`do` but only run the callbacks from a given context. + Same as :py:meth:`do` but only run the callbacks from a given context. + + :param name: name of the action for which callbacks will be run. + :param context: limit the set of callback actions to those that + were declared within a certain context (see + :py:func:`tutor.core.hooks.contexts.enter`). """ for callback in self.callbacks: if callback.is_in_context(context): @@ -125,7 +154,15 @@ class Action(t.Generic[P]): """ Clear all or part of the callbacks associated to an action - This is similar to :py:func:`clear`. + :param name: name of the action callbacks to remove. + :param context: when defined, will clear only the actions that were + created within that context. + + Actions will be removed from the list of callbacks and will no longer be + run in :py:meth:`do` calls. + + This function should almost certainly never be called by plugins. It is + mostly useful to disable some plugins at runtime or in unit tests. """ self.callbacks = [ callback @@ -134,13 +171,26 @@ class Action(t.Generic[P]): ] -class ActionTemplate(t.Generic[P]): +class ActionTemplate(t.Generic[T]): """ Action templates are for actions for which the name needs to be formatted before the action can be applied. Action templates can generate different :py:class:`Action` objects for which the name matches a certain template. + + Templated actions must be formatted with ``(*args)`` before being applied. For example:: + + action_template = ActionTemplate("namespace:{0}") + + # Return the action named "namespace:name" + my_action = action_template("name") + + @my_action.add() + def my_callback(): + ... + + my_action.do() """ def __init__(self, name: str): @@ -149,8 +199,10 @@ class ActionTemplate(t.Generic[P]): def __repr__(self) -> str: return f"{self.__class__.__name__}('{self.template}')" - def __call__(self, *args: t.Any, **kwargs: t.Any) -> Action[P]: - return get(self.template.format(*args, **kwargs)) + def __call__(self, *args: t.Any, **kwargs: t.Any) -> Action[T]: + name = self.template.format(*args, **kwargs) + action: Action[T] = Action.get(name) + return action # Syntactic sugar @@ -160,85 +212,41 @@ get = Action.get def get_template(name: str) -> ActionTemplate[t.Any]: """ Create an action with a template name. - - Templated actions must be formatted with ``(*args)`` before being applied. For example:: - - action_template = actions.get_template("namespace:{0}") - - @action_template("name").add() - def my_callback(): - ... - - action_template("name").do() """ return ActionTemplate(name) def add( name: str, priority: t.Optional[int] = None -) -> t.Callable[[t.Callable[P, None]], t.Callable[P, None]]: +) -> t.Callable[[t.Callable[T, None]], t.Callable[T, None]]: """ Decorator to add a callback action associated to a name. - - :param name: name of the action. For forward compatibility, it is - recommended not to hardcode any string here, but to pick a value from - :py:class:`tutor.hooks.Actions` instead. - :param priority: optional order in which the action callbacks are performed. Higher - values mean that they will be performed later. The default value is - ``priorities.DEFAULT`` (10). Actions that should be performed last should have a - priority of 100. - - Usage:: - - from tutor import hooks - - @hooks.actions.add("my-action") - def do_stuff(): - ... - - The ``do_stuff`` callback function will be called on ``hooks.actions.do("my-action")``. (see :py:func:`do`) - - The signature of each callback action function must match the signature of the corresponding ``hooks.actions.do`` call. Callback action functions are not supposed to return any value. Returned values will be ignored. """ return get(name).add(priority=priority) def do( name: str, - *args: P.args, - **kwargs: P.kwargs, + *args: T.args, + **kwargs: T.kwargs, ) -> None: """ Run action callbacks associated to a name/context. - - :param name: name of the action for which callbacks will be run. - - Extra ``*args`` and ``*kwargs`` arguments will be passed as-is to - callback functions. - - Callbacks are executed in order of priority, then FIFO. There is no error - management here: a single exception will cause all following callbacks - not to be run and the exception to be bubbled up. """ - action: Action[P] = Action.get(name) + action: Action[T] = Action.get(name) action.do(*args, **kwargs) def do_from_context( context: str, name: str, - *args: P.args, - **kwargs: P.kwargs, + *args: T.args, + **kwargs: T.kwargs, ) -> None: """ Same as :py:func:`do` but only run the callbacks that were created in a given context. - - :param name: name of the action for which callbacks will be run. - :param context: limit the set of callback actions to those that - were declared within a certain context (see - :py:func:`tutor.hooks.contexts.enter`). """ - action: Action[P] = Action.get(name) + action: Action[T] = Action.get(name) action.do_from_context(context, *args, **kwargs) @@ -255,15 +263,5 @@ def clear_all(context: t.Optional[str] = None) -> None: def clear(name: str, context: t.Optional[str] = None) -> None: """ Clear any previously defined action with the given name and context. - - :param name: name of the action callbacks to remove. - :param context: when defined, will clear only the actions that were - created within that context. - - Actions will be removed from the list of callbacks and will no longer be - run in :py:func:`do` calls. - - This function should almost certainly never be called by plugins. It is - mostly useful to disable some plugins at runtime or in unit tests. """ Action.get(name).clear(context=context) diff --git a/tutor/hooks/contexts.py b/tutor/core/hooks/contexts.py similarity index 77% rename from tutor/hooks/contexts.py rename to tutor/core/hooks/contexts.py index 75a3182..1dc48a8 100644 --- a/tutor/hooks/contexts.py +++ b/tutor/core/hooks/contexts.py @@ -8,6 +8,27 @@ from contextlib import contextmanager class Context: + """ + Contexts are used to track in which parts of the code filters and actions have been + declared. Let's look at an example:: + + from tutor.core.hooks import contexts + + with contexts.enter("c1"): + @filters.add("f1") + def add_stuff_to_filter(...): + ... + + The fact that our custom filter was added in a certain context allows us to later + remove it. To do so, we write:: + + from tutor import hooks + filters.clear("f1", context="c1") + + For instance, contexts make it easy to disable side-effects by plugins, provided + they were created with a specific context. + """ + CURRENT: list[str] = [] def __init__(self, name: str): @@ -61,13 +82,14 @@ def enter(name: str) -> t.ContextManager[None]: Usage:: - from tutor import hooks + from tutor.core import hooks with hooks.contexts.enter("my-context"): # declare new actions and filters ... - # Later on, actions and filters can be disabled with: + # Later on, actions and filters that were created within this context can be + # disabled with: hooks.actions.clear_all(context="my-context") hooks.filters.clear_all(context="my-context") diff --git a/tutor/hooks/filters.py b/tutor/core/hooks/filters.py similarity index 55% rename from tutor/hooks/filters.py rename to tutor/core/hooks/filters.py index 4aa10f9..f2b505e 100644 --- a/tutor/hooks/filters.py +++ b/tutor/core/hooks/filters.py @@ -10,29 +10,34 @@ from typing_extensions import Concatenate, ParamSpec from . import contexts, priorities -T = t.TypeVar("T") -P = ParamSpec("P") -# Specialized typevar for list elements -E = t.TypeVar("E") +#: Filter generic return value, which is also the type of the first callback argument. +T1 = t.TypeVar("T1") +#: Filter generic signature for all arguments after the first one. +T2 = ParamSpec("T2") +#: Specialized typevar for list elements +L = t.TypeVar("L") + # I wish we could create such an alias, which would greatly simply the definitions # below. Unfortunately this does not work, yet. It will once the following issue is # resolved: https://github.com/python/mypy/issues/11855 -# CallableFilter = t.Callable[Concatenate[T, P], T] +# CallableFilter = t.Callable[Concatenate[T1, T2], T1] -class FilterCallback(contexts.Contextualized, t.Generic[T, P]): +class FilterCallback(contexts.Contextualized, t.Generic[T1, T2]): def __init__( - self, func: t.Callable[Concatenate[T, P], T], priority: t.Optional[int] = None + self, + func: t.Callable[Concatenate[T1, T2], T1], + priority: t.Optional[int] = None, ): super().__init__() self.func = func self.priority = priority or priorities.DEFAULT - def apply(self, value: T, *args: P.args, **kwargs: P.kwargs) -> T: + def apply(self, value: T1, *args: T2.args, **kwargs: T2.kwargs) -> T1: return self.func(value, *args, **kwargs) -class Filter(t.Generic[T, P]): +class Filter(t.Generic[T1, T2]): """ Filter hooks have callbacks that are triggered as a chain. @@ -41,9 +46,9 @@ class Filter(t.Generic[T, P]): This is the typical filter lifecycle: - 1. Create an action with method :py:meth:`get` (or function :py:func:`get`). - 2. Add callbacks with method :py:meth:`add` (or function :py:func:`add`). - 3. Call the filter callbacks with method :py:meth:`apply` (or function :py:func:`apply`). + 1. Create an action with method :py:meth:`get`. + 2. Add callbacks with method :py:meth:`add`. + 3. Call the filter callbacks with method :py:meth:`apply`. The result of each callback is passed as the first argument to the next one. Thus, the type of the first argument must match the callback return type. @@ -64,7 +69,7 @@ class Filter(t.Generic[T, P]): def __init__(self, name: str) -> None: self.name = name - self.callbacks: list[FilterCallback[T, P]] = [] + self.callbacks: list[FilterCallback[T1, T2]] = [] def __repr__(self) -> str: return f"{self.__class__.__name__}('{self.name}')" @@ -79,11 +84,37 @@ class Filter(t.Generic[T, P]): def add( self, priority: t.Optional[int] = None ) -> t.Callable[ - [t.Callable[Concatenate[T, P], T]], t.Callable[Concatenate[T, P], T] + [t.Callable[Concatenate[T1, T2], T1]], t.Callable[Concatenate[T1, T2], T1] ]: + """ + Decorator to add a filter callback. + + Callbacks are added by increasing priority. Highest priority score are called + last. + + :param int priority: optional order in which the filter callbacks are called. Higher + values mean that they will be performed later. The default value is + ``priorities.DEFAULT`` (10). Filters that must be called last should have a + priority of 100. + + The return value of each filter function callback will be passed as the first argument to the next one. + + Usage:: + + @my_filter.add() + def my_func(value, some_other_arg): + # Do something with `value` + ... + return value + + After filters have been created, the result of calling all filter callbacks is obtained by running: + + final_value = my_filter.apply(initial_value, some_other_argument_value) + """ + def inner( - func: t.Callable[Concatenate[T, P], T] - ) -> t.Callable[Concatenate[T, P], T]: + func: t.Callable[Concatenate[T1, T2], T1] + ) -> t.Callable[Concatenate[T1, T2], T1]: callback = FilterCallback(func, priority=priority) priorities.insert_callback(callback, self.callbacks) return func @@ -92,10 +123,10 @@ class Filter(t.Generic[T, P]): def apply( self, - value: T, - *args: t.Any, - **kwargs: t.Any, - ) -> T: + value: T1, + *args: T2.args, + **kwargs: T2.kwargs, + ) -> T1: """ Apply all declared filters to a single value, passing along the additional arguments. @@ -113,10 +144,15 @@ class Filter(t.Generic[T, P]): def apply_from_context( self, context: t.Optional[str], - value: T, - *args: P.args, - **kwargs: P.kwargs, - ) -> T: + value: T1, + *args: T2.args, + **kwargs: T2.kwargs, + ) -> T1: + """ + Same as :py:meth:`apply` but only run the callbacks that were created in a given context. + + If ``context`` is None then it is ignored. + """ for callback in self.callbacks: if callback.is_in_context(context): try: @@ -135,7 +171,7 @@ class Filter(t.Generic[T, P]): def clear(self, context: t.Optional[str] = None) -> None: """ - Clear any previously defined filter with the given name and context. + Clear any previously defined filter with the given context. """ self.callbacks = [ callback @@ -145,45 +181,118 @@ class Filter(t.Generic[T, P]): # The methods below are specific to filters which take lists as first arguments def add_item( - self: "Filter[list[E], P]", item: E, priority: t.Optional[int] = None + self: "Filter[list[L], T2]", item: L, priority: t.Optional[int] = None ) -> None: + """ + Convenience decorator to add a single item to a filter that returns a list of items. + + This method is only valid for filters that return list of items. + + :param object item: item that will be appended to the resulting list. + :param int priority: see :py:data:`Filter.add`. + + Usage:: + + my_filter.add_item("item1") + my_filter.add_item("item2") + + assert ["item1", "item2"] == my_filter.apply([]) + """ self.add_items([item], priority=priority) def add_items( - self: "Filter[list[E], P]", items: list[E], priority: t.Optional[int] = None + self: "Filter[list[L], T2]", items: list[L], priority: t.Optional[int] = None ) -> None: + """ + Convenience function to add multiple items to a filter that returns a list of items. + + This method is only valid for filters that return list of items. + + This is a similar method to :py:data:`Filter.add_item` except that it can be + used to add multiple items at the same time. If you find yourself calling + ``add_item`` multiple times on the same filter, then you probably want to use a + single call to ``add_items`` instead. + + :param name: filter name. + :param list[object] items: items that will be appended to the resulting list. + + Usage:: + + my_filter.add_items(["item1", "item2"]) + my_filter.add_items(["item3", "item4"]) + + assert ["item1", "item2", "item3", "item4"] == my_filter.apply([]) + + The following are equivalent:: + + # Single call to add_items + my_filter.add_items(["item1", "item2"]) + + # Multiple calls to add_item + my_filter.add_item("item1") + my_filter.add_item("item2") + """ # Unfortunately we have to type-ignore this line. If not, mypy complains with: # - # Argument 1 has incompatible type "Callable[[Arg(List[E], 'values'), **P], List[E]]"; expected "Callable[[List[E], **P], List[E]]" + # Argument 1 has incompatible type "Callable[[Arg(List[E], 'values'), **T2], List[E]]"; expected "Callable[[List[E], **T2], List[E]]" # This is likely because "callback" has named arguments: "values". Consider marking them positional-only # # But we are unable to mark arguments positional-only (by adding / after values arg) in Python 3.7. # Get rid of this statement after Python 3.7 EOL. @self.add(priority=priority) # type: ignore - def callback(values: list[E], *_args: P.args, **_kwargs: P.kwargs) -> list[E]: + def callback(values: list[L], *_args: T2.args, **_kwargs: T2.kwargs) -> list[L]: return values + items def iterate( - self: "Filter[list[E], P]", *args: P.args, **kwargs: P.kwargs - ) -> t.Iterator[E]: + self: "Filter[list[L], T2]", *args: T2.args, **kwargs: T2.kwargs + ) -> t.Iterator[L]: + """ + Convenient function to iterate over the results of a filter result list. + + This method is only valid for filters that return list of items. + + This pieces of code are equivalent:: + + for value in my_filter.apply([], *args, **kwargs): + ... + + for value in my_filter.iterate(*args, **kwargs): + ... + + :rtype iterator[T]: iterator over the list of items from the filter + """ yield from self.iterate_from_context(None, *args, **kwargs) def iterate_from_context( - self: "Filter[list[E], P]", + self: "Filter[list[L], T2]", context: t.Optional[str], - *args: P.args, - **kwargs: P.kwargs, - ) -> t.Iterator[E]: + *args: T2.args, + **kwargs: T2.kwargs, + ) -> t.Iterator[L]: + """ + Same as :py:func:`Filter.iterate` but apply only callbacks from a given context. + """ yield from self.apply_from_context(context, [], *args, **kwargs) -class FilterTemplate(t.Generic[T, P]): +class FilterTemplate(t.Generic[T1, T2]): """ Filter templates are for filters for which the name needs to be formatted before the filter can be applied. - Similar to :py:class:`tutor.hooks.actions.ActionTemplate`, filter templates are used to generate + Similar to :py:class:`tutor.core.hooks.ActionTemplate`, filter templates are used to generate :py:class:`Filter` objects for which the name matches a certain template. + + Templated filters must be formatted with ``(*args)`` before being applied. For example:: + + filter_template = FilterTemplate("namespace:{0}") + named_filter = filter_template("name") + + @named_filter.add() + def my_callback(x: int) -> int: + ... + + named_filter.apply(42) """ def __init__(self, name: str): @@ -192,7 +301,7 @@ class FilterTemplate(t.Generic[T, P]): def __repr__(self) -> str: return f"{self.__class__.__name__}('{self.template}')" - def __call__(self, *args: t.Any, **kwargs: t.Any) -> Filter[T, P]: + def __call__(self, *args: t.Any, **kwargs: t.Any) -> Filter[T1, T2]: return get(self.template.format(*args, **kwargs)) @@ -203,144 +312,68 @@ get = Filter.get def get_template(name: str) -> FilterTemplate[t.Any, t.Any]: """ Create a filter with a template name. - - Templated filters must be formatted with ``(*args)`` before being applied. For example:: - - filter_template = filters.get_template("namespace:{0}") - named_filter = filter_template("name") - - @named_filter.add() - def my_callback(x: int) -> int: - ... - - named_filter.apply(42) """ return FilterTemplate(name) def add( name: str, priority: t.Optional[int] = None -) -> t.Callable[[t.Callable[Concatenate[T, P], T]], t.Callable[Concatenate[T, P], T]]: +) -> t.Callable[ + [t.Callable[Concatenate[T1, T2], T1]], t.Callable[Concatenate[T1, T2], T1] +]: """ Decorator for functions that will be applied to a single named filter. - - :param str name: name of the filter to which the decorated function should be added. - :param int priority: optional order in which the filter callbacks are called. Higher - values mean that they will be performed later. The default value is - ``priorities.DEFAULT`` (10). Filters that should be called last should have a - priority of 100. - - The return value of each filter function callback will be passed as the first argument to the next one. - - Usage:: - - from tutor import hooks - - @hooks.filters.add("my-filter") - def my_func(value, some_other_arg): - # Do something with `value` - ... - return value - - # After filters have been created, the result of calling all filter callbacks is obtained by running: - hooks.filters.apply("my-filter", initial_value, some_other_argument_value) """ return Filter.get(name).add(priority=priority) -def add_item(name: str, item: T, priority: t.Optional[int] = None) -> None: +def add_item(name: str, item: T1, priority: t.Optional[int] = None) -> None: """ Convenience function to add a single item to a filter that returns a list of items. - - :param name: filter name. - :param object item: item that will be appended to the resulting list. - :param int priority: see :py:data:`add`. - - Usage:: - - from tutor import hooks - - hooks.filters.add_item("my-filter", "item1") - hooks.filters.add_item("my-filter", "item2") - - assert ["item1", "item2"] == hooks.filters.apply("my-filter", []) """ get(name).add_item(item, priority=priority) -def add_items(name: str, items: list[T], priority: t.Optional[int] = None) -> None: +def add_items(name: str, items: list[T1], priority: t.Optional[int] = None) -> None: """ - Convenience function to add multiple item to a filter that returns a list of items. - - :param name: filter name. - :param list[object] items: items that will be appended to the resulting list. - - Usage:: - - from tutor import hooks - - hooks.filters.add_items("my-filter", ["item1", "item2"]) - - assert ["item1", "item2"] == hooks.filters.apply("my-filter", []) + Convenience decorator to add multiple item to a filter that returns a list of items. """ get(name).add_items(items, priority=priority) -def iterate(name: str, *args: t.Any, **kwargs: t.Any) -> t.Iterator[T]: +def iterate(name: str, *args: t.Any, **kwargs: t.Any) -> t.Iterator[T1]: """ Convenient function to iterate over the results of a filter result list. - - This pieces of code are equivalent:: - - for value in filters.apply("my-filter", [], *args, **kwargs): - ... - - for value in filters.iterate("my-filter", *args, **kwargs): - ... - - :rtype iterator[T]: iterator over the list items from the filter with the same name. """ yield from iterate_from_context(None, name, *args, **kwargs) def iterate_from_context( context: t.Optional[str], name: str, *args: t.Any, **kwargs: t.Any -) -> t.Iterator[T]: - """ - Same as :py:func:`iterate` but apply only callbacks from a given context. - """ +) -> t.Iterator[T1]: yield from Filter.get(name).iterate_from_context(context, *args, **kwargs) -def apply(name: str, value: T, *args: t.Any, **kwargs: t.Any) -> T: +def apply(name: str, value: T1, *args: t.Any, **kwargs: t.Any) -> T1: """ Apply all declared filters to a single value, passing along the additional arguments. - - The return value of every filter is passed as the first argument to the next callback. - - Usage:: - - results = filters.apply("my-filter", ["item0"]) - - :type value: object - :rtype: same as the type of ``value``. """ return apply_from_context(None, name, value, *args, **kwargs) def apply_from_context( - context: t.Optional[str], name: str, value: T, *args: P.args, **kwargs: P.kwargs -) -> T: + context: t.Optional[str], name: str, value: T1, *args: T2.args, **kwargs: T2.kwargs +) -> T1: """ Same as :py:func:`apply` but only run the callbacks that were created in a given context. """ - filtre: Filter[T, P] = Filter.get(name) + filtre: Filter[T1, T2] = Filter.get(name) return filtre.apply_from_context(context, value, *args, **kwargs) def clear_all(context: t.Optional[str] = None) -> None: """ - Clear any previously defined filter with the given context. + Clear any previously defined filter with the given context. """ for name in Filter.INDEX: clear(name, context=context) diff --git a/tutor/hooks/priorities.py b/tutor/core/hooks/priorities.py similarity index 99% rename from tutor/hooks/priorities.py rename to tutor/core/hooks/priorities.py index c493bdb..616debc 100644 --- a/tutor/hooks/priorities.py +++ b/tutor/core/hooks/priorities.py @@ -1,4 +1,5 @@ from __future__ import annotations + import typing as t from typing_extensions import Protocol diff --git a/tutor/env.py b/tutor/env.py index 908a38e..e2a3a1b 100644 --- a/tutor/env.py +++ b/tutor/env.py @@ -1,4 +1,5 @@ from __future__ import annotations + import os import re import shutil diff --git a/tutor/hooks/__init__.py b/tutor/hooks/__init__.py index 3944f29..9853e57 100644 --- a/tutor/hooks/__init__.py +++ b/tutor/hooks/__init__.py @@ -3,14 +3,7 @@ __license__ = "Apache 2.0" import typing as t -# These imports are the hooks API -from . import actions, contexts, filters, priorities -from .consts import * +# The imports that follow are the hooks API +from tutor.core.hooks import clear_all, priorities - -def clear_all(context: t.Optional[str] = None) -> None: - """ - Clear both actions and filters. - """ - filters.clear_all(context=context) - actions.clear_all(context=context) +from .catalog import Actions, Contexts, Filters diff --git a/tutor/hooks/consts.py b/tutor/hooks/catalog.py similarity index 83% rename from tutor/hooks/consts.py rename to tutor/hooks/catalog.py index 986e2df..39ef13d 100644 --- a/tutor/hooks/consts.py +++ b/tutor/hooks/catalog.py @@ -11,35 +11,59 @@ from typing import Any, Callable, Iterable import click +from tutor.core.hooks import ( + Action, + ActionTemplate, + Context, + ContextTemplate, + Filter, + FilterTemplate, + actions, + filters, +) from tutor.types import Config -from . import actions, contexts, filters -from .actions import Action, ActionTemplate -from .filters import Filter, FilterTemplate - __all__ = ["Actions", "Filters", "Contexts"] class Actions: """ - This class is a container for the names of all actions used across Tutor - (see :py:mod:`tutor.hooks.actions.do`). For each action, we describe the - arguments that are passed to the callback functions. + This class is a container for all actions used across Tutor (see + :py:class:`tutor.core.hooks.Action`). Actions are used to trigger callback functions at + specific moments in the Tutor life cycle. - To create a new callback for an existing action, write the following:: + To create a new callback for an existing action, start by importing the hooks + module:: from tutor import hooks - @hooks.Actions.YOUR_ACTION.add() + Then create your callback function and decorate it with the :py:meth:`add ` method of the + action you're interested in:: + + @hooks.Actions.SOME_ACTION.add() def your_action(): # Do stuff here + + Your callback function should have the same signature as the original action. For + instance, to add a callback to the :py:data:`COMPOSE_PROJECT_STARTED` action:: + + @hooks.Actions.COMPOSE_PROJECT_STARTED.add(): + def run_this_on_start(root, config, name): + print(root, config["LMS_HOST", name]) + + Your callback function will then be called whenever the ``COMPOSE_PROJECT_STARTED.do`` method + is called, i.e: when ``tutor local start`` or ``tutor dev start`` is run. + + Note that action callbacks do not return anything. + + For more information about how actions work, check out the :py:class:`tutor.core.hooks.Action` API. """ #: Triggered whenever a "docker-compose start", "up" or "restart" command is executed. #: - #: :parameter: str root: project root. - #: :parameter: dict config: project configuration. - #: :parameter: str name: docker-compose project name. + #: :parameter str root: project root. + #: :parameter dict config: project configuration. + #: :parameter str name: docker-compose project name. COMPOSE_PROJECT_STARTED: Action[[str, Config, str]] = actions.get( "compose:project:started" ) @@ -59,18 +83,13 @@ class Actions: #: This action does not have any parameter. CORE_READY: Action[[]] = actions.get("core:ready") - #: Called just before triggering the job tasks of any `... do ` command. + #: Called just before triggering the job tasks of any ``... do `` command. #: - #: :parameter: str job: job name. - #: :parameter: args: job positional arguments. - #: :parameter: kwargs: job named arguments. + #: :parameter str job: job name. + #: :parameter args: job positional arguments. + #: :parameter kwargs: job named arguments. DO_JOB: Action[[str, Any]] = actions.get("do:job") - #: Called as soon as we have access to the Tutor project root. - #: - #: :parameter str root: absolute path to the project root. - PROJECT_ROOT_READY: Action[str] = actions.get("project:root:ready") - #: Triggered when a single plugin needs to be loaded. Only plugins that have previously been #: discovered can be loaded (see :py:data:`CORE_READY`). #: @@ -103,24 +122,56 @@ class Actions: #: :parameter config: full project configuration PLUGIN_UNLOADED: Action[str, str, Config] = actions.get("plugins:unloaded") + #: Called as soon as we have access to the Tutor project root. + #: + #: :parameter str root: absolute path to the project root. + PROJECT_ROOT_READY: Action[str] = actions.get("project:root:ready") + class Filters: """ - Here are the names of all filters used across Tutor. For each filter, the - type of the first argument also indicates the type of the expected returned value. + Here are the names of all filters used across Tutor. (see + :py:class:`tutor.core.hooks.Filter`) Filters are used to modify some data at + specific points during the Tutor life cycle. - Filter names are all namespaced with domains separated by colons (":"). - - To add custom data to any filter, write the following in your plugin:: + To add a callback to an existing filter, start by importing the hooks module:: from tutor import hooks - @hooks.Filters.YOUR_FILTER.add() - def your_filter(items): - # do stuff with items + Then create your callback function and decorate it with :py:meth:`add + ` method of the filter instance you need:: + + @hooks.Filters.SOME_FILTER.add() + def your_filter_callback(some_data): + # Do stuff here with the data ... - # return the modified list of items - return items + # return the modified data + return some_data + + Note that your filter callback should have the same signature as the original + filter. The return value should also have the same type as the first argument of the + callback function. + + Many filters have a list of items as the first argument. Quite often, plugin + developers just want to add a new item at the end of that list. In such cases there + is no need for a callback function. Instead, you can use the `add_item` method. For + instance, you can add a "hello" to the init task of the lms container by modifying + the :py:data:`CLI_DO_INIT_TASKS` filter:: + + hooks.CLI_DO_INIT_TASKS.add_item(("lms", "echo hello")) + + To add multiple items at a time, use `add_items`:: + + hooks.CLI_DO_INIT_TASKS.add_items( + ("lms", "echo 'hello from lms'"), + ("cms", "echo 'hello from cms'"), + ) + + The ``echo`` commands will then be run every time the "init" tasks are run, for + instance during `tutor local launch`. + + For more information about how filters work, check out the + :py:class:`tutor.core.hooks.Filter` API. """ #: List of command line interface (CLI) commands. @@ -382,47 +433,31 @@ class Filters: class Contexts: """ - Contexts are used to track in which parts of the code filters and actions have been - declared. Let's look at an example:: - - from tutor import hooks - - with hooks.contexts.enter("c1"): - @filters.add("f1") def add_stuff_to_filter(...): - ... - - The fact that our custom filter was added in a certain context allows us to later - remove it. To do so, we write:: - - from tutor import hooks - filters.clear("f1", context="c1") - - This makes it easy to disable side-effects by plugins, provided they were created with appropriate contexts. - - Here we list all the contexts that are used across Tutor. It is not expected that + Here we list all the :py:class:`contexts ` that are used across Tutor. It is not expected that plugin developers will ever need to use contexts. But if you do, this is how it should be done:: from tutor import hooks - with hooks.Contexts.MY_CONTEXT.enter(): - # do stuff and all created hooks will include MY_CONTEXT + with hooks.Contexts.SOME_CONTEXT.enter(): + # do stuff and all created hooks will include SOME_CONTEXT + ... - # Apply only the hook callbacks that were created within MY_CONTEXT - hooks.Actions.MY_ACTION.do_from_context(str(hooks.Contexts.MY_CONTEXT)) - hooks.Filters.MY_FILTER.apply_from_context(hooks.Contexts.MY_CONTEXT.name) + # Apply only the hook callbacks that were created within SOME_CONTEXT + hooks.Actions.MY_ACTION.do_from_context(str(hooks.Contexts.SOME_CONTEXT)) + hooks.Filters.MY_FILTER.apply_from_context(hooks.Contexts.SOME_CONTEXT.name) """ #: We enter this context whenever we create hooks for a specific application or : #: plugin. For instance, plugin "myplugin" will be enabled within the "app:myplugin" #: context. - APP = contexts.ContextTemplate("app:{0}") + APP = ContextTemplate("app:{0}") #: Plugins will be installed and enabled within this context. - PLUGINS = contexts.Context("plugins") + PLUGINS = Context("plugins") #: YAML-formatted v0 plugins will be installed within this context. - PLUGINS_V0_YAML = contexts.Context("plugins:v0:yaml") + PLUGINS_V0_YAML = Context("plugins:v0:yaml") #: Python entrypoint plugins will be installed within this context. - PLUGINS_V0_ENTRYPOINT = contexts.Context("plugins:v0:entrypoint") + PLUGINS_V0_ENTRYPOINT = Context("plugins:v0:entrypoint") diff --git a/tutor/plugins/__init__.py b/tutor/plugins/__init__.py index fd137be..b8e5564 100644 --- a/tutor/plugins/__init__.py +++ b/tutor/plugins/__init__.py @@ -2,6 +2,7 @@ Provide API for plugin features. """ from __future__ import annotations + import typing as t from copy import deepcopy diff --git a/tutor/serialize.py b/tutor/serialize.py index a838d40..b46d414 100644 --- a/tutor/serialize.py +++ b/tutor/serialize.py @@ -1,4 +1,5 @@ from __future__ import annotations + import re import typing as t diff --git a/tutor/types.py b/tutor/types.py index c6156b3..bdf127d 100644 --- a/tutor/types.py +++ b/tutor/types.py @@ -20,6 +20,7 @@ ConfigValue: TypeAlias = t.Union[ t.Dict[t.Any, t.Any], ] +#: Type alias for the user configuration. Config: TypeAlias = t.Dict[str, ConfigValue]