Merge remote-tracking branch 'origin/master' into nightly

CHANGELOG.md

@@ -18,6 +18,9 @@ Every user-facing change should have an entry in this changelog. Please respect
 
 ## Unreleased
 
+- [Feature] Make it possible to force the rendering of a given template, even when the template path matches an ignore pattern. (by @regisb)
+- 💥[Fix] Get rid of the `tutor config render` command, which is useless now that themes can be implemented as plugins. (by @regisb)
+
 ## v13.2.3 (2022-05-30)
 
 - [Fix] Truncate site display name to 50 characters with a warning, fixing data too long error for long site names. (by @navinkarkera)

tests/commands/test_config.py

@@ -1,9 +1,6 @@
-import os
-import tempfile
 import unittest
 
 from tests.helpers import temporary_root
-from tutor import config as tutor_config
 
 from .base import TestCommandMixin
 
@@ -61,29 +58,3 @@ class ConfigTests(unittest.TestCase, TestCommandMixin):
         self.assertFalse(result.exception)
         self.assertEqual(0, result.exit_code)
         self.assertTrue(result.output)
-
-    def test_config_render(self) -> None:
-        with tempfile.TemporaryDirectory() as dest:
-            with temporary_root() as root:
-                self.invoke_in_root(root, ["config", "save"])
-                result = self.invoke_in_root(root, ["config", "render", root, dest])
-        self.assertEqual(0, result.exit_code)
-        self.assertFalse(result.exception)
-
-    def test_config_render_with_extra_configs(self) -> None:
-        with tempfile.TemporaryDirectory() as dest:
-            with temporary_root() as root:
-                self.invoke_in_root(root, ["config", "save"])
-                result = self.invoke_in_root(
-                    root,
-                    [
-                        "config",
-                        "render",
-                        "-x",
-                        os.path.join(root, tutor_config.CONFIG_FILENAME),
-                        root,
-                        dest,
-                    ],
-                )
-        self.assertEqual(0, result.exit_code)
-        self.assertFalse(result.exception)

tests/test_env.py

@@ -13,22 +13,36 @@ from tutor.types import Config
 
 class EnvTests(PluginsTestCase):
     def test_walk_templates(self) -> None:
-        renderer = env.Renderer({}, [env.TEMPLATES_ROOT])
+        renderer = env.Renderer()
         templates = list(renderer.walk_templates("local"))
         self.assertIn("local/docker-compose.yml", templates)
 
     def test_walk_templates_partials_are_ignored(self) -> None:
         template_name = "apps/openedx/settings/partials/common_all.py"
-        renderer = env.Renderer({}, [env.TEMPLATES_ROOT], ignore_folders=["partials"])
+        renderer = env.Renderer()
         templates = list(renderer.walk_templates("apps"))
         self.assertIn(template_name, renderer.environment.loader.list_templates())
         self.assertNotIn(template_name, templates)
 
+    def test_files_are_rendered(self) -> None:
+        self.assertTrue(env.is_rendered("some/file"))
+        self.assertFalse(env.is_rendered(".git"))
+        self.assertFalse(env.is_rendered(".git/subdir"))
+        self.assertFalse(env.is_rendered("directory/.git"))
+        self.assertFalse(env.is_rendered("directory/.git/somefile"))
+        self.assertFalse(env.is_rendered("directory/somefile.pyc"))
+        self.assertTrue(env.is_rendered("directory/somedir.pyc/somefile"))
+        self.assertFalse(env.is_rendered("directory/__pycache__"))
+        self.assertFalse(env.is_rendered("directory/__pycache__/somefile"))
+        self.assertFalse(env.is_rendered("directory/partials/extra.scss"))
+        self.assertFalse(env.is_rendered("directory/partials"))
+        self.assertFalse(env.is_rendered("partials/somefile"))
+
     def test_is_binary_file(self) -> None:
         self.assertTrue(env.is_binary_file("/home/somefile.ico"))
 
     def test_find_os_path(self) -> None:
-        renderer = env.Renderer({}, [env.TEMPLATES_ROOT])
+        renderer = env.Renderer()
         path = renderer.find_os_path("local/docker-compose.yml")
         self.assertTrue(os.path.exists(path))
 
@@ -180,14 +194,14 @@ class EnvTests(PluginsTestCase):
 
             # Load env once
             config: Config = {"PLUGINS": []}
-            env1 = env.Renderer.instance(config).environment
+            env1 = env.Renderer(config).environment
 
             # Enable plugins
             plugins.load("plugin1")
 
             # Load env a second time
             config["PLUGINS"] = ["myplugin"]
-            env2 = env.Renderer.instance(config).environment
+            env2 = env.Renderer(config).environment
 
             self.assertNotIn("plugin1/myplugin.txt", env1.loader.list_templates())
             self.assertIn("plugin1/myplugin.txt", env2.loader.list_templates())
@@ -199,7 +213,7 @@ class EnvTests(PluginsTestCase):
             "notsomething_test_app": 2,
             "something3_test_app": 3,
         }
-        renderer = env.Renderer.instance(config)
+        renderer = env.Renderer(config)
         self.assertEqual([2, 3], list(renderer.iter_values_named(suffix="test_app")))
         self.assertEqual([1, 3], list(renderer.iter_values_named(prefix="something")))
         self.assertEqual(

tutor/commands/config.py

@@ -64,29 +64,6 @@ def save(
     env.save(context.root, config)
 
 
-@click.command(help="Render a template folder with eventual extra configuration files")
-@click.option(
-    "-x",
-    "--extra-config",
-    "extra_configs",
-    multiple=True,
-    type=click.Path(exists=True, resolve_path=True, dir_okay=False),
-    help="Load extra configuration file (can be used multiple times)",
-)
-@click.argument("src", type=click.Path(exists=True, resolve_path=True))
-@click.argument("dst")
-@click.pass_obj
-def render(context: Context, extra_configs: List[str], src: str, dst: str) -> None:
-    config = tutor_config.load(context.root)
-    for extra_config in extra_configs:
-        config.update(
-            env.render_unknown(config, tutor_config.get_yaml_file(extra_config))
-        )
-    renderer = env.Renderer(config, [src])
-    renderer.render_all_to(dst)
-    fmt.echo_info(f"Templates rendered to {dst}")
-
-
 @click.command(help="Print the project root")
 @click.pass_obj
 def printroot(context: Context) -> None:
@@ -106,6 +83,5 @@ def printvalue(context: Context, key: str) -> None:
 
 
 config_command.add_command(save)
-config_command.add_command(render)
 config_command.add_command(printroot)
 config_command.add_command(printvalue)

tutor/env.py

@@ -1,4 +1,5 @@
 import os
+import re
 import shutil
 import typing as t
 from copy import deepcopy
@@ -67,26 +68,13 @@ class JinjaEnvironment(jinja2.Environment):
 
 
 class Renderer:
-    @classmethod
+    def __init__(self, config: t.Optional[Config] = None):
-    def instance(cls: t.Type["Renderer"], config: Config) -> "Renderer":
+        config = config or {}
-        # Load template roots: these are required to be able to use
-        # {% include .. %} directives
-        template_roots = hooks.Filters.ENV_TEMPLATE_ROOTS.apply([TEMPLATES_ROOT])
-        return cls(config, template_roots, ignore_folders=["partials"])
-
-    def __init__(
-        self,
-        config: Config,
-        template_roots: t.List[str],
-        ignore_folders: t.Optional[t.List[str]] = None,
-    ):
         self.config = deepcopy(config)
-        self.template_roots = template_roots
+        self.template_roots = hooks.Filters.ENV_TEMPLATE_ROOTS.apply([TEMPLATES_ROOT])
-        self.ignore_folders = ignore_folders or []
-        self.ignore_folders.append(".git")
 
         # Create environment with extra filters and globals
-        self.environment = JinjaEnvironment(template_roots)
+        self.environment = JinjaEnvironment(self.template_roots)
 
         # Filters
         plugin_filters: t.Iterator[
@@ -116,7 +104,10 @@ class Renderer:
         full_prefix = "/".join(prefix)
         env_templates: t.List[str] = self.environment.loader.list_templates()
         for template in env_templates:
-            if template.startswith(full_prefix) and self.is_part_of_env(template):
+            if template.startswith(full_prefix):
+                # Exclude templates that match certain patterns
+                # Note that here we don't rely on the OS separator, as we are handling templates.
+                if is_rendered(template):
                     yield template
 
     def iter_values_named(
@@ -148,23 +139,7 @@ class Renderer:
         Yield:
             path: template path relative to the template root
         """
-        yield from self.iter_templates_in(subdir + "/")
+        yield from self.iter_templates_in(subdir)
-
-    def is_part_of_env(self, path: str) -> bool:
-        """
-        Determines whether a template should be rendered or not. Note that here we don't
-        rely on the OS separator, as we are handling templates
-        """
-        parts = path.split("/")
-        basename = parts[-1]
-        is_excluded = False
-        is_excluded = (
-            is_excluded or basename.startswith(".") or basename.endswith(".pyc")
-        )
-        is_excluded = is_excluded or basename == "__pycache__"
-        for ignore_folder in self.ignore_folders:
-            is_excluded = is_excluded or ignore_folder in parts
-        return not is_excluded
 
     def find_os_path(self, template_name: str) -> str:
         path = template_name.replace("/", os.sep)
@@ -238,6 +213,43 @@ class Renderer:
             raise exceptions.TutorError(f"Missing configuration value: {e.args[0]}")
 
 
+def is_rendered(path: str) -> bool:
+    """
+    Return whether the template should be rendered or not.
+
+    If the path matches an include pattern, it is rendered. If not and it matches an
+    ignore pattern, it is not rendered. By default, all files are rendered.
+    """
+    include_patterns: t.Iterator[str] = hooks.Filters.ENV_PATTERNS_INCLUDE.iterate()
+    for include_pattern in include_patterns:
+        if re.match(include_pattern, path):
+            return True
+    ignore_patterns: t.Iterator[str] = hooks.Filters.ENV_PATTERNS_IGNORE.iterate()
+    for ignore_pattern in ignore_patterns:
+        if re.match(ignore_pattern, path):
+            return False
+    return True
+
+
+# Skip rendering some files that follow commonly-ignored patterns:
+#
+#   .*
+#   *.pyc
+#   __pycache__
+#   partials
+hooks.Filters.ENV_PATTERNS_IGNORE.add_items(
+    [
+        # Skip all hidden files
+        r"(.*/)?\.",
+        # Skip compiled python files
+        r"(.*/)?__pycache__(/.*)?$",
+        r".*\.pyc$",
+        # Skip files from "partials" folders
+        r"(.*/)?partials(/.*)?$",
+    ]
+)
+
+
 def save(root: str, config: Config) -> None:
     """
     Save the full environment, including version information.
@@ -264,7 +276,7 @@ def save_all_from(prefix: str, dst: str, config: Config) -> None:
     Render the templates that start with `prefix` and store them with the same
     hierarchy at `dst`. Here, `prefix` can be the result of os.path.join(...).
     """
-    renderer = Renderer.instance(config)
+    renderer = Renderer(config)
     renderer.render_all_to(dst, prefix.replace(os.sep, "/"))
 
 
@@ -285,7 +297,7 @@ def render_file(config: Config, *path: str) -> t.Union[str, bytes]:
     """
     Return the rendered contents of a template.
     """
-    renderer = Renderer.instance(config)
+    renderer = Renderer(config)
     template_name = "/".join(path)
     return renderer.render_template(template_name)
 
@@ -312,7 +324,7 @@ def render_str(config: Config, text: str) -> str:
     Return:
         substituted (str)
     """
-    return Renderer.instance(config).render_str(text)
+    return Renderer(config).render_str(text)
 
 
 def check_is_up_to_date(root: str) -> None:

tutor/hooks/consts.py

@@ -223,6 +223,23 @@ class Filters:
     #:   filter to modify the Tutor templates.
     ENV_PATCHES = filters.get("env:patches")
 
+    #: List of template path patterns to be ignored when rendering templates to the project root. By default, we ignore:
+    #:
+    #: - hidden files (``.*``)
+    #: - ``__pycache__`` directories and ``*.pyc`` files
+    #: - "partials" directories.
+    #:
+    #: Ignored patterns are overridden by include patterns; see :py:data:`ENV_PATTERNS_INCLUDE`.
+    #:
+    #: :parameter list[str] patterns: list of regular expression patterns. E.g: ``r"(.*/)?ignored_file_name(/.*)?"``.
+    ENV_PATTERNS_IGNORE = filters.get("env:patterns:ignore")
+
+    #: List of template path patterns to be included when rendering templates to the project root.
+    #: Patterns from this list will take priority over the patterns from :py:data:`ENV_PATTERNS_IGNORE`.
+    #:
+    #: :parameter list[str] patterns: list of regular expression patterns. See :py:data:`ENV_PATTERNS_IGNORE`.
+    ENV_PATTERNS_INCLUDE = filters.get("env:patterns:include")
+
     #: List of all template root folders.
     #:
     #: :parameter list[str] templates_root: absolute paths to folders which contain templates.