feat: render files in ignored directories

When rendering theme files in a plugin, the *.scss files are stored in a
"partials" subdirectory, which was ignored by the environment rendering logic.
To render these files, we move the path ignoring logic to a filter, which is a
list of regular expressions. Values in this filter can be overridden by another
filter.

See the corresponding issue in the indigo theme plugin:
https://github.com/overhangio/tutor-indigo/issues/24

CHANGELOG.md

@@ -18,6 +18,7 @@ Every user-facing change should have an entry in this changelog. Please respect
 
 ## Unreleased
 
+- [Feature] Make it possible to force the rendering of a given template, even when the template path matches an ignore pattern. (by @regisb)
 - 💥[Fix] Get rid of the `tutor config render` command, which is useless now that themes can be implemented as plugins. (by @regisb)
 
 ## v13.2.3 (2022-05-30)

tests/test_env.py

@@ -24,6 +24,20 @@ class EnvTests(PluginsTestCase):
         self.assertIn(template_name, renderer.environment.loader.list_templates())
         self.assertNotIn(template_name, templates)
 
+    def test_files_are_rendered(self) -> None:
+        self.assertTrue(env.is_rendered("some/file"))
+        self.assertFalse(env.is_rendered(".git"))
+        self.assertFalse(env.is_rendered(".git/subdir"))
+        self.assertFalse(env.is_rendered("directory/.git"))
+        self.assertFalse(env.is_rendered("directory/.git/somefile"))
+        self.assertFalse(env.is_rendered("directory/somefile.pyc"))
+        self.assertTrue(env.is_rendered("directory/somedir.pyc/somefile"))
+        self.assertFalse(env.is_rendered("directory/__pycache__"))
+        self.assertFalse(env.is_rendered("directory/__pycache__/somefile"))
+        self.assertFalse(env.is_rendered("directory/partials/extra.scss"))
+        self.assertFalse(env.is_rendered("directory/partials"))
+        self.assertFalse(env.is_rendered("partials/somefile"))
+
     def test_is_binary_file(self) -> None:
         self.assertTrue(env.is_binary_file("/home/somefile.ico"))
 

tutor/env.py

@@ -1,4 +1,5 @@
 import os
+import re
 import shutil
 import typing as t
 from copy import deepcopy
@@ -67,17 +68,10 @@ class JinjaEnvironment(jinja2.Environment):
 
 
 class Renderer:
-    def __init__(
-        self,
-        config: t.Optional[Config] = None,
-        ignore_folders: t.Optional[t.List[str]] = None,
-    ):
+    def __init__(self, config: t.Optional[Config] = None):
         config = config or {}
         self.config = deepcopy(config)
         self.template_roots = hooks.Filters.ENV_TEMPLATE_ROOTS.apply([TEMPLATES_ROOT])
-        self.ignore_folders = ["partials", ".git"]
-        if ignore_folders is not None:
-            self.ignore_folders = ignore_folders
 
         # Create environment with extra filters and globals
         self.environment = JinjaEnvironment(self.template_roots)
@@ -110,8 +104,11 @@ class Renderer:
         full_prefix = "/".join(prefix)
         env_templates: t.List[str] = self.environment.loader.list_templates()
         for template in env_templates:
-            if template.startswith(full_prefix) and self.is_part_of_env(template):
-                yield template
+            if template.startswith(full_prefix):
+                # Exclude templates that match certain patterns
+                # Note that here we don't rely on the OS separator, as we are handling templates.
+                if is_rendered(template):
+                    yield template
 
     def iter_values_named(
         self,
@@ -142,23 +139,7 @@ class Renderer:
         Yield:
             path: template path relative to the template root
         """
-        yield from self.iter_templates_in(subdir + "/")
-
-    def is_part_of_env(self, path: str) -> bool:
-        """
-        Determines whether a template should be rendered or not. Note that here we don't
-        rely on the OS separator, as we are handling templates
-        """
-        parts = path.split("/")
-        basename = parts[-1]
-        is_excluded = False
-        is_excluded = (
-            is_excluded or basename.startswith(".") or basename.endswith(".pyc")
-        )
-        is_excluded = is_excluded or basename == "__pycache__"
-        for ignore_folder in self.ignore_folders:
-            is_excluded = is_excluded or ignore_folder in parts
-        return not is_excluded
+        yield from self.iter_templates_in(subdir)
 
     def find_os_path(self, template_name: str) -> str:
         path = template_name.replace("/", os.sep)
@@ -232,6 +213,43 @@ class Renderer:
             raise exceptions.TutorError(f"Missing configuration value: {e.args[0]}")
 
 
+def is_rendered(path: str) -> bool:
+    """
+    Return whether the template should be rendered or not.
+
+    If the path matches an include pattern, it is rendered. If not and it matches an
+    ignore pattern, it is not rendered. By default, all files are rendered.
+    """
+    include_patterns: t.Iterator[str] = hooks.Filters.ENV_PATTERNS_INCLUDE.iterate()
+    for include_pattern in include_patterns:
+        if re.match(include_pattern, path):
+            return True
+    ignore_patterns: t.Iterator[str] = hooks.Filters.ENV_PATTERNS_IGNORE.iterate()
+    for ignore_pattern in ignore_patterns:
+        if re.match(ignore_pattern, path):
+            return False
+    return True
+
+
+# Skip rendering some files that follow commonly-ignored patterns:
+#
+#   .*
+#   *.pyc
+#   __pycache__
+#   partials
+hooks.Filters.ENV_PATTERNS_IGNORE.add_items(
+    [
+        # Skip all hidden files
+        r"(.*/)?\.",
+        # Skip compiled python files
+        r"(.*/)?__pycache__(/.*)?$",
+        r".*\.pyc$",
+        # Skip files from "partials" folders
+        r"(.*/)?partials(/.*)?$",
+    ]
+)
+
+
 def save(root: str, config: Config) -> None:
     """
     Save the full environment, including version information.

tutor/hooks/consts.py

@@ -223,6 +223,23 @@ class Filters:
     #:   filter to modify the Tutor templates.
     ENV_PATCHES = filters.get("env:patches")
 
+    #: List of template path patterns to be ignored when rendering templates to the project root. By default, we ignore:
+    #:
+    #: - hidden files (``.*``)
+    #: - ``__pycache__`` directories and ``*.pyc`` files
+    #: - "partials" directories.
+    #:
+    #: Ignored patterns are overridden by include patterns; see :py:data:`ENV_PATTERNS_INCLUDE`.
+    #:
+    #: :parameter list[str] patterns: list of regular expression patterns. E.g: ``r"(.*/)?ignored_file_name(/.*)?"``.
+    ENV_PATTERNS_IGNORE = filters.get("env:patterns:ignore")
+
+    #: List of template path patterns to be included when rendering templates to the project root.
+    #: Patterns from this list will take priority over the patterns from :py:data:`ENV_PATTERNS_IGNORE`.
+    #:
+    #: :parameter list[str] patterns: list of regular expression patterns. See :py:data:`ENV_PATTERNS_IGNORE`.
+    ENV_PATTERNS_INCLUDE = filters.get("env:patterns:include")
+
     #: List of all template root folders.
     #:
     #: :parameter list[str] templates_root: absolute paths to folders which contain templates.