diff --git a/CHANGELOG.md b/CHANGELOG.md index a4b4dbd..8d57dc9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,9 @@ Every user-facing change should have an entry in this changelog. Please respect ## Unreleased +- [Feature] Make it possible to force the rendering of a given template, even when the template path matches an ignore pattern. (by @regisb) +- 💥[Fix] Get rid of the `tutor config render` command, which is useless now that themes can be implemented as plugins. (by @regisb) + ## v13.2.3 (2022-05-30) - [Fix] Truncate site display name to 50 characters with a warning, fixing data too long error for long site names. (by @navinkarkera) diff --git a/tests/commands/test_config.py b/tests/commands/test_config.py index f044e11..731a183 100644 --- a/tests/commands/test_config.py +++ b/tests/commands/test_config.py @@ -1,9 +1,6 @@ -import os -import tempfile import unittest from tests.helpers import temporary_root -from tutor import config as tutor_config from .base import TestCommandMixin @@ -61,29 +58,3 @@ class ConfigTests(unittest.TestCase, TestCommandMixin): self.assertFalse(result.exception) self.assertEqual(0, result.exit_code) self.assertTrue(result.output) - - def test_config_render(self) -> None: - with tempfile.TemporaryDirectory() as dest: - with temporary_root() as root: - self.invoke_in_root(root, ["config", "save"]) - result = self.invoke_in_root(root, ["config", "render", root, dest]) - self.assertEqual(0, result.exit_code) - self.assertFalse(result.exception) - - def test_config_render_with_extra_configs(self) -> None: - with tempfile.TemporaryDirectory() as dest: - with temporary_root() as root: - self.invoke_in_root(root, ["config", "save"]) - result = self.invoke_in_root( - root, - [ - "config", - "render", - "-x", - os.path.join(root, tutor_config.CONFIG_FILENAME), - root, - dest, - ], - ) - self.assertEqual(0, result.exit_code) - self.assertFalse(result.exception) diff --git a/tests/test_env.py b/tests/test_env.py index b1a5e79..44447be 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -13,22 +13,36 @@ from tutor.types import Config class EnvTests(PluginsTestCase): def test_walk_templates(self) -> None: - renderer = env.Renderer({}, [env.TEMPLATES_ROOT]) + renderer = env.Renderer() templates = list(renderer.walk_templates("local")) self.assertIn("local/docker-compose.yml", templates) def test_walk_templates_partials_are_ignored(self) -> None: template_name = "apps/openedx/settings/partials/common_all.py" - renderer = env.Renderer({}, [env.TEMPLATES_ROOT], ignore_folders=["partials"]) + renderer = env.Renderer() templates = list(renderer.walk_templates("apps")) self.assertIn(template_name, renderer.environment.loader.list_templates()) self.assertNotIn(template_name, templates) + def test_files_are_rendered(self) -> None: + self.assertTrue(env.is_rendered("some/file")) + self.assertFalse(env.is_rendered(".git")) + self.assertFalse(env.is_rendered(".git/subdir")) + self.assertFalse(env.is_rendered("directory/.git")) + self.assertFalse(env.is_rendered("directory/.git/somefile")) + self.assertFalse(env.is_rendered("directory/somefile.pyc")) + self.assertTrue(env.is_rendered("directory/somedir.pyc/somefile")) + self.assertFalse(env.is_rendered("directory/__pycache__")) + self.assertFalse(env.is_rendered("directory/__pycache__/somefile")) + self.assertFalse(env.is_rendered("directory/partials/extra.scss")) + self.assertFalse(env.is_rendered("directory/partials")) + self.assertFalse(env.is_rendered("partials/somefile")) + def test_is_binary_file(self) -> None: self.assertTrue(env.is_binary_file("/home/somefile.ico")) def test_find_os_path(self) -> None: - renderer = env.Renderer({}, [env.TEMPLATES_ROOT]) + renderer = env.Renderer() path = renderer.find_os_path("local/docker-compose.yml") self.assertTrue(os.path.exists(path)) @@ -180,14 +194,14 @@ class EnvTests(PluginsTestCase): # Load env once config: Config = {"PLUGINS": []} - env1 = env.Renderer.instance(config).environment + env1 = env.Renderer(config).environment # Enable plugins plugins.load("plugin1") # Load env a second time config["PLUGINS"] = ["myplugin"] - env2 = env.Renderer.instance(config).environment + env2 = env.Renderer(config).environment self.assertNotIn("plugin1/myplugin.txt", env1.loader.list_templates()) self.assertIn("plugin1/myplugin.txt", env2.loader.list_templates()) @@ -199,7 +213,7 @@ class EnvTests(PluginsTestCase): "notsomething_test_app": 2, "something3_test_app": 3, } - renderer = env.Renderer.instance(config) + renderer = env.Renderer(config) self.assertEqual([2, 3], list(renderer.iter_values_named(suffix="test_app"))) self.assertEqual([1, 3], list(renderer.iter_values_named(prefix="something"))) self.assertEqual( diff --git a/tutor/commands/config.py b/tutor/commands/config.py index b329c95..83d347a 100644 --- a/tutor/commands/config.py +++ b/tutor/commands/config.py @@ -64,29 +64,6 @@ def save( env.save(context.root, config) -@click.command(help="Render a template folder with eventual extra configuration files") -@click.option( - "-x", - "--extra-config", - "extra_configs", - multiple=True, - type=click.Path(exists=True, resolve_path=True, dir_okay=False), - help="Load extra configuration file (can be used multiple times)", -) -@click.argument("src", type=click.Path(exists=True, resolve_path=True)) -@click.argument("dst") -@click.pass_obj -def render(context: Context, extra_configs: List[str], src: str, dst: str) -> None: - config = tutor_config.load(context.root) - for extra_config in extra_configs: - config.update( - env.render_unknown(config, tutor_config.get_yaml_file(extra_config)) - ) - renderer = env.Renderer(config, [src]) - renderer.render_all_to(dst) - fmt.echo_info(f"Templates rendered to {dst}") - - @click.command(help="Print the project root") @click.pass_obj def printroot(context: Context) -> None: @@ -106,6 +83,5 @@ def printvalue(context: Context, key: str) -> None: config_command.add_command(save) -config_command.add_command(render) config_command.add_command(printroot) config_command.add_command(printvalue) diff --git a/tutor/env.py b/tutor/env.py index 634372c..bd0636c 100644 --- a/tutor/env.py +++ b/tutor/env.py @@ -1,4 +1,5 @@ import os +import re import shutil import typing as t from copy import deepcopy @@ -67,26 +68,13 @@ class JinjaEnvironment(jinja2.Environment): class Renderer: - @classmethod - def instance(cls: t.Type["Renderer"], config: Config) -> "Renderer": - # Load template roots: these are required to be able to use - # {% include .. %} directives - template_roots = hooks.Filters.ENV_TEMPLATE_ROOTS.apply([TEMPLATES_ROOT]) - return cls(config, template_roots, ignore_folders=["partials"]) - - def __init__( - self, - config: Config, - template_roots: t.List[str], - ignore_folders: t.Optional[t.List[str]] = None, - ): + def __init__(self, config: t.Optional[Config] = None): + config = config or {} self.config = deepcopy(config) - self.template_roots = template_roots - self.ignore_folders = ignore_folders or [] - self.ignore_folders.append(".git") + self.template_roots = hooks.Filters.ENV_TEMPLATE_ROOTS.apply([TEMPLATES_ROOT]) # Create environment with extra filters and globals - self.environment = JinjaEnvironment(template_roots) + self.environment = JinjaEnvironment(self.template_roots) # Filters plugin_filters: t.Iterator[ @@ -116,8 +104,11 @@ class Renderer: full_prefix = "/".join(prefix) env_templates: t.List[str] = self.environment.loader.list_templates() for template in env_templates: - if template.startswith(full_prefix) and self.is_part_of_env(template): - yield template + if template.startswith(full_prefix): + # Exclude templates that match certain patterns + # Note that here we don't rely on the OS separator, as we are handling templates. + if is_rendered(template): + yield template def iter_values_named( self, @@ -148,23 +139,7 @@ class Renderer: Yield: path: template path relative to the template root """ - yield from self.iter_templates_in(subdir + "/") - - def is_part_of_env(self, path: str) -> bool: - """ - Determines whether a template should be rendered or not. Note that here we don't - rely on the OS separator, as we are handling templates - """ - parts = path.split("/") - basename = parts[-1] - is_excluded = False - is_excluded = ( - is_excluded or basename.startswith(".") or basename.endswith(".pyc") - ) - is_excluded = is_excluded or basename == "__pycache__" - for ignore_folder in self.ignore_folders: - is_excluded = is_excluded or ignore_folder in parts - return not is_excluded + yield from self.iter_templates_in(subdir) def find_os_path(self, template_name: str) -> str: path = template_name.replace("/", os.sep) @@ -238,6 +213,43 @@ class Renderer: raise exceptions.TutorError(f"Missing configuration value: {e.args[0]}") +def is_rendered(path: str) -> bool: + """ + Return whether the template should be rendered or not. + + If the path matches an include pattern, it is rendered. If not and it matches an + ignore pattern, it is not rendered. By default, all files are rendered. + """ + include_patterns: t.Iterator[str] = hooks.Filters.ENV_PATTERNS_INCLUDE.iterate() + for include_pattern in include_patterns: + if re.match(include_pattern, path): + return True + ignore_patterns: t.Iterator[str] = hooks.Filters.ENV_PATTERNS_IGNORE.iterate() + for ignore_pattern in ignore_patterns: + if re.match(ignore_pattern, path): + return False + return True + + +# Skip rendering some files that follow commonly-ignored patterns: +# +# .* +# *.pyc +# __pycache__ +# partials +hooks.Filters.ENV_PATTERNS_IGNORE.add_items( + [ + # Skip all hidden files + r"(.*/)?\.", + # Skip compiled python files + r"(.*/)?__pycache__(/.*)?$", + r".*\.pyc$", + # Skip files from "partials" folders + r"(.*/)?partials(/.*)?$", + ] +) + + def save(root: str, config: Config) -> None: """ Save the full environment, including version information. @@ -264,7 +276,7 @@ def save_all_from(prefix: str, dst: str, config: Config) -> None: Render the templates that start with `prefix` and store them with the same hierarchy at `dst`. Here, `prefix` can be the result of os.path.join(...). """ - renderer = Renderer.instance(config) + renderer = Renderer(config) renderer.render_all_to(dst, prefix.replace(os.sep, "/")) @@ -285,7 +297,7 @@ def render_file(config: Config, *path: str) -> t.Union[str, bytes]: """ Return the rendered contents of a template. """ - renderer = Renderer.instance(config) + renderer = Renderer(config) template_name = "/".join(path) return renderer.render_template(template_name) @@ -312,7 +324,7 @@ def render_str(config: Config, text: str) -> str: Return: substituted (str) """ - return Renderer.instance(config).render_str(text) + return Renderer(config).render_str(text) def check_is_up_to_date(root: str) -> None: diff --git a/tutor/hooks/consts.py b/tutor/hooks/consts.py index 64e6dd4..92f5709 100644 --- a/tutor/hooks/consts.py +++ b/tutor/hooks/consts.py @@ -223,6 +223,23 @@ class Filters: #: filter to modify the Tutor templates. ENV_PATCHES = filters.get("env:patches") + #: List of template path patterns to be ignored when rendering templates to the project root. By default, we ignore: + #: + #: - hidden files (``.*``) + #: - ``__pycache__`` directories and ``*.pyc`` files + #: - "partials" directories. + #: + #: Ignored patterns are overridden by include patterns; see :py:data:`ENV_PATTERNS_INCLUDE`. + #: + #: :parameter list[str] patterns: list of regular expression patterns. E.g: ``r"(.*/)?ignored_file_name(/.*)?"``. + ENV_PATTERNS_IGNORE = filters.get("env:patterns:ignore") + + #: List of template path patterns to be included when rendering templates to the project root. + #: Patterns from this list will take priority over the patterns from :py:data:`ENV_PATTERNS_IGNORE`. + #: + #: :parameter list[str] patterns: list of regular expression patterns. See :py:data:`ENV_PATTERNS_IGNORE`. + ENV_PATTERNS_INCLUDE = filters.get("env:patterns:include") + #: List of all template root folders. #: #: :parameter list[str] templates_root: absolute paths to folders which contain templates.